From efddf485006559bef5e8791b637e742d865df434 Mon Sep 17 00:00:00 2001
From: cinap_lenrek <cinap_lenrek@felloff.net>
Date: Wed, 15 Mar 2017 19:30:33 +0100
Subject: [PATCH] upas/fs: getting rid of the fine-grain locking...

now that locking works, its time to get rid of it.
given that the only concurrency is between the periodic reader
process and the filesystem code, just make sure the filesystem
rpc handlers do not run concurrenty while syncing is in progress
with a single qlock.
---
 sys/src/cmd/upas/fs/dat.h  |   2 -
 sys/src/cmd/upas/fs/fs.c   | 101 +++++++------------------------------
 sys/src/cmd/upas/fs/mbox.c |  46 ++---------------
 3 files changed, 22 insertions(+), 127 deletions(-)

diff --git a/sys/src/cmd/upas/fs/dat.h b/sys/src/cmd/upas/fs/dat.h
index 0220f09dc..f955fd23c 100644
--- a/sys/src/cmd/upas/fs/dat.h
+++ b/sys/src/cmd/upas/fs/dat.h
@@ -147,7 +147,6 @@ struct Mcache {
 
 typedef struct Mailbox Mailbox;
 struct Mailbox {
-	QLock;
 	int	refs;
 	Mailbox	*next;
 	int	id;
@@ -355,7 +354,6 @@ extern ulong	msgallocd;
 extern ulong	msgfreed;
 extern Mailbox	*mbl;
 extern Message	*root;
-extern QLock	mbllock;
 extern Refs	*rtab;
 
 #define	dprint(...)	if(debug) fprint(2, __VA_ARGS__); else {}
diff --git a/sys/src/cmd/upas/fs/fs.c b/sys/src/cmd/upas/fs/fs.c
index 880d8501f..d6c9b1380 100644
--- a/sys/src/cmd/upas/fs/fs.c
+++ b/sys/src/cmd/upas/fs/fs.c
@@ -109,7 +109,6 @@ enum
 	Hsize=	1999,
 };
 
-
 char	*mntpt;
 char	user[Elemlen];
 int	Dflag;
@@ -121,7 +120,6 @@ int	debug;
 int	plumbing = 1;
 ulong	cachetarg = Maxcache;
 Mailbox	*mbl;
-QLock	mbllock;
 
 static	int	messagesize = 8*1024 + IOHDRSZ;
 static	int	mfd[2];
@@ -133,6 +131,7 @@ static	Hash	*htab[Hsize];
 static	Fcall	rhdr;
 static	Fcall	thdr;
 static	Fid	*fids;
+static QLock	synclock;
 
 void
 sanemsg(Message *m)
@@ -778,17 +777,13 @@ doclone(Fid *f, int nfid)
 	nf->busy = 1;
 	nf->open = 0;
 	nf->mb = f->mb;
-	if(nf->mb){
-		qlock(nf->mb);
+	if(nf->mb)
 		mboxincref(nf->mb);
-	}
 	if(nf->m = f->m)
 		msgincref(gettopmsg(nf->mb, nf->m));
 	if(nf->mtop = f->mtop)
 		msgincref(nf->mtop);
 	nf->qid = f->qid;
-	if(nf->mb)
-		qunlock(nf->mb);
 	return nf;
 }
 
@@ -816,10 +811,6 @@ dowalk(Fid *f, char *name)
 	t = FILE(f->qid.path);
 	rv = Enotexist;
 
-	qlock(&mbllock);
-	if(f->mb)
-		qlock(f->mb);
-
 	/* this must catch everything except . and .. */
 retry:
 	t1 = FILE(f->qid.path);
@@ -833,12 +824,8 @@ retry:
 	if(h != nil){
 		if(f->m)
 			msgdecref(f->mb, gettopmsg(f->mb, f->m));
-		if(f->mb && f->mb != h->mb){
-			qunlock(f->mb);
+		if(f->mb && f->mb != h->mb)
 			mboxdecref(f->mb);
-		}
-		if(h->mb && h->mb != f->mb)
-			qlock(h->mb);
 		f->mb = h->mb;
 		f->m = h->m;
 		if(f->m)
@@ -864,10 +851,6 @@ retry:
 		goto retry;
 	}
 
-	if(f->mb)
-		qunlock(f->mb);
-	qunlock(&mbllock);
-
 	if(rv == nil)
 		return rv;
 
@@ -890,12 +873,9 @@ retry:
 			f->qid.vers = 0;
 			mb = f->mb;
 			f->mb = nil;
-			qlock(&mbllock);
 			mboxdecref(mb);
-			qunlock(&mbllock);
 			break;
 		case Qdir:
-			qlock(f->mb);
 			if(Topmsg(f->mb, f->m)){
 				f->qid.path = PATH(f->mb->id, Qmbox);
 				f->qid.type = QTDIR;
@@ -909,7 +889,6 @@ retry:
 				f->qid.path = PATH(f->m->id, Qdir);
 				f->qid.type = QTDIR;
 			}
-			qunlock(f->mb);
 			break;
 		}
 		rv = nil;
@@ -975,12 +954,10 @@ ropen(Fid *f)
 
 	/* make sure we've decoded */
 	if(file == Qbody){
-		qlock(f->mb);
 		cachebody(f->mb, f->m);
 		decode(f->m);
 		convert(f->m);
 		putcache(f->mb, f->m);
-		qunlock(f->mb);
 	}
 
 	rhdr.iounit = 0;
@@ -1003,24 +980,20 @@ readtopdir(Fid*, uchar *buf, long off, int cnt, int blen)
 	long pos;
 	Mailbox *mb;
 
-	qlock(&mbllock);
-
 	n = 0;
 	pos = 0;
 	mkstat(&d, nil, nil, Qctl);
 	m = convD2M(&d, &buf[n], blen);
 	if(off <= pos){
 		if(m <= BIT16SZ || m > cnt)
-			goto out;
+			return n;
 		n += m;
 		cnt -= m;
 	}
 	pos += m;
 		
 	for(mb = mbl; mb != nil; mb = mb->next){
-		qlock(mb);
 		mkstat(&d, mb, nil, Qmbox);
-		qunlock(mb);
 		m = convD2M(&d, &buf[n], blen - n);
 		if(off <= pos){
 			if(m <= BIT16SZ || m > cnt)
@@ -1030,8 +1003,6 @@ readtopdir(Fid*, uchar *buf, long off, int cnt, int blen)
 		}
 		pos += m;
 	}
-out:
-	qlock(&mbllock);
 	return n;
 }
 
@@ -1043,7 +1014,6 @@ readmboxdir(Fid *f, uchar *buf, long off, int cnt, int blen)
 	long pos;
 	Message *msg;
 
-	qlock(f->mb);
 	if(off == 0)
 		syncmbox(f->mb, 1);
 
@@ -1054,7 +1024,7 @@ readmboxdir(Fid *f, uchar *buf, long off, int cnt, int blen)
 		if(off == 0){
 			if(m <= BIT16SZ || m > cnt){
 				f->fptr = nil;
-				goto out;
+				return n;
 			}
 			n += m;
 			cnt -= m;
@@ -1091,8 +1061,6 @@ readmboxdir(Fid *f, uchar *buf, long off, int cnt, int blen)
 	f->foff = pos;
 	f->fptr = msg;
 	f->fvers = f->mb->vers;
-out:
-	qunlock(f->mb);
 	return n;
 }
 
@@ -1104,7 +1072,6 @@ readmsgdir(Fid *f, uchar *buf, long off, int cnt, int blen)
 	long pos;
 	Message *msg;
 
-	qlock(f->mb);
 	n = 0;
 	pos = 0;
 	for(i = 0; i < Qmax; i++){
@@ -1112,7 +1079,7 @@ readmsgdir(Fid *f, uchar *buf, long off, int cnt, int blen)
 		m = convD2M(&d, &buf[n], blen - n);
 		if(off <= pos){
 			if(m <= BIT16SZ || m > cnt)
-				goto out;
+				return n;
 			n += m;
 			cnt -= m;
 		}
@@ -1129,8 +1096,6 @@ readmsgdir(Fid *f, uchar *buf, long off, int cnt, int blen)
 		}
 		pos += m;
 	}
-out:
-	qunlock(f->mb);
 	return n;
 }
 
@@ -1171,7 +1136,6 @@ rread(Fid *f)
 		return nil;
 	}
 
-	qlock(f->mb);
 	switch(t){
 	case Qctl:
 		rhdr.count = 0;
@@ -1214,7 +1178,6 @@ rread(Fid *f)
 		}
 		break;
 	}
-	qunlock(f->mb);
 	return nil;
 }
 
@@ -1335,10 +1298,7 @@ rwrite(Fid *f)
 			argc = tokenize(thdr.data, argv, nelem(argvbuf));
 			if(argc == 0)
 				return Ebadctl;
-			qlock(f->mb);
-			err = f->mb->ctl(f->mb, argc, argv);
-			qunlock(f->mb);
-			return err;
+			return f->mb->ctl(f->mb, argc, argv);
 		}
 		break;
 	case Qflags:
@@ -1347,11 +1307,8 @@ rwrite(Fid *f)
 		 */
 		if(!f->mb || !f->m)
 			break;
-		qlock(f->mb);
 		m = gettopmsg(f->mb, f->m);
-		err = modflags(f->mb, m, thdr.data);
-		qunlock(f->mb);
-		return err;
+		return modflags(f->mb, m, thdr.data);
 	}
 	return Eperm;
 }
@@ -1366,8 +1323,6 @@ rclunk(Fid *f)
 	f->fid = -1;
 	f->open = 0;
 	mb = f->mb;
-	if(mb)
-		qlock(mb);
 	if(f->mtop)
 		msgdecref(mb, f->mtop);
 	if(f->m)
@@ -1375,10 +1330,7 @@ rclunk(Fid *f)
 	f->m = f->mtop = nil;
 	if(mb){
 		f->mb = nil;
-		qunlock(mb);
-		qlock(&mbllock);
 		mboxdecref(mb);
-		qunlock(&mbllock);
 	}
 	f->busy = 0;
 	return 0;
@@ -1388,11 +1340,9 @@ char *
 rremove(Fid *f)
 {
 	if(f->m != nil){
-		qlock(f->mb);
 		if(!f->m->deleted)
 			mailplumb(f->mb, f->m, 1);
 		f->m->deleted = Deleted;
-		qunlock(f->mb);
 	}
 	return rclunk(f);
 }
@@ -1402,15 +1352,11 @@ rstat(Fid *f)
 {
 	Dir d;
 
-	if(f->mb)
-		qlock(f->mb);
 	if(FILE(f->qid.path) == Qmbox)
 		syncmbox(f->mb, 1);
 	mkstat(&d, f->mb, f->m, FILE(f->qid.path));
 	rhdr.nstat = convD2M(&d, mbuf, messagesize - IOHDRSZ);
 	rhdr.stat = mbuf;
-	if(f->mb)
-		qunlock(f->mb);
 	return 0;
 }
 
@@ -1473,6 +1419,7 @@ io(void)
 		if(Dflag)
 			fprint(2, "%s:<-%F\n", argv0, &thdr);
 
+		qlock(&synclock);
 		rhdr.data = (char*)mdata + messagesize;
 		if(!fcalls[thdr.type])
 			err = "bad fcall type";
@@ -1486,6 +1433,8 @@ io(void)
 			rhdr.fid = thdr.fid;
 		}
 		rhdr.tag = thdr.tag;
+		qunlock(&synclock);
+
 		if(Dflag)
 			fprint(2, "%s:->%F\n", argv0, &rhdr);
 		n = convS2M(&rhdr, mdata, messagesize);
@@ -1506,11 +1455,9 @@ reader(void)
 	setname(readerargv);
 	sleep(15*1000);
 	for(;;){
+		qlock(&synclock);
 		t = time(0);
-		qlock(&mbllock);
 		for(mb = mbl; mb != nil; mb = mb->next){
-			if(!canqlock(mb))
-				continue;
 			if(mb->waketime != 0 && t >= mb->waketime){
 				mb->waketime = 0;
 				break;
@@ -1526,14 +1473,14 @@ reader(void)
 					free(d);
 				}
 			}
-			qunlock(mb);
 		}
-		qunlock(&mbllock);
-		if(mb != nil){
+		if(mb != nil) {
 			syncmbox(mb, 1);
-			qunlock(mb);
-		} else
+			qunlock(&synclock);
+		} else {
+			qunlock(&synclock);
 			sleep(15*1000);
+		}
 	}
 }
 
@@ -1662,8 +1609,6 @@ readheader(Message *m, char *buf, int off, int cnt)
 	return to - buf;
 }
 
-QLock hashlock;
-
 uint
 hash(ulong ppath, char *name)
 {
@@ -1684,14 +1629,10 @@ hlook(ulong ppath, char *name)
 	int h;
 	Hash *hp;
 
-	qlock(&hashlock);
 	h = hash(ppath, name);
 	for(hp = htab[h]; hp != nil; hp = hp->next)
-		if(ppath == hp->ppath && strcmp(name, hp->name) == 0){
-			qunlock(&hashlock);
+		if(ppath == hp->ppath && strcmp(name, hp->name) == 0)
 			return hp;
-		}
-	qunlock(&hashlock);
 	return nil;
 }
 
@@ -1701,7 +1642,6 @@ henter(ulong ppath, char *name, Qid qid, Message *m, Mailbox *mb)
 	int h;
 	Hash *hp, **l;
 
-	qlock(&hashlock);
 	h = hash(ppath, name);
 	for(l = &htab[h]; *l != nil; l = &(*l)->next){
 		hp = *l;
@@ -1709,18 +1649,15 @@ henter(ulong ppath, char *name, Qid qid, Message *m, Mailbox *mb)
 			hp->m = m;
 			hp->mb = mb;
 			hp->qid = qid;
-			qunlock(&hashlock);
 			return;
 		}
 	}
-
 	*l = hp = emalloc(sizeof(*hp));
 	hp->m = m;
 	hp->mb = mb;
 	hp->qid = qid;
 	hp->name = name;
 	hp->ppath = ppath;
-	qunlock(&hashlock);
 }
 
 void
@@ -1729,7 +1666,6 @@ hfree(ulong ppath, char *name)
 	int h;
 	Hash *hp, **l;
 
-	qlock(&hashlock);
 	h = hash(ppath, name);
 	for(l = &htab[h]; *l != nil; l = &(*l)->next){
 		hp = *l;
@@ -1740,7 +1676,6 @@ hfree(ulong ppath, char *name)
 			break;
 		}
 	}
-	qunlock(&hashlock);
 }
 
 void
diff --git a/sys/src/cmd/upas/fs/mbox.c b/sys/src/cmd/upas/fs/mbox.c
index 22a8fedc7..7d05ad731 100644
--- a/sys/src/cmd/upas/fs/mbox.c
+++ b/sys/src/cmd/upas/fs/mbox.c
@@ -65,7 +65,6 @@ syncmbox(Mailbox *mb, int doplumb)
 	int n, d, y, a;
 	Message *m, *next;
 
-	assert(!canqlock(mb));
 	a = mb->root->subname;
 	if(rdidxfile(mb, doplumb) == -2)
 		wridxfile(mb);
@@ -146,9 +145,6 @@ mboxrename(char *a, char *b, int flags)
 	henter(PATH(0, Qtop), mb->name,
 		(Qid){PATH(mb->id, Qmbox), mb->vers, QTDIR}, nil, mb);
 done:
-	if(mb == nil)
-		return err;
-	qunlock(mb);
 	return err;
 }
 
@@ -206,7 +202,6 @@ newmbox(char *path, char *name, int flags, Mailbox **r)
 	}
 
 	/* make sure name isn't taken */
-	qlock(&mbllock);
 	for(l = &mbl; *l != nil; l = &(*l)->next)
 		if(strcmp((*l)->name, mb->name) == 0){
 			if(strcmp(path, (*l)->path) == 0)
@@ -216,7 +211,6 @@ newmbox(char *path, char *name, int flags, Mailbox **r)
 			if(mb->close)
 				mb->close(mb);
 			free(mb);
-			qunlock(&mbllock);
 			return rv;
 		}
 
@@ -229,9 +223,7 @@ newmbox(char *path, char *name, int flags, Mailbox **r)
 	mb->mtree = avlcreate(mtreecmp);
 
 	*l = mb;
-	qunlock(&mbllock);
 
-	qlock(mb);
 	henter(PATH(0, Qtop), mb->name,
 		(Qid){PATH(mb->id, Qmbox), mb->vers, QTDIR}, nil, mb);
 	if(mb->ctl)
@@ -240,8 +232,6 @@ newmbox(char *path, char *name, int flags, Mailbox **r)
 	rv = syncmbox(mb, 0);
 	if(r)
 		*r = mb;
-	else
-		qunlock(mb);
 
 	return rv;
 }
@@ -252,7 +242,6 @@ freembox(char *name)
 {
 	Mailbox **l, *mb;
 
-	qlock(&mbllock);
 	for(l=&mbl; *l != nil; l=&(*l)->next)
 		if(strcmp(name, (*l)->name) == 0){
 			mb = *l;
@@ -261,7 +250,6 @@ freembox(char *name)
 			break;
 		}
 	hfree(PATH(0, Qtop), name);
-	qunlock(&mbllock);
 }
 
 void
@@ -270,14 +258,9 @@ syncallmboxes(void)
 	char *err;
 	Mailbox *m;
 
-	qlock(&mbllock);
-	for(m = mbl; m != nil; m = m->next){
-		qlock(m);
+	for(m = mbl; m != nil; m = m->next)
 		if(err = syncmbox(m, 1))
 			eprint("syncmbox: %s\n", err);
-		qunlock(m);
-	}
-	qunlock(&mbllock);
 }
 
 
@@ -288,7 +271,6 @@ removembox(char *name, int flags)
 	Mailbox **l, *mb;
 
 	found = 0;
-	qlock(&mbllock);
 	for(l=&mbl; *l != nil; l=&(*l)->next)
 		if(strcmp(name, (*l)->path) == 0){
 			mb = *l;
@@ -300,7 +282,6 @@ removembox(char *name, int flags)
 			break;
 		}
 	hfree(PATH(0, Qtop), name);
-	qunlock(&mbllock);
 
 	if(found == 0)
 		return "maibox not found";
@@ -1065,13 +1046,9 @@ delmessages(int ac, char **av)
 	Mailbox *mb;
 	Message *m;
 
-	qlock(&mbllock);
 	for(mb = mbl; mb != nil; mb = mb->next)
-		if(strcmp(av[0], mb->name) == 0){
-			qlock(mb);
+		if(strcmp(av[0], mb->name) == 0)
 			break;
-		}
-	qunlock(&mbllock);
 	if(mb == nil)
 		return "no such mailbox";
 
@@ -1089,7 +1066,6 @@ delmessages(int ac, char **av)
 			}
 	if(needwrite)
 		syncmbox(mb, 1);
-	qunlock(mb);
 	return 0;
 }
 
@@ -1103,13 +1079,9 @@ flagmessages(int argc, char **argv)
 
 	if(argc%2)
 		return "bad flags";
-	qlock(&mbllock);
 	for(mb = mbl; mb; mb = mb->next)
-		if(strcmp(*argv, mb->name) == 0){
-			qlock(mb);
+		if(strcmp(*argv, mb->name) == 0)
 			break;
-		}
-	qunlock(&mbllock);
 	if(mb == nil)
 		return "no such mailbox";
 	needwrite = 0;
@@ -1124,13 +1096,9 @@ flagmessages(int argc, char **argv)
 			}
 	if(needwrite)
 		syncmbox(mb, 1);
-	qunlock(mb);
 	return rerr;
 }
 
-/*
- *  the following are called with the mailbox qlocked
- */
 void
 msgincref(Message *m)
 {
@@ -1140,7 +1108,6 @@ msgincref(Message *m)
 void
 msgdecref(Mailbox *mb, Message *m)
 {
-	assert(!canqlock(mb));
 	assert(m->refs > 0);
 	m->refs--;
 	if(m->refs == 0){
@@ -1151,9 +1118,6 @@ msgdecref(Mailbox *mb, Message *m)
 	}
 }
 
-/*
- *  the following are called with mbllock'd
- */
 void
 mboxincref(Mailbox *mb)
 {
@@ -1177,7 +1141,6 @@ mbrmidx(char *path, int flags)
 void
 mboxdecref(Mailbox *mb)
 {
-	qlock(mb);
 	assert(mb->refs > 0);
 	mb->refs--;
 	if(mb->refs == 0){
@@ -1193,8 +1156,7 @@ mboxdecref(Mailbox *mb)
 		free(mb->mtree);
 		free(mb->d);
 		free(mb);
-	} else
-		qunlock(mb);
+	}
 }