kernel: avoid palloc lock during mmurelease()

Previously, mmurelease() was always called with
palloc spinlock held.

This is unneccesary for some mmurelease()
implementations as they wont release pages
to the palloc pool.

This change removes pagechainhead() and
pagechaindone() and replaces them with just
freepages() call, which aquires the palloc
lock internally as needed.

freepages() avoids holding the palloc lock
while walking the linked list of pages,
avoding some lock contention.

sys/src/9/bcm/mmu.c

@@ -173,17 +173,9 @@ flushmmu(void)
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
+
-		next = page->next;
+	freepages(proc->mmul2cache, nil, 0);
-		if(--page->ref)
-			panic("mmurelease: page->ref %lud", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
 	proc->mmul2cache = nil;
 
 	mmul1empty();

sys/src/9/bcm64/mmu.c

@@ -518,20 +518,10 @@ mmuswitch(Proc *p)
 void
 mmurelease(Proc *p)
 {
-	Page *t;
-
 	mmuswitch(nil);
 	mmufree(p);
-
+	freepages(p->mmufree, nil, 0);
-	if((t = p->mmufree) != nil){
+	p->mmufree = nil;
-		do {
-			p->mmufree = t->next;
-			if(--t->ref != 0)
-				panic("mmurelease: bad page ref");
-			pagechainhead(t);
-		} while((t = p->mmufree) != nil);
-		pagechaindone();
-	}
 }
 
 void

sys/src/9/cycv/mmu.c

@@ -197,23 +197,21 @@ flushmmu(void)
 void
 mmurelease(Proc *proc)
 {
-	Page *p, *n;
+	Page *p;
 
-	if(islo())
-		panic("mmurelease: islo");
-	
 	l1switch(&m->l1, 0);
-	if(proc->kmaptable != nil){
+	if((p = proc->kmaptable) != nil){
+		if(p->ref != 1)
+			panic("mmurelease: kmap ref %ld", p->ref);
 		if(proc->l1 == nil)
 			panic("mmurelease: no l1");
-		if(decref(proc->kmaptable) != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
-		if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
+		if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
-			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
+			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
 		proc->l1->va[L1X(KMAP)] = 0;
-		pagechainhead(proc->kmaptable);
+		p->next = proc->mmufree;
+		proc->mmufree = p;
 		proc->kmaptable = nil;
 	}
 	if(proc->l1 != nil){
@@ -221,14 +219,7 @@ mmurelease(Proc *proc)
 		l1free(proc->l1);
 		proc->l1 = nil;
 	}
-	for(p = proc->mmufree; p != nil; p = n){
+	freepages(proc->mmufree, nil, 0);
-		n = p->next;
-		if(decref(p) != 0)
-			panic("mmurelease: p->ref %ld", p->ref);
-		pagechainhead(p);
-	}
-	if(proc->mmufree != nil)
-		pagechaindone();
 	proc->mmufree = nil;
 }
 

sys/src/9/kw/mmu.c

@@ -252,20 +252,12 @@ flushmmu(void)
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate l1 caches */
 	cacheuwbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
+
-		next = page->next;
+	freepages(proc->mmul2cache, nil, 0);
-		if(--page->ref)
-			panic("mmurelease: page->ref %lud", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
 	proc->mmul2cache = nil;
 
 	mmul1empty();

sys/src/9/omap/mmu.c

@@ -234,20 +234,12 @@ flushmmu(void)
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate l1 caches */
 	cacheuwbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
+
-		next = page->next;
+	freepages(proc->mmul2cache, nil, 0);
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
 	proc->mmul2cache = nil;
 
 	mmul1empty();

sys/src/9/pc/mmu.c

@@ -320,54 +320,48 @@ mmuswitch(Proc* proc)
  *   cleaning any user entries in the pdb (proc->mmupdb);
  *   if there's a pdb put it in the cache of pre-initialised pdb's
  *   for this processor (m->pdbpool) or on the process' free list;
- *   finally, place any pages freed back into the free pool (palloc).
+ *   finally, place any pages freed back into the free pool (freepages).
- * This routine is only called from schedinit() with palloc locked.
  */
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
 	ulong *pdb;
+	Page *page;
 
-	if(islo())
-		panic("mmurelease: islo");
 	taskswitch(PADDR(m->pdb), (ulong)m + BY2PG);
-	if(proc->kmaptable != nil){
+	if((page = proc->kmaptable) != nil){
+		if(page->ref != 1)
+			panic("mmurelease: kmap ref %ld", page->ref);
 		if(proc->mmupdb == nil)
 			panic("mmurelease: no mmupdb");
-		if(--proc->kmaptable->ref != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
 		/*
 		 * remove kmaptable from pdb before putting pdb up for reuse.
 		 */
 		pdb = tmpmap(proc->mmupdb);
-		if(PPN(pdb[PDX(KMAP)]) != proc->kmaptable->pa)
+		if(PPN(pdb[PDX(KMAP)]) != page->pa)
 			panic("mmurelease: bad kmap pde %#.8lux kmap %#.8lux",
-				pdb[PDX(KMAP)], proc->kmaptable->pa);
+				pdb[PDX(KMAP)], page->pa);
 		pdb[PDX(KMAP)] = 0;
 		tmpunmap(pdb);
+
 		/*
 		 * move kmaptable to free list.
 		 */
-		pagechainhead(proc->kmaptable);
+		page->next = proc->mmufree;
+		proc->mmufree = page;
 		proc->kmaptable = nil;
 	}
-	if(proc->mmupdb != nil){
+	if((page = proc->mmupdb) != nil){
 		mmuptefree(proc);
-		mmupdbfree(proc, proc->mmupdb);
+		mmupdbfree(proc, page);
 		proc->mmupdb = nil;
 	}
-	for(page = proc->mmufree; page != nil; page = next){
+	if((page = proc->mmufree) != nil){
-		next = page->next;
+		freepages(page, nil, 0);
-		if(--page->ref != 0)
+		proc->mmufree = nil;
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
 	}
-	if(proc->mmufree != nil)
-		pagechaindone();
-	proc->mmufree = nil;
 	if(proc->ldt != nil){
 		free(proc->ldt);
 		proc->ldt = nil;

sys/src/9/port/devsegment.c

@@ -469,7 +469,7 @@ fixedseg(uintptr va, ulong len)
 {
 	KMap *k;
 	Segment *s;
-	Page **f, *p, *l, *h;
+	Page **f, *p, *l, *h, *t;
 	ulong n, i;
 	int color;
 
@@ -492,12 +492,13 @@ fixedseg(uintptr va, ulong len)
 			continue;
 
 		i = 0;
-		h = nil;
+		h = t = nil;
 		f = &palloc.head;
 		while((p = *f) != nil){
 			if(p > &l[-len] && p <= l){
 				*f = p->next;
-				p->next = h;
+				if((p->next = h) == nil)
+					t = p;
 				h = p;
 				if(++i < len)
 					continue;
@@ -505,15 +506,15 @@ fixedseg(uintptr va, ulong len)
 			}
 			f = &p->next;
 		}
-		palloc.freecount -= i;
 
 		if(i != len){
-			while((p = h) != nil){
+			if(h != nil){
-				h = h->next;
+				t->next = palloc.head;
-				pagechainhead(p);
+				palloc.head = h;
 			}
 			goto Retry;
 		}
+		palloc.freecount -= i;
 		unlock(&palloc);
 
 		p = &l[-len];

sys/src/9/port/page.c

@@ -11,7 +11,7 @@ void
 pageinit(void)
 {
 	int color, i, j;
-	Page *p;
+	Page *p, **t;
 	Pallocmem *pm;
 	vlong m, v, u;
 
@@ -29,8 +29,12 @@ pageinit(void)
 	}
 
 	color = 0;
+	palloc.freecount = 0;
 	palloc.head = nil;
+
+	t = &palloc.head;
 	p = palloc.pages;
+
 	for(i=0; i<nelem(palloc.mem); i++){
 		pm = &palloc.mem[i];
 		for(j=0; j<pm->npage; j++){
@@ -40,7 +44,8 @@ pageinit(void)
 				continue;
 			p->color = color;
 			color = (color+1)%NCOLOR;
-			pagechainhead(p);
+			*t = p, t = &p->next;
+			palloc.freecount++;
 			p++;
 		}
 	}
@@ -65,15 +70,7 @@ pageinit(void)
 	print("%lldM swap\n", v/(1024*1024));
 }
 
-void
+static void
-pagechainhead(Page *p)
-{
-	p->next = palloc.head;
-	palloc.head = p;
-	palloc.freecount++;
-}
-
-void
 pagechaindone(void)
 {
 	if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil)
@@ -85,11 +82,23 @@ pagechaindone(void)
 void
 freepages(Page *head, Page *tail, ulong np)
 {
-	assert(palloc.Lock.p == up);
+	if(head == nil)
+		return;
+	if(tail == nil){
+		tail = head;
+		for(np = 1;; np++){
+			tail->ref = 0;
+			if(tail->next == nil)
+				break;
+			tail = tail->next;
+		}
+	}
+	lock(&palloc);
 	tail->next = palloc.head;
 	palloc.head = head;
 	palloc.freecount += np;
 	pagechaindone();
+	unlock(&palloc);
 }
 
 ulong
@@ -138,11 +147,8 @@ pagereclaim(Image *i)
 	}
 	putimage(i);
 
-	if(np > 0){
+	if(np > 0)
-		lock(&palloc);
 		freepages(fh, ft, np);
-		unlock(&palloc);
-	}
 
 	return np;
 }
@@ -237,11 +243,8 @@ putpage(Page *p)
 		decref(p);
 		return;
 	}
-	if(decref(p) == 0){
+	if(decref(p) == 0)
-		lock(&palloc);
 		freepages(p, p, 1);
-		unlock(&palloc);
-	}
 }
 
 void

sys/src/9/port/portfns.h

@@ -218,8 +218,6 @@ int		okaddr(uintptr, ulong, int);
 int		openmode(ulong);
 Block*		packblock(Block*);
 Block*		padblock(Block*, int);
-void		pagechaindone(void);
-void		pagechainhead(Page*);
 void		pageinit(void);
 ulong		pagereclaim(Image*);
 void		panic(char*, ...);

sys/src/9/port/proc.c

@@ -81,27 +81,21 @@ schedinit(void)		/* never returns */
 		case Moribund:
 			up->state = Dead;
 			edfstop(up);
-			if(up->edf != nil)
+			if(up->edf != nil){
 				free(up->edf);
-			up->edf = nil;
+				up->edf = nil;
+			}
 
-			/*
-			 * Holding locks from pexit:
-			 * 	procalloc
-			 *	palloc
-			 */
 			mmurelease(up);
-			unlock(&palloc);
 
-			updatecpu(up);
+			lock(&procalloc);
 			up->mach = nil;
-
 			up->qnext = procalloc.free;
 			procalloc.free = up;
-
 			/* proc is free now, make sure unlock() wont touch it */
 			up = procalloc.Lock.p = nil;
 			unlock(&procalloc);
+
 			sched();
 		}
 		coherence();
@@ -1223,10 +1217,6 @@ pexit(char *exitstr, int freemem)
 	}
 	qunlock(&up->seglock);
 
-	/* Sched must not loop for these locks */
-	lock(&procalloc);
-	lock(&palloc);
-
 	edfstop(up);
 	up->state = Moribund;
 	sched();

sys/src/9/teg2/mmu.c

@@ -475,20 +475,12 @@ flushmmu(void)
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate caches */
 	l1cache->wbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
+
-		next = page->next;
+	freepages(proc->mmul2cache, nil, 0);
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
 	proc->mmul2cache = nil;
 
 	mmul1empty();

sys/src/9/xen/mmu.c

@@ -282,15 +282,8 @@ mmurelease(Proc* proc)
 		}
 	}
 
-	for(page = proc->mmufree; page; page = next){
+	freepages(proc->mmufree, nil, 0);
-		next = page->next;
+	proc->mmufree = nil;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld\n", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmufree)
-		pagechaindone();
-	proc->mmufree = 0;
 }
 
 static Page*

sys/src/9/zynq/mmu.c

@@ -205,23 +205,22 @@ flushmmu(void)
 void
 mmurelease(Proc *proc)
 {
-	Page *p, *n;
+	Page *p;
 
-	if(islo())
-		panic("mmurelease: islo");
-	
 	l1switch(&m->l1, 0);
-	if(proc->kmaptable != nil){
+	if((p = proc->kmaptable) != nil){
+		if(p->ref != 1)
+			panic("mmurelease: kmap ref %ld", p->ref);
 		if(proc->l1 == nil)
 			panic("mmurelease: no l1");
-		if(decref(proc->kmaptable) != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
-		if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
+		if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
-			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
+			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
 		proc->l1->va[L1X(KMAP)] = 0;
-		pagechainhead(proc->kmaptable);
+
+		p->next = proc->mmufree;
+		proc->mmufree = p;
 		proc->kmaptable = nil;
 	}
 	if(proc->l1 != nil){
@@ -229,14 +228,7 @@ mmurelease(Proc *proc)
 		l1free(proc->l1);
 		proc->l1 = nil;
 	}
-	for(p = proc->mmufree; p != nil; p = n){
+	freepages(proc->mmufree, nil, 0);
-		n = p->next;
-		if(decref(p) != 0)
-			panic("mmurelease: p->ref %ld", p->ref);
-		pagechainhead(p);
-	}
-	if(proc->mmufree != nil)
-		pagechaindone();
 	proc->mmufree = nil;
 }