From 29f60cace1a71edd730c60ddac8dfbd962741038 Mon Sep 17 00:00:00 2001 From: cinap_lenrek Date: Tue, 22 Dec 2020 16:29:55 +0100 Subject: [PATCH] kernel: avoid palloc lock during mmurelease() Previously, mmurelease() was always called with palloc spinlock held. This is unneccesary for some mmurelease() implementations as they wont release pages to the palloc pool. This change removes pagechainhead() and pagechaindone() and replaces them with just freepages() call, which aquires the palloc lock internally as needed. freepages() avoids holding the palloc lock while walking the linked list of pages, avoding some lock contention. --- sys/src/9/bcm/mmu.c | 12 ++--------- sys/src/9/bcm64/mmu.c | 14 ++---------- sys/src/9/cycv/mmu.c | 27 ++++++++--------------- sys/src/9/kw/mmu.c | 12 ++--------- sys/src/9/omap/mmu.c | 12 ++--------- sys/src/9/pc/mmu.c | 36 +++++++++++++------------------ sys/src/9/port/devsegment.c | 15 +++++++------ sys/src/9/port/page.c | 43 ++++++++++++++++++++----------------- sys/src/9/port/portfns.h | 2 -- sys/src/9/port/proc.c | 20 +++++------------ sys/src/9/teg2/mmu.c | 12 ++--------- sys/src/9/xen/mmu.c | 11 ++-------- sys/src/9/zynq/mmu.c | 28 +++++++++--------------- 13 files changed, 82 insertions(+), 162 deletions(-) diff --git a/sys/src/9/bcm/mmu.c b/sys/src/9/bcm/mmu.c index 864757079..655d847ae 100644 --- a/sys/src/9/bcm/mmu.c +++ b/sys/src/9/bcm/mmu.c @@ -173,17 +173,9 @@ flushmmu(void) void mmurelease(Proc* proc) { - Page *page, *next; - mmul2empty(proc, 0); - for(page = proc->mmul2cache; page != nil; page = next){ - next = page->next; - if(--page->ref) - panic("mmurelease: page->ref %lud", page->ref); - pagechainhead(page); - } - if(proc->mmul2cache != nil) - pagechaindone(); + + freepages(proc->mmul2cache, nil, 0); proc->mmul2cache = nil; mmul1empty(); diff --git a/sys/src/9/bcm64/mmu.c b/sys/src/9/bcm64/mmu.c index 4c664554b..8538369fa 100644 --- a/sys/src/9/bcm64/mmu.c +++ b/sys/src/9/bcm64/mmu.c @@ -518,20 +518,10 @@ mmuswitch(Proc *p) void mmurelease(Proc *p) { - Page *t; - mmuswitch(nil); mmufree(p); - - if((t = p->mmufree) != nil){ - do { - p->mmufree = t->next; - if(--t->ref != 0) - panic("mmurelease: bad page ref"); - pagechainhead(t); - } while((t = p->mmufree) != nil); - pagechaindone(); - } + freepages(p->mmufree, nil, 0); + p->mmufree = nil; } void diff --git a/sys/src/9/cycv/mmu.c b/sys/src/9/cycv/mmu.c index 8a449e70b..213a88178 100644 --- a/sys/src/9/cycv/mmu.c +++ b/sys/src/9/cycv/mmu.c @@ -197,23 +197,21 @@ flushmmu(void) void mmurelease(Proc *proc) { - Page *p, *n; + Page *p; - if(islo()) - panic("mmurelease: islo"); - l1switch(&m->l1, 0); - if(proc->kmaptable != nil){ + if((p = proc->kmaptable) != nil){ + if(p->ref != 1) + panic("mmurelease: kmap ref %ld", p->ref); if(proc->l1 == nil) panic("mmurelease: no l1"); - if(decref(proc->kmaptable) != 0) - panic("mmurelease: kmap ref %ld", proc->kmaptable->ref); if(proc->nkmap) panic("mmurelease: nkmap %d", proc->nkmap); - if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa) - panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa); + if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa) + panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa); proc->l1->va[L1X(KMAP)] = 0; - pagechainhead(proc->kmaptable); + p->next = proc->mmufree; + proc->mmufree = p; proc->kmaptable = nil; } if(proc->l1 != nil){ @@ -221,14 +219,7 @@ mmurelease(Proc *proc) l1free(proc->l1); proc->l1 = nil; } - for(p = proc->mmufree; p != nil; p = n){ - n = p->next; - if(decref(p) != 0) - panic("mmurelease: p->ref %ld", p->ref); - pagechainhead(p); - } - if(proc->mmufree != nil) - pagechaindone(); + freepages(proc->mmufree, nil, 0); proc->mmufree = nil; } diff --git a/sys/src/9/kw/mmu.c b/sys/src/9/kw/mmu.c index bab4238fd..0696bff35 100644 --- a/sys/src/9/kw/mmu.c +++ b/sys/src/9/kw/mmu.c @@ -252,20 +252,12 @@ flushmmu(void) void mmurelease(Proc* proc) { - Page *page, *next; - /* write back dirty and invalidate l1 caches */ cacheuwbinv(); mmul2empty(proc, 0); - for(page = proc->mmul2cache; page != nil; page = next){ - next = page->next; - if(--page->ref) - panic("mmurelease: page->ref %lud", page->ref); - pagechainhead(page); - } - if(proc->mmul2cache != nil) - pagechaindone(); + + freepages(proc->mmul2cache, nil, 0); proc->mmul2cache = nil; mmul1empty(); diff --git a/sys/src/9/omap/mmu.c b/sys/src/9/omap/mmu.c index d427f1d76..12383c962 100644 --- a/sys/src/9/omap/mmu.c +++ b/sys/src/9/omap/mmu.c @@ -234,20 +234,12 @@ flushmmu(void) void mmurelease(Proc* proc) { - Page *page, *next; - /* write back dirty and invalidate l1 caches */ cacheuwbinv(); mmul2empty(proc, 0); - for(page = proc->mmul2cache; page != nil; page = next){ - next = page->next; - if(--page->ref) - panic("mmurelease: page->ref %ld", page->ref); - pagechainhead(page); - } - if(proc->mmul2cache != nil) - pagechaindone(); + + freepages(proc->mmul2cache, nil, 0); proc->mmul2cache = nil; mmul1empty(); diff --git a/sys/src/9/pc/mmu.c b/sys/src/9/pc/mmu.c index df0c23520..a21c94197 100644 --- a/sys/src/9/pc/mmu.c +++ b/sys/src/9/pc/mmu.c @@ -320,54 +320,48 @@ mmuswitch(Proc* proc) * cleaning any user entries in the pdb (proc->mmupdb); * if there's a pdb put it in the cache of pre-initialised pdb's * for this processor (m->pdbpool) or on the process' free list; - * finally, place any pages freed back into the free pool (palloc). - * This routine is only called from schedinit() with palloc locked. + * finally, place any pages freed back into the free pool (freepages). */ void mmurelease(Proc* proc) { - Page *page, *next; ulong *pdb; + Page *page; - if(islo()) - panic("mmurelease: islo"); taskswitch(PADDR(m->pdb), (ulong)m + BY2PG); - if(proc->kmaptable != nil){ + if((page = proc->kmaptable) != nil){ + if(page->ref != 1) + panic("mmurelease: kmap ref %ld", page->ref); if(proc->mmupdb == nil) panic("mmurelease: no mmupdb"); - if(--proc->kmaptable->ref != 0) - panic("mmurelease: kmap ref %ld", proc->kmaptable->ref); if(proc->nkmap) panic("mmurelease: nkmap %d", proc->nkmap); /* * remove kmaptable from pdb before putting pdb up for reuse. */ pdb = tmpmap(proc->mmupdb); - if(PPN(pdb[PDX(KMAP)]) != proc->kmaptable->pa) + if(PPN(pdb[PDX(KMAP)]) != page->pa) panic("mmurelease: bad kmap pde %#.8lux kmap %#.8lux", - pdb[PDX(KMAP)], proc->kmaptable->pa); + pdb[PDX(KMAP)], page->pa); pdb[PDX(KMAP)] = 0; tmpunmap(pdb); + /* * move kmaptable to free list. */ - pagechainhead(proc->kmaptable); + page->next = proc->mmufree; + proc->mmufree = page; proc->kmaptable = nil; } - if(proc->mmupdb != nil){ + if((page = proc->mmupdb) != nil){ mmuptefree(proc); - mmupdbfree(proc, proc->mmupdb); + mmupdbfree(proc, page); proc->mmupdb = nil; } - for(page = proc->mmufree; page != nil; page = next){ - next = page->next; - if(--page->ref != 0) - panic("mmurelease: page->ref %ld", page->ref); - pagechainhead(page); + if((page = proc->mmufree) != nil){ + freepages(page, nil, 0); + proc->mmufree = nil; } - if(proc->mmufree != nil) - pagechaindone(); - proc->mmufree = nil; if(proc->ldt != nil){ free(proc->ldt); proc->ldt = nil; diff --git a/sys/src/9/port/devsegment.c b/sys/src/9/port/devsegment.c index fcaf8f3e4..dfbd34fbf 100644 --- a/sys/src/9/port/devsegment.c +++ b/sys/src/9/port/devsegment.c @@ -469,7 +469,7 @@ fixedseg(uintptr va, ulong len) { KMap *k; Segment *s; - Page **f, *p, *l, *h; + Page **f, *p, *l, *h, *t; ulong n, i; int color; @@ -492,12 +492,13 @@ fixedseg(uintptr va, ulong len) continue; i = 0; - h = nil; + h = t = nil; f = &palloc.head; while((p = *f) != nil){ if(p > &l[-len] && p <= l){ *f = p->next; - p->next = h; + if((p->next = h) == nil) + t = p; h = p; if(++i < len) continue; @@ -505,15 +506,15 @@ fixedseg(uintptr va, ulong len) } f = &p->next; } - palloc.freecount -= i; if(i != len){ - while((p = h) != nil){ - h = h->next; - pagechainhead(p); + if(h != nil){ + t->next = palloc.head; + palloc.head = h; } goto Retry; } + palloc.freecount -= i; unlock(&palloc); p = &l[-len]; diff --git a/sys/src/9/port/page.c b/sys/src/9/port/page.c index 0a039c099..b7194058e 100644 --- a/sys/src/9/port/page.c +++ b/sys/src/9/port/page.c @@ -11,7 +11,7 @@ void pageinit(void) { int color, i, j; - Page *p; + Page *p, **t; Pallocmem *pm; vlong m, v, u; @@ -29,8 +29,12 @@ pageinit(void) } color = 0; + palloc.freecount = 0; palloc.head = nil; + + t = &palloc.head; p = palloc.pages; + for(i=0; inpage; j++){ @@ -40,7 +44,8 @@ pageinit(void) continue; p->color = color; color = (color+1)%NCOLOR; - pagechainhead(p); + *t = p, t = &p->next; + palloc.freecount++; p++; } } @@ -65,15 +70,7 @@ pageinit(void) print("%lldM swap\n", v/(1024*1024)); } -void -pagechainhead(Page *p) -{ - p->next = palloc.head; - palloc.head = p; - palloc.freecount++; -} - -void +static void pagechaindone(void) { if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil) @@ -85,11 +82,23 @@ pagechaindone(void) void freepages(Page *head, Page *tail, ulong np) { - assert(palloc.Lock.p == up); + if(head == nil) + return; + if(tail == nil){ + tail = head; + for(np = 1;; np++){ + tail->ref = 0; + if(tail->next == nil) + break; + tail = tail->next; + } + } + lock(&palloc); tail->next = palloc.head; palloc.head = head; palloc.freecount += np; pagechaindone(); + unlock(&palloc); } ulong @@ -138,11 +147,8 @@ pagereclaim(Image *i) } putimage(i); - if(np > 0){ - lock(&palloc); + if(np > 0) freepages(fh, ft, np); - unlock(&palloc); - } return np; } @@ -237,11 +243,8 @@ putpage(Page *p) decref(p); return; } - if(decref(p) == 0){ - lock(&palloc); + if(decref(p) == 0) freepages(p, p, 1); - unlock(&palloc); - } } void diff --git a/sys/src/9/port/portfns.h b/sys/src/9/port/portfns.h index d1d2c485c..7446513be 100644 --- a/sys/src/9/port/portfns.h +++ b/sys/src/9/port/portfns.h @@ -218,8 +218,6 @@ int okaddr(uintptr, ulong, int); int openmode(ulong); Block* packblock(Block*); Block* padblock(Block*, int); -void pagechaindone(void); -void pagechainhead(Page*); void pageinit(void); ulong pagereclaim(Image*); void panic(char*, ...); diff --git a/sys/src/9/port/proc.c b/sys/src/9/port/proc.c index a51232d44..567ab0756 100644 --- a/sys/src/9/port/proc.c +++ b/sys/src/9/port/proc.c @@ -81,27 +81,21 @@ schedinit(void) /* never returns */ case Moribund: up->state = Dead; edfstop(up); - if(up->edf != nil) + if(up->edf != nil){ free(up->edf); - up->edf = nil; + up->edf = nil; + } - /* - * Holding locks from pexit: - * procalloc - * palloc - */ mmurelease(up); - unlock(&palloc); - updatecpu(up); + lock(&procalloc); up->mach = nil; - up->qnext = procalloc.free; procalloc.free = up; - /* proc is free now, make sure unlock() wont touch it */ up = procalloc.Lock.p = nil; unlock(&procalloc); + sched(); } coherence(); @@ -1223,10 +1217,6 @@ pexit(char *exitstr, int freemem) } qunlock(&up->seglock); - /* Sched must not loop for these locks */ - lock(&procalloc); - lock(&palloc); - edfstop(up); up->state = Moribund; sched(); diff --git a/sys/src/9/teg2/mmu.c b/sys/src/9/teg2/mmu.c index 871e1c1c3..6005e5065 100644 --- a/sys/src/9/teg2/mmu.c +++ b/sys/src/9/teg2/mmu.c @@ -475,20 +475,12 @@ flushmmu(void) void mmurelease(Proc* proc) { - Page *page, *next; - /* write back dirty and invalidate caches */ l1cache->wbinv(); mmul2empty(proc, 0); - for(page = proc->mmul2cache; page != nil; page = next){ - next = page->next; - if(--page->ref) - panic("mmurelease: page->ref %ld", page->ref); - pagechainhead(page); - } - if(proc->mmul2cache != nil) - pagechaindone(); + + freepages(proc->mmul2cache, nil, 0); proc->mmul2cache = nil; mmul1empty(); diff --git a/sys/src/9/xen/mmu.c b/sys/src/9/xen/mmu.c index 1a5c3d067..250d08371 100644 --- a/sys/src/9/xen/mmu.c +++ b/sys/src/9/xen/mmu.c @@ -282,15 +282,8 @@ mmurelease(Proc* proc) } } - for(page = proc->mmufree; page; page = next){ - next = page->next; - if(--page->ref) - panic("mmurelease: page->ref %ld\n", page->ref); - pagechainhead(page); - } - if(proc->mmufree) - pagechaindone(); - proc->mmufree = 0; + freepages(proc->mmufree, nil, 0); + proc->mmufree = nil; } static Page* diff --git a/sys/src/9/zynq/mmu.c b/sys/src/9/zynq/mmu.c index ed6412e70..9b4c67d4c 100644 --- a/sys/src/9/zynq/mmu.c +++ b/sys/src/9/zynq/mmu.c @@ -205,23 +205,22 @@ flushmmu(void) void mmurelease(Proc *proc) { - Page *p, *n; + Page *p; - if(islo()) - panic("mmurelease: islo"); - l1switch(&m->l1, 0); - if(proc->kmaptable != nil){ + if((p = proc->kmaptable) != nil){ + if(p->ref != 1) + panic("mmurelease: kmap ref %ld", p->ref); if(proc->l1 == nil) panic("mmurelease: no l1"); - if(decref(proc->kmaptable) != 0) - panic("mmurelease: kmap ref %ld", proc->kmaptable->ref); if(proc->nkmap) panic("mmurelease: nkmap %d", proc->nkmap); - if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa) - panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa); + if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa) + panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa); proc->l1->va[L1X(KMAP)] = 0; - pagechainhead(proc->kmaptable); + + p->next = proc->mmufree; + proc->mmufree = p; proc->kmaptable = nil; } if(proc->l1 != nil){ @@ -229,14 +228,7 @@ mmurelease(Proc *proc) l1free(proc->l1); proc->l1 = nil; } - for(p = proc->mmufree; p != nil; p = n){ - n = p->next; - if(decref(p) != 0) - panic("mmurelease: p->ref %ld", p->ref); - pagechainhead(p); - } - if(proc->mmufree != nil) - pagechaindone(); + freepages(proc->mmufree, nil, 0); proc->mmufree = nil; }