xfnw/plan9fox
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

kernel: avoid palloc lock during mmurelease()

Browse source 
Previously, mmurelease() was always called with
palloc spinlock held.

This is unneccesary for some mmurelease()
implementations as they wont release pages
to the palloc pool.

This change removes pagechainhead() and
pagechaindone() and replaces them with just
freepages() call, which aquires the palloc
lock internally as needed.

freepages() avoids holding the palloc lock
while walking the linked list of pages,
avoding some lock contention.
This commit is contained in:
cinap_lenrek 2020-12-22 16:29:55 +01:00
parent 2fb5fbbd73
commit 29f60cace1
13 changed files with 82 additions and 162 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
sys/src/9/bcm/mmu.c
View file

@ -173,17 +173,9 @@ flushmmu(void)
void
mmurelease(Proc* proc)
{
Page *page, *next;
mmul2empty(proc, 0);
for(page = proc->mmul2cache; page != nil; page = next){
next = page->next;
if(--page->ref)
panic("mmurelease: page->ref %lud", page->ref);
pagechainhead(page);
}
if(proc->mmul2cache != nil)
pagechaindone();
freepages(proc->mmul2cache, nil, 0);
proc->mmul2cache = nil;
mmul1empty();

14
sys/src/9/bcm64/mmu.c
View file

@ -518,20 +518,10 @@ mmuswitch(Proc *p)
void
mmurelease(Proc *p)
{
Page *t;
mmuswitch(nil);
mmufree(p);
if((t = p->mmufree) != nil){
do {
p->mmufree = t->next;
if(--t->ref != 0)
panic("mmurelease: bad page ref");
pagechainhead(t);
} while((t = p->mmufree) != nil);
pagechaindone();
}
freepages(p->mmufree, nil, 0);
p->mmufree = nil;
}
void

27
sys/src/9/cycv/mmu.c
View file

@ -197,23 +197,21 @@ flushmmu(void)
void
mmurelease(Proc *proc)
{
Page *p, *n;
Page *p;
if(islo())
panic("mmurelease: islo");
l1switch(&m->l1, 0);
if(proc->kmaptable != nil){
if((p = proc->kmaptable) != nil){
if(p->ref != 1)
panic("mmurelease: kmap ref %ld", p->ref);
if(proc->l1 == nil)
panic("mmurelease: no l1");
if(decref(proc->kmaptable) != 0)
panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
if(proc->nkmap)
panic("mmurelease: nkmap %d", proc->nkmap);
if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
proc->l1->va[L1X(KMAP)] = 0;
pagechainhead(proc->kmaptable);
p->next = proc->mmufree;
proc->mmufree = p;
proc->kmaptable = nil;
}
if(proc->l1 != nil){
@ -221,14 +219,7 @@ mmurelease(Proc *proc)
l1free(proc->l1);
proc->l1 = nil;
}
for(p = proc->mmufree; p != nil; p = n){
n = p->next;
if(decref(p) != 0)
panic("mmurelease: p->ref %ld", p->ref);
pagechainhead(p);
}
if(proc->mmufree != nil)
pagechaindone();
freepages(proc->mmufree, nil, 0);
proc->mmufree = nil;
}

12
sys/src/9/kw/mmu.c
View file

@ -252,20 +252,12 @@ flushmmu(void)
void
mmurelease(Proc* proc)
{
Page *page, *next;
/* write back dirty and invalidate l1 caches */
cacheuwbinv();
mmul2empty(proc, 0);
for(page = proc->mmul2cache; page != nil; page = next){
next = page->next;
if(--page->ref)
panic("mmurelease: page->ref %lud", page->ref);
pagechainhead(page);
}
if(proc->mmul2cache != nil)
pagechaindone();
freepages(proc->mmul2cache, nil, 0);
proc->mmul2cache = nil;
mmul1empty();

12
sys/src/9/omap/mmu.c
View file

@ -234,20 +234,12 @@ flushmmu(void)
void
mmurelease(Proc* proc)
{
Page *page, *next;
/* write back dirty and invalidate l1 caches */
cacheuwbinv();
mmul2empty(proc, 0);
for(page = proc->mmul2cache; page != nil; page = next){
next = page->next;
if(--page->ref)
panic("mmurelease: page->ref %ld", page->ref);
pagechainhead(page);
}
if(proc->mmul2cache != nil)
pagechaindone();
freepages(proc->mmul2cache, nil, 0);
proc->mmul2cache = nil;
mmul1empty();

36
sys/src/9/pc/mmu.c
View file

@ -320,54 +320,48 @@ mmuswitch(Proc* proc)
* cleaning any user entries in the pdb (proc->mmupdb);
* if there's a pdb put it in the cache of pre-initialised pdb's
* for this processor (m->pdbpool) or on the process' free list;
* finally, place any pages freed back into the free pool (palloc).
* This routine is only called from schedinit() with palloc locked.
* finally, place any pages freed back into the free pool (freepages).
*/
void
mmurelease(Proc* proc)
{
Page *page, *next;
ulong *pdb;
Page *page;
if(islo())
panic("mmurelease: islo");
taskswitch(PADDR(m->pdb), (ulong)m + BY2PG);
if(proc->kmaptable != nil){
if((page = proc->kmaptable) != nil){
if(page->ref != 1)
panic("mmurelease: kmap ref %ld", page->ref);
if(proc->mmupdb == nil)
panic("mmurelease: no mmupdb");
if(--proc->kmaptable->ref != 0)
panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
if(proc->nkmap)
panic("mmurelease: nkmap %d", proc->nkmap);
/*
* remove kmaptable from pdb before putting pdb up for reuse.
*/
pdb = tmpmap(proc->mmupdb);
if(PPN(pdb[PDX(KMAP)]) != proc->kmaptable->pa)
if(PPN(pdb[PDX(KMAP)]) != page->pa)
panic("mmurelease: bad kmap pde %#.8lux kmap %#.8lux",
pdb[PDX(KMAP)], proc->kmaptable->pa);
pdb[PDX(KMAP)], page->pa);
pdb[PDX(KMAP)] = 0;
tmpunmap(pdb);
/*
* move kmaptable to free list.
*/
pagechainhead(proc->kmaptable);
page->next = proc->mmufree;
proc->mmufree = page;
proc->kmaptable = nil;
}
if(proc->mmupdb != nil){
if((page = proc->mmupdb) != nil){
mmuptefree(proc);
mmupdbfree(proc, proc->mmupdb);
mmupdbfree(proc, page);
proc->mmupdb = nil;
}
for(page = proc->mmufree; page != nil; page = next){
next = page->next;
if(--page->ref != 0)
panic("mmurelease: page->ref %ld", page->ref);
pagechainhead(page);
if((page = proc->mmufree) != nil){
freepages(page, nil, 0);
proc->mmufree = nil;
}
if(proc->mmufree != nil)
pagechaindone();
proc->mmufree = nil;
if(proc->ldt != nil){
free(proc->ldt);
proc->ldt = nil;

15
sys/src/9/port/devsegment.c
View file

@ -469,7 +469,7 @@ fixedseg(uintptr va, ulong len)
{
KMap *k;
Segment *s;
Page **f, *p, *l, *h;
Page **f, *p, *l, *h, *t;
ulong n, i;
int color;
@ -492,12 +492,13 @@ fixedseg(uintptr va, ulong len)
continue;
i = 0;
h = nil;
h = t = nil;
f = &palloc.head;
while((p = *f) != nil){
if(p > &l[-len] && p <= l){
*f = p->next;
p->next = h;
if((p->next = h) == nil)
t = p;
h = p;
if(++i < len)
continue;
@ -505,15 +506,15 @@ fixedseg(uintptr va, ulong len)
}
f = &p->next;
}
palloc.freecount -= i;
if(i != len){
while((p = h) != nil){
h = h->next;
pagechainhead(p);
if(h != nil){
t->next = palloc.head;
palloc.head = h;
}
goto Retry;
}
palloc.freecount -= i;
unlock(&palloc);
p = &l[-len];

43
sys/src/9/port/page.c
View file

@ -11,7 +11,7 @@ void
pageinit(void)
{
int color, i, j;
Page *p;
Page *p, **t;
Pallocmem *pm;
vlong m, v, u;
@ -29,8 +29,12 @@ pageinit(void)
}
color = 0;
palloc.freecount = 0;
palloc.head = nil;
t = &palloc.head;
p = palloc.pages;
for(i=0; i<nelem(palloc.mem); i++){
pm = &palloc.mem[i];
for(j=0; j<pm->npage; j++){
@ -40,7 +44,8 @@ pageinit(void)
continue;
p->color = color;
color = (color+1)%NCOLOR;
pagechainhead(p);
*t = p, t = &p->next;
palloc.freecount++;
p++;
}
}
@ -65,15 +70,7 @@ pageinit(void)
print("%lldM swap\n", v/(1024*1024));
}
void
pagechainhead(Page *p)
{
p->next = palloc.head;
palloc.head = p;
palloc.freecount++;
}
void
static void
pagechaindone(void)
{
if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil)
@ -85,11 +82,23 @@ pagechaindone(void)
void
freepages(Page *head, Page *tail, ulong np)
{
assert(palloc.Lock.p == up);
if(head == nil)
return;
if(tail == nil){
tail = head;
for(np = 1;; np++){
tail->ref = 0;
if(tail->next == nil)
break;
tail = tail->next;
}
}
lock(&palloc);
tail->next = palloc.head;
palloc.head = head;
palloc.freecount += np;
pagechaindone();
unlock(&palloc);
}
ulong
@ -138,11 +147,8 @@ pagereclaim(Image *i)
}
putimage(i);
if(np > 0){
lock(&palloc);
if(np > 0)
freepages(fh, ft, np);
unlock(&palloc);
}
return np;
}
@ -237,11 +243,8 @@ putpage(Page *p)
decref(p);
return;
}
if(decref(p) == 0){
lock(&palloc);
if(decref(p) == 0)
freepages(p, p, 1);
unlock(&palloc);
}
}
void

2
sys/src/9/port/portfns.h
View file

@ -218,8 +218,6 @@ int okaddr(uintptr, ulong, int);
int openmode(ulong);
Block* packblock(Block*);
Block* padblock(Block*, int);
void pagechaindone(void);
void pagechainhead(Page*);
void pageinit(void);
ulong pagereclaim(Image*);
void panic(char*, ...);

20
sys/src/9/port/proc.c
View file

@ -81,27 +81,21 @@ schedinit(void) /* never returns */
case Moribund:
up->state = Dead;
edfstop(up);
if(up->edf != nil)
if(up->edf != nil){
free(up->edf);
up->edf = nil;
up->edf = nil;
}
/*
* Holding locks from pexit:
* procalloc
* palloc
*/
mmurelease(up);
unlock(&palloc);
updatecpu(up);
lock(&procalloc);
up->mach = nil;
up->qnext = procalloc.free;
procalloc.free = up;
/* proc is free now, make sure unlock() wont touch it */
up = procalloc.Lock.p = nil;
unlock(&procalloc);
sched();
}
coherence();
@ -1223,10 +1217,6 @@ pexit(char *exitstr, int freemem)
}
qunlock(&up->seglock);
/* Sched must not loop for these locks */
lock(&procalloc);
lock(&palloc);
edfstop(up);
up->state = Moribund;
sched();

12
sys/src/9/teg2/mmu.c
View file

@ -475,20 +475,12 @@ flushmmu(void)
void
mmurelease(Proc* proc)
{
Page *page, *next;
/* write back dirty and invalidate caches */
l1cache->wbinv();
mmul2empty(proc, 0);
for(page = proc->mmul2cache; page != nil; page = next){
next = page->next;
if(--page->ref)
panic("mmurelease: page->ref %ld", page->ref);
pagechainhead(page);
}
if(proc->mmul2cache != nil)
pagechaindone();
freepages(proc->mmul2cache, nil, 0);
proc->mmul2cache = nil;
mmul1empty();

11
sys/src/9/xen/mmu.c
View file

@ -282,15 +282,8 @@ mmurelease(Proc* proc)
}
}
for(page = proc->mmufree; page; page = next){
next = page->next;
if(--page->ref)
panic("mmurelease: page->ref %ld\n", page->ref);
pagechainhead(page);
}
if(proc->mmufree)
pagechaindone();
proc->mmufree = 0;
freepages(proc->mmufree, nil, 0);
proc->mmufree = nil;
}
static Page*

28
sys/src/9/zynq/mmu.c
View file

@ -205,23 +205,22 @@ flushmmu(void)
void
mmurelease(Proc *proc)
{
Page *p, *n;
Page *p;
if(islo())
panic("mmurelease: islo");
l1switch(&m->l1, 0);
if(proc->kmaptable != nil){
if((p = proc->kmaptable) != nil){
if(p->ref != 1)
panic("mmurelease: kmap ref %ld", p->ref);
if(proc->l1 == nil)
panic("mmurelease: no l1");
if(decref(proc->kmaptable) != 0)
panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
if(proc->nkmap)
panic("mmurelease: nkmap %d", proc->nkmap);
if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
proc->l1->va[L1X(KMAP)] = 0;
pagechainhead(proc->kmaptable);
p->next = proc->mmufree;
proc->mmufree = p;
proc->kmaptable = nil;
}
if(proc->l1 != nil){
@ -229,14 +228,7 @@ mmurelease(Proc *proc)
l1free(proc->l1);
proc->l1 = nil;
}
for(p = proc->mmufree; p != nil; p = n){
n = p->next;
if(decref(p) != 0)
panic("mmurelease: p->ref %ld", p->ref);
pagechainhead(p);
}
if(proc->mmufree != nil)
pagechaindone();
freepages(proc->mmufree, nil, 0);
proc->mmufree = nil;
}