![cinap_lenrek](/assets/img/avatar_default.png)
To avoid a MAXMACH limit of 32 and make txtflush into an array for the bitmap. Provide portable macros for testing and clearing the bits: needtxtflush(), donetxtflush(). On pc/pc64, define inittxtflush()/settxtflush() as no-op macros, avoiding the storage overhead of the txtflush array alltogether.
470 lines
8.6 KiB
C
470 lines
8.6 KiB
C
#include "u.h"
|
|
#include "../port/lib.h"
|
|
#include "mem.h"
|
|
#include "dat.h"
|
|
#include "fns.h"
|
|
#include "ureg.h"
|
|
|
|
/*
|
|
* tlb entry 0 is used only by mmuswitch() to set the current tlb pid.
|
|
*
|
|
* It is apparently assumed that user tlb entries are not
|
|
* overwritten during start-up, so ...
|
|
* During system start-up (before up first becomes non-nil),
|
|
* Kmap entries start at tlb index 1 and work their way up until
|
|
* kmapinval() removes them. They then restart at 1. As long as there
|
|
* are few kmap entries they will not pass tlbroff (the WIRED tlb entry
|
|
* limit) and interfere with user tlb entries.
|
|
* Once start-up is over, we combine the kernel and user tlb pools into one,
|
|
* in the hope of making better use of the tlb on systems with small ones.
|
|
*
|
|
* All invalidations of the tlb are via indexed entries. The virtual
|
|
* address used is always 'KZERO | (x<<(PGSHIFT+1) | currentpid' where
|
|
* 'x' is the index into the tlb. This ensures that the current pid doesn't
|
|
* change and that no two invalidated entries have matching virtual
|
|
* addresses just in case SGI/MIPS ever makes a chip that cares (as
|
|
* they keep threatening). These entries should never be used in
|
|
* lookups since accesses to KZERO addresses don't go through the tlb
|
|
* (actually only true of KSEG0 and KSEG1; KSEG2 and KSEG3 do go
|
|
* through the tlb).
|
|
*/
|
|
|
|
#define TLBINVAL(x, pid) puttlbx(x, KZERO|((x)<<(PGSHIFT+1))|(pid), 0, 0, PGSZ)
|
|
|
|
enum {
|
|
Debugswitch = 0,
|
|
Debughash = 0,
|
|
};
|
|
|
|
static ulong ktime[8]; /* only for first 8 cpus */
|
|
|
|
void
|
|
tlbinit(void)
|
|
{
|
|
int i;
|
|
|
|
for(i=0; i<NTLB; i++)
|
|
TLBINVAL(i, 0);
|
|
}
|
|
|
|
Lock kmaplock;
|
|
KMap kpte[KPTESIZE];
|
|
KMap* kmapfree;
|
|
|
|
static int minfree = KPTESIZE;
|
|
static int lastfree;
|
|
static int tlbroff = TLBROFF;
|
|
|
|
static void
|
|
nfree(void)
|
|
{
|
|
int i;
|
|
KMap *k;
|
|
|
|
i = 0;
|
|
for(k=kmapfree; k; k=k->next)
|
|
i++;
|
|
if(i<minfree){
|
|
iprint("%d free\n", i);
|
|
minfree = i;
|
|
}
|
|
lastfree = i;
|
|
}
|
|
|
|
void
|
|
kmapinit(void)
|
|
{
|
|
KMap *k, *klast;
|
|
|
|
lock(&kmaplock);
|
|
kmapfree = kpte;
|
|
klast = &kpte[KPTESIZE-1];
|
|
for(k=kpte; k<klast; k++)
|
|
k->next = k+1;
|
|
k->next = 0;
|
|
unlock(&kmaplock);
|
|
|
|
m->ktlbnext = TLBOFF;
|
|
}
|
|
|
|
void
|
|
kmapdump(void)
|
|
{
|
|
int i;
|
|
|
|
for(i=0; i<KPTESIZE; i++)
|
|
iprint("%d: %lud pc=%#lux\n", i, kpte[i].ref, kpte[i].pc);
|
|
}
|
|
|
|
static int
|
|
putktlb(KMap *k)
|
|
{
|
|
int x;
|
|
ulong virt;
|
|
ulong tlbent[3];
|
|
|
|
virt = k->virt & ~BY2PG | TLBPID(tlbvirt());
|
|
x = gettlbp(virt, tlbent);
|
|
if (!m->paststartup)
|
|
if (up) { /* startup just ended? */
|
|
tlbroff = 1;
|
|
setwired(tlbroff); /* share all-but-one entries */
|
|
m->paststartup = 1;
|
|
} else if (x < 0) { /* no such entry? use next */
|
|
x = m->ktlbnext++;
|
|
if(m->ktlbnext >= tlbroff)
|
|
m->ktlbnext = TLBOFF;
|
|
}
|
|
if (x < 0) x = getrandom(); /* no entry for va? overwrite random one */
|
|
puttlbx(x, virt, k->phys0, k->phys1, PGSZ);
|
|
m->ktlbx[x] = 1;
|
|
return x;
|
|
}
|
|
|
|
/*
|
|
* Arrange that the KMap'd virtual address will hit the same
|
|
* primary cache line as pg->va by making bits 14...12 of the
|
|
* tag the same as virtual address. These bits are the index
|
|
* into the primary cache and are checked whenever accessing
|
|
* the secondary cache through the primary. Violation causes
|
|
* a VCE trap.
|
|
*/
|
|
KMap *
|
|
kmap(Page *pg)
|
|
{
|
|
int s, printed = 0;
|
|
ulong pte, virt;
|
|
KMap *k;
|
|
|
|
s = splhi();
|
|
lock(&kmaplock);
|
|
|
|
if(kmapfree == 0) {
|
|
retry:
|
|
unlock(&kmaplock);
|
|
kmapinval(); /* try and free some */
|
|
lock(&kmaplock);
|
|
if(kmapfree == 0){
|
|
unlock(&kmaplock);
|
|
splx(s);
|
|
if(printed++ == 0){
|
|
/* using iprint here we get mixed up with other prints */
|
|
print("%d KMAP RETRY %#lux ktime %ld %ld %ld %ld %ld %ld %ld %ld\n",
|
|
m->machno, getcallerpc(&pg),
|
|
ktime[0], ktime[1], ktime[2], ktime[3],
|
|
ktime[4], ktime[5], ktime[6], ktime[7]);
|
|
delay(200);
|
|
}
|
|
splhi();
|
|
lock(&kmaplock);
|
|
goto retry;
|
|
}
|
|
}
|
|
|
|
k = kmapfree;
|
|
kmapfree = k->next;
|
|
|
|
k->pg = pg;
|
|
/*
|
|
* One for the allocation,
|
|
* One for kactive
|
|
*/
|
|
k->pc = getcallerpc(&pg);
|
|
k->ref = 2;
|
|
k->konmach[m->machno] = m->kactive;
|
|
m->kactive = k;
|
|
|
|
virt = pg->va;
|
|
/* bits 14..12 form the secondary-cache virtual index */
|
|
virt &= PIDX;
|
|
virt |= KMAPADDR | ((k-kpte)<<KMAPSHIFT);
|
|
|
|
k->virt = virt;
|
|
pte = PPN(pg->pa)|PTECACHED|PTEGLOBL|PTEWRITE|PTEVALID;
|
|
if(virt & BY2PG) {
|
|
k->phys0 = PTEGLOBL | PTECACHED;
|
|
k->phys1 = pte;
|
|
}
|
|
else {
|
|
k->phys0 = pte;
|
|
k->phys1 = PTEGLOBL | PTECACHED;
|
|
}
|
|
|
|
putktlb(k);
|
|
unlock(&kmaplock);
|
|
|
|
splx(s);
|
|
return k;
|
|
}
|
|
|
|
void
|
|
kunmap(KMap *k)
|
|
{
|
|
int s;
|
|
|
|
s = splhi();
|
|
if(decref(k) == 0) {
|
|
k->virt = 0;
|
|
k->phys0 = 0;
|
|
k->phys1 = 0;
|
|
k->pg = 0;
|
|
|
|
lock(&kmaplock);
|
|
k->next = kmapfree;
|
|
kmapfree = k;
|
|
//nfree();
|
|
unlock(&kmaplock);
|
|
}
|
|
splx(s);
|
|
}
|
|
|
|
void
|
|
kfault(Ureg *ur) /* called from trap() */
|
|
{
|
|
ulong index, addr;
|
|
KMap *k, *f;
|
|
|
|
addr = ur->badvaddr;
|
|
index = (addr & ~KSEGM) >> KMAPSHIFT;
|
|
if(index >= KPTESIZE)
|
|
panic("kmapfault: va=%#lux", addr);
|
|
|
|
k = &kpte[index];
|
|
if(k->virt == 0)
|
|
panic("kmapfault: unmapped %#lux", addr);
|
|
|
|
for(f = m->kactive; f; f = f->konmach[m->machno])
|
|
if(f == k)
|
|
break;
|
|
if(f == 0) {
|
|
incref(k);
|
|
k->konmach[m->machno] = m->kactive;
|
|
m->kactive = k;
|
|
}
|
|
putktlb(k);
|
|
}
|
|
|
|
void
|
|
kmapinval(void)
|
|
{
|
|
int mno, i, curpid;
|
|
KMap *k, *next;
|
|
uchar *ktlbx;
|
|
|
|
if(m->machno < nelem(ktime))
|
|
ktime[m->machno] = MACHP(0)->ticks;
|
|
if(m->kactive == 0)
|
|
return;
|
|
|
|
curpid = PTEPID(TLBPID(tlbvirt()));
|
|
ktlbx = m->ktlbx;
|
|
for(i = 0; i < NTLB; i++, ktlbx++){
|
|
if(*ktlbx == 0)
|
|
continue;
|
|
TLBINVAL(i, curpid);
|
|
*ktlbx = 0;
|
|
}
|
|
|
|
mno = m->machno;
|
|
for(k = m->kactive; k; k = next) {
|
|
next = k->konmach[mno];
|
|
kunmap(k);
|
|
}
|
|
|
|
m->kactive = 0;
|
|
m->ktlbnext = TLBOFF;
|
|
}
|
|
|
|
/*
|
|
* Process must be splhi
|
|
*/
|
|
static int
|
|
newtlbpid(Proc *p)
|
|
{
|
|
int i, s;
|
|
Proc **h;
|
|
|
|
i = m->lastpid;
|
|
h = m->pidproc;
|
|
for(s = 0; s < NTLBPID; s++) {
|
|
i++;
|
|
if(i >= NTLBPID)
|
|
i = 1;
|
|
if(h[i] == 0)
|
|
break;
|
|
}
|
|
|
|
if(h[i])
|
|
purgetlb(i);
|
|
if(h[i] != 0)
|
|
panic("newtlb");
|
|
|
|
m->pidproc[i] = p;
|
|
p->pidonmach[m->machno] = i;
|
|
m->lastpid = i;
|
|
|
|
return i;
|
|
}
|
|
|
|
void
|
|
mmuswitch(Proc *p)
|
|
{
|
|
int tp;
|
|
static char lasttext[32];
|
|
|
|
if(Debugswitch && !p->kp){
|
|
if(strncmp(lasttext, p->text, sizeof lasttext) != 0)
|
|
iprint("[%s]", p->text);
|
|
strncpy(lasttext, p->text, sizeof lasttext);
|
|
}
|
|
|
|
if(p->newtlb) {
|
|
memset(p->pidonmach, 0, sizeof p->pidonmach);
|
|
p->newtlb = 0;
|
|
}
|
|
tp = p->pidonmach[m->machno];
|
|
if(tp == 0)
|
|
tp = newtlbpid(p);
|
|
puttlbx(0, KZERO|PTEPID(tp), 0, 0, PGSZ);
|
|
}
|
|
|
|
void
|
|
mmurelease(Proc *p)
|
|
{
|
|
memset(p->pidonmach, 0, sizeof p->pidonmach);
|
|
}
|
|
|
|
|
|
/* tlbvirt also has TLBPID() in its low byte as the asid */
|
|
static Softtlb*
|
|
putstlb(ulong tlbvirt, ulong tlbphys)
|
|
{
|
|
int odd;
|
|
Softtlb *entry;
|
|
|
|
/* identical calculation in l.s/utlbmiss */
|
|
entry = &m->stb[stlbhash(tlbvirt)];
|
|
odd = tlbvirt & BY2PG; /* even/odd bit */
|
|
tlbvirt &= ~BY2PG; /* zero even/odd bit */
|
|
if(entry->virt != tlbvirt) { /* not my entry? overwrite it */
|
|
if(entry->virt != 0) {
|
|
m->hashcoll++;
|
|
if (Debughash)
|
|
iprint("putstlb: hash collision: %#lx old virt "
|
|
"%#lux new virt %#lux page %#lux\n",
|
|
entry - m->stb, entry->virt, tlbvirt,
|
|
tlbvirt >> (PGSHIFT+1));
|
|
}
|
|
entry->virt = tlbvirt;
|
|
entry->phys0 = 0;
|
|
entry->phys1 = 0;
|
|
}
|
|
|
|
if(odd)
|
|
entry->phys1 = tlbphys;
|
|
else
|
|
entry->phys0 = tlbphys;
|
|
|
|
if(entry->phys0 == 0 && entry->phys1 == 0)
|
|
entry->virt = 0;
|
|
|
|
return entry;
|
|
}
|
|
|
|
void
|
|
putmmu(ulong tlbvirt, ulong tlbphys, Page *pg)
|
|
{
|
|
short tp;
|
|
ulong tlbent[3];
|
|
Softtlb *entry;
|
|
int s, x;
|
|
|
|
s = splhi();
|
|
tp = up->pidonmach[m->machno];
|
|
if(tp == 0)
|
|
tp = newtlbpid(up);
|
|
|
|
tlbvirt |= PTEPID(tp);
|
|
entry = putstlb(tlbvirt, tlbphys);
|
|
x = gettlbp(tlbvirt, tlbent);
|
|
if(x < 0) x = getrandom();
|
|
puttlbx(x, entry->virt, entry->phys0, entry->phys1, PGSZ);
|
|
if(needtxtflush(pg)){
|
|
icflush((void*)pg->va, BY2PG);
|
|
donetxtflush(pg);
|
|
}
|
|
splx(s);
|
|
}
|
|
|
|
void
|
|
purgetlb(int pid)
|
|
{
|
|
int i, mno;
|
|
Proc *sp, **pidproc;
|
|
Softtlb *entry, *etab;
|
|
|
|
m->tlbpurge++;
|
|
|
|
/*
|
|
* find all pid entries that are no longer used by processes
|
|
*/
|
|
mno = m->machno;
|
|
pidproc = m->pidproc;
|
|
for(i=1; i<NTLBPID; i++) {
|
|
sp = pidproc[i];
|
|
if(sp && sp->pidonmach[mno] != i)
|
|
pidproc[i] = 0;
|
|
}
|
|
|
|
/*
|
|
* shoot down the one we want
|
|
*/
|
|
sp = pidproc[pid];
|
|
if(sp != 0)
|
|
sp->pidonmach[mno] = 0;
|
|
pidproc[pid] = 0;
|
|
|
|
/*
|
|
* clean out all dead pids from the stlb;
|
|
*/
|
|
entry = m->stb;
|
|
for(etab = &entry[STLBSIZE]; entry < etab; entry++)
|
|
if(pidproc[TLBPID(entry->virt)] == 0)
|
|
entry->virt = 0;
|
|
|
|
/*
|
|
* clean up the hardware
|
|
*/
|
|
for(i=tlbroff; i<NTLB; i++)
|
|
if(pidproc[TLBPID(gettlbvirt(i))] == 0)
|
|
TLBINVAL(i, pid);
|
|
}
|
|
|
|
void
|
|
flushmmu(void)
|
|
{
|
|
int s;
|
|
|
|
s = splhi();
|
|
up->newtlb = 1;
|
|
mmuswitch(up);
|
|
splx(s);
|
|
}
|
|
|
|
void
|
|
checkmmu(ulong, ulong)
|
|
{
|
|
}
|
|
|
|
/*
|
|
* Return the number of bytes that can be accessed via KADDR(pa).
|
|
* If pa is not a valid argument to KADDR, return 0.
|
|
*/
|
|
ulong
|
|
cankaddr(ulong pa)
|
|
{
|
|
if(pa >= KZERO)
|
|
return 0;
|
|
return -KZERO - pa;
|
|
}
|