128ea44a89
a portable SG_NOEXEC segment attribute was added to allow non-executable (physical) segments. which will set the PTENOEXEC bits for putmmu(). in the future, this can be used to make non-executable stack / bss segments. the SG_DEVICE attribute was added to distinguish between mmio regions and uncached memory. only matterns on arm64. on arm, theres the issue that PTEUNCACHED would have no bits set when using the hardware bit definitions. this is the reason bcm, kw, teg2 and omap kernels use arteficial PTE constants. on zynq, the XN bit was used as a hack to give PTEUNCACHED a non-zero value and when the bit is clear then cache attributes where added to the pte. to fix this, PTECACHED constant was added. the portable mmu code in fault.c will now explicitely set PTECACHED bits for cached memory and PTEUNCACHED for uncached memory. that way the hardware bit definitions can be used everywhere.
470 lines
8.6 KiB
C
470 lines
8.6 KiB
C
#include "u.h"
|
|
#include "../port/lib.h"
|
|
#include "mem.h"
|
|
#include "dat.h"
|
|
#include "fns.h"
|
|
#include "ureg.h"
|
|
|
|
/*
|
|
* tlb entry 0 is used only by mmuswitch() to set the current tlb pid.
|
|
*
|
|
* It is apparently assumed that user tlb entries are not
|
|
* overwritten during start-up, so ...
|
|
* During system start-up (before up first becomes non-nil),
|
|
* Kmap entries start at tlb index 1 and work their way up until
|
|
* kmapinval() removes them. They then restart at 1. As long as there
|
|
* are few kmap entries they will not pass tlbroff (the WIRED tlb entry
|
|
* limit) and interfere with user tlb entries.
|
|
* Once start-up is over, we combine the kernel and user tlb pools into one,
|
|
* in the hope of making better use of the tlb on systems with small ones.
|
|
*
|
|
* All invalidations of the tlb are via indexed entries. The virtual
|
|
* address used is always 'KZERO | (x<<(PGSHIFT+1) | currentpid' where
|
|
* 'x' is the index into the tlb. This ensures that the current pid doesn't
|
|
* change and that no two invalidated entries have matching virtual
|
|
* addresses just in case SGI/MIPS ever makes a chip that cares (as
|
|
* they keep threatening). These entries should never be used in
|
|
* lookups since accesses to KZERO addresses don't go through the tlb
|
|
* (actually only true of KSEG0 and KSEG1; KSEG2 and KSEG3 do go
|
|
* through the tlb).
|
|
*/
|
|
|
|
#define TLBINVAL(x, pid) puttlbx(x, KZERO|((x)<<(PGSHIFT+1))|(pid), 0, 0, PGSZ)
|
|
|
|
enum {
|
|
Debugswitch = 0,
|
|
Debughash = 0,
|
|
};
|
|
|
|
static ulong ktime[8]; /* only for first 8 cpus */
|
|
|
|
void
|
|
tlbinit(void)
|
|
{
|
|
int i;
|
|
|
|
for(i=0; i<NTLB; i++)
|
|
TLBINVAL(i, 0);
|
|
}
|
|
|
|
Lock kmaplock;
|
|
KMap kpte[KPTESIZE];
|
|
KMap* kmapfree;
|
|
|
|
static int minfree = KPTESIZE;
|
|
static int lastfree;
|
|
static int tlbroff = TLBROFF;
|
|
|
|
static void
|
|
nfree(void)
|
|
{
|
|
int i;
|
|
KMap *k;
|
|
|
|
i = 0;
|
|
for(k=kmapfree; k; k=k->next)
|
|
i++;
|
|
if(i<minfree){
|
|
iprint("%d free\n", i);
|
|
minfree = i;
|
|
}
|
|
lastfree = i;
|
|
}
|
|
|
|
void
|
|
kmapinit(void)
|
|
{
|
|
KMap *k, *klast;
|
|
|
|
lock(&kmaplock);
|
|
kmapfree = kpte;
|
|
klast = &kpte[KPTESIZE-1];
|
|
for(k=kpte; k<klast; k++)
|
|
k->next = k+1;
|
|
k->next = 0;
|
|
unlock(&kmaplock);
|
|
|
|
m->ktlbnext = TLBOFF;
|
|
}
|
|
|
|
void
|
|
kmapdump(void)
|
|
{
|
|
int i;
|
|
|
|
for(i=0; i<KPTESIZE; i++)
|
|
iprint("%d: %lud pc=%#lux\n", i, kpte[i].ref, kpte[i].pc);
|
|
}
|
|
|
|
static int
|
|
putktlb(KMap *k)
|
|
{
|
|
int x;
|
|
ulong virt;
|
|
ulong tlbent[3];
|
|
|
|
virt = k->virt & ~BY2PG | TLBPID(tlbvirt());
|
|
x = gettlbp(virt, tlbent);
|
|
if (!m->paststartup)
|
|
if (up) { /* startup just ended? */
|
|
tlbroff = 1;
|
|
setwired(tlbroff); /* share all-but-one entries */
|
|
m->paststartup = 1;
|
|
} else if (x < 0) { /* no such entry? use next */
|
|
x = m->ktlbnext++;
|
|
if(m->ktlbnext >= tlbroff)
|
|
m->ktlbnext = TLBOFF;
|
|
}
|
|
if (x < 0) x = getrandom(); /* no entry for va? overwrite random one */
|
|
puttlbx(x, virt, k->phys0, k->phys1, PGSZ);
|
|
m->ktlbx[x] = 1;
|
|
return x;
|
|
}
|
|
|
|
/*
|
|
* Arrange that the KMap'd virtual address will hit the same
|
|
* primary cache line as pg->va by making bits 14...12 of the
|
|
* tag the same as virtual address. These bits are the index
|
|
* into the primary cache and are checked whenever accessing
|
|
* the secondary cache through the primary. Violation causes
|
|
* a VCE trap.
|
|
*/
|
|
KMap *
|
|
kmap(Page *pg)
|
|
{
|
|
int s, printed = 0;
|
|
ulong pte, virt;
|
|
KMap *k;
|
|
|
|
s = splhi();
|
|
lock(&kmaplock);
|
|
|
|
if(kmapfree == 0) {
|
|
retry:
|
|
unlock(&kmaplock);
|
|
kmapinval(); /* try and free some */
|
|
lock(&kmaplock);
|
|
if(kmapfree == 0){
|
|
unlock(&kmaplock);
|
|
splx(s);
|
|
if(printed++ == 0){
|
|
/* using iprint here we get mixed up with other prints */
|
|
print("%d KMAP RETRY %#lux ktime %ld %ld %ld %ld %ld %ld %ld %ld\n",
|
|
m->machno, getcallerpc(&pg),
|
|
ktime[0], ktime[1], ktime[2], ktime[3],
|
|
ktime[4], ktime[5], ktime[6], ktime[7]);
|
|
delay(200);
|
|
}
|
|
splhi();
|
|
lock(&kmaplock);
|
|
goto retry;
|
|
}
|
|
}
|
|
|
|
k = kmapfree;
|
|
kmapfree = k->next;
|
|
|
|
k->pg = pg;
|
|
/*
|
|
* One for the allocation,
|
|
* One for kactive
|
|
*/
|
|
k->pc = getcallerpc(&pg);
|
|
k->ref = 2;
|
|
k->konmach[m->machno] = m->kactive;
|
|
m->kactive = k;
|
|
|
|
virt = pg->va;
|
|
/* bits 14..12 form the secondary-cache virtual index */
|
|
virt &= PIDX;
|
|
virt |= KMAPADDR | ((k-kpte)<<KMAPSHIFT);
|
|
|
|
k->virt = virt;
|
|
pte = PPN(pg->pa)|PTECACHED|PTEGLOBL|PTEWRITE|PTEVALID;
|
|
if(virt & BY2PG) {
|
|
k->phys0 = PTEGLOBL | PTECACHED;
|
|
k->phys1 = pte;
|
|
}
|
|
else {
|
|
k->phys0 = pte;
|
|
k->phys1 = PTEGLOBL | PTECACHED;
|
|
}
|
|
|
|
putktlb(k);
|
|
unlock(&kmaplock);
|
|
|
|
splx(s);
|
|
return k;
|
|
}
|
|
|
|
void
|
|
kunmap(KMap *k)
|
|
{
|
|
int s;
|
|
|
|
s = splhi();
|
|
if(decref(k) == 0) {
|
|
k->virt = 0;
|
|
k->phys0 = 0;
|
|
k->phys1 = 0;
|
|
k->pg = 0;
|
|
|
|
lock(&kmaplock);
|
|
k->next = kmapfree;
|
|
kmapfree = k;
|
|
//nfree();
|
|
unlock(&kmaplock);
|
|
}
|
|
splx(s);
|
|
}
|
|
|
|
void
|
|
kfault(Ureg *ur) /* called from trap() */
|
|
{
|
|
ulong index, addr;
|
|
KMap *k, *f;
|
|
|
|
addr = ur->badvaddr;
|
|
index = (addr & ~KSEGM) >> KMAPSHIFT;
|
|
if(index >= KPTESIZE)
|
|
panic("kmapfault: va=%#lux", addr);
|
|
|
|
k = &kpte[index];
|
|
if(k->virt == 0)
|
|
panic("kmapfault: unmapped %#lux", addr);
|
|
|
|
for(f = m->kactive; f; f = f->konmach[m->machno])
|
|
if(f == k)
|
|
break;
|
|
if(f == 0) {
|
|
incref(k);
|
|
k->konmach[m->machno] = m->kactive;
|
|
m->kactive = k;
|
|
}
|
|
putktlb(k);
|
|
}
|
|
|
|
void
|
|
kmapinval(void)
|
|
{
|
|
int mno, i, curpid;
|
|
KMap *k, *next;
|
|
uchar *ktlbx;
|
|
|
|
if(m->machno < nelem(ktime))
|
|
ktime[m->machno] = MACHP(0)->ticks;
|
|
if(m->kactive == 0)
|
|
return;
|
|
|
|
curpid = PTEPID(TLBPID(tlbvirt()));
|
|
ktlbx = m->ktlbx;
|
|
for(i = 0; i < NTLB; i++, ktlbx++){
|
|
if(*ktlbx == 0)
|
|
continue;
|
|
TLBINVAL(i, curpid);
|
|
*ktlbx = 0;
|
|
}
|
|
|
|
mno = m->machno;
|
|
for(k = m->kactive; k; k = next) {
|
|
next = k->konmach[mno];
|
|
kunmap(k);
|
|
}
|
|
|
|
m->kactive = 0;
|
|
m->ktlbnext = TLBOFF;
|
|
}
|
|
|
|
/*
|
|
* Process must be splhi
|
|
*/
|
|
static int
|
|
newtlbpid(Proc *p)
|
|
{
|
|
int i, s;
|
|
Proc **h;
|
|
|
|
i = m->lastpid;
|
|
h = m->pidproc;
|
|
for(s = 0; s < NTLBPID; s++) {
|
|
i++;
|
|
if(i >= NTLBPID)
|
|
i = 1;
|
|
if(h[i] == 0)
|
|
break;
|
|
}
|
|
|
|
if(h[i])
|
|
purgetlb(i);
|
|
if(h[i] != 0)
|
|
panic("newtlb");
|
|
|
|
m->pidproc[i] = p;
|
|
p->pidonmach[m->machno] = i;
|
|
m->lastpid = i;
|
|
|
|
return i;
|
|
}
|
|
|
|
void
|
|
mmuswitch(Proc *p)
|
|
{
|
|
int tp;
|
|
static char lasttext[32];
|
|
|
|
if(Debugswitch && !p->kp){
|
|
if(strncmp(lasttext, p->text, sizeof lasttext) != 0)
|
|
iprint("[%s]", p->text);
|
|
strncpy(lasttext, p->text, sizeof lasttext);
|
|
}
|
|
|
|
if(p->newtlb) {
|
|
memset(p->pidonmach, 0, sizeof p->pidonmach);
|
|
p->newtlb = 0;
|
|
}
|
|
tp = p->pidonmach[m->machno];
|
|
if(tp == 0)
|
|
tp = newtlbpid(p);
|
|
puttlbx(0, KZERO|PTEPID(tp), 0, 0, PGSZ);
|
|
}
|
|
|
|
void
|
|
mmurelease(Proc *p)
|
|
{
|
|
memset(p->pidonmach, 0, sizeof p->pidonmach);
|
|
}
|
|
|
|
|
|
/* tlbvirt also has TLBPID() in its low byte as the asid */
|
|
static Softtlb*
|
|
putstlb(ulong tlbvirt, ulong tlbphys)
|
|
{
|
|
int odd;
|
|
Softtlb *entry;
|
|
|
|
/* identical calculation in l.s/utlbmiss */
|
|
entry = &m->stb[stlbhash(tlbvirt)];
|
|
odd = tlbvirt & BY2PG; /* even/odd bit */
|
|
tlbvirt &= ~BY2PG; /* zero even/odd bit */
|
|
if(entry->virt != tlbvirt) { /* not my entry? overwrite it */
|
|
if(entry->virt != 0) {
|
|
m->hashcoll++;
|
|
if (Debughash)
|
|
iprint("putstlb: hash collision: %#lx old virt "
|
|
"%#lux new virt %#lux page %#lux\n",
|
|
entry - m->stb, entry->virt, tlbvirt,
|
|
tlbvirt >> (PGSHIFT+1));
|
|
}
|
|
entry->virt = tlbvirt;
|
|
entry->phys0 = 0;
|
|
entry->phys1 = 0;
|
|
}
|
|
|
|
if(odd)
|
|
entry->phys1 = tlbphys;
|
|
else
|
|
entry->phys0 = tlbphys;
|
|
|
|
if(entry->phys0 == 0 && entry->phys1 == 0)
|
|
entry->virt = 0;
|
|
|
|
return entry;
|
|
}
|
|
|
|
void
|
|
putmmu(ulong tlbvirt, ulong tlbphys, Page *pg)
|
|
{
|
|
short tp;
|
|
ulong tlbent[3];
|
|
Softtlb *entry;
|
|
int s, x;
|
|
|
|
s = splhi();
|
|
tp = up->pidonmach[m->machno];
|
|
if(tp == 0)
|
|
tp = newtlbpid(up);
|
|
|
|
tlbvirt |= PTEPID(tp);
|
|
entry = putstlb(tlbvirt, tlbphys);
|
|
x = gettlbp(tlbvirt, tlbent);
|
|
if(x < 0) x = getrandom();
|
|
puttlbx(x, entry->virt, entry->phys0, entry->phys1, PGSZ);
|
|
if(pg->txtflush & (1<<m->machno)){
|
|
icflush((void*)pg->va, BY2PG);
|
|
pg->txtflush &= ~(1<<m->machno);
|
|
}
|
|
splx(s);
|
|
}
|
|
|
|
void
|
|
purgetlb(int pid)
|
|
{
|
|
int i, mno;
|
|
Proc *sp, **pidproc;
|
|
Softtlb *entry, *etab;
|
|
|
|
m->tlbpurge++;
|
|
|
|
/*
|
|
* find all pid entries that are no longer used by processes
|
|
*/
|
|
mno = m->machno;
|
|
pidproc = m->pidproc;
|
|
for(i=1; i<NTLBPID; i++) {
|
|
sp = pidproc[i];
|
|
if(sp && sp->pidonmach[mno] != i)
|
|
pidproc[i] = 0;
|
|
}
|
|
|
|
/*
|
|
* shoot down the one we want
|
|
*/
|
|
sp = pidproc[pid];
|
|
if(sp != 0)
|
|
sp->pidonmach[mno] = 0;
|
|
pidproc[pid] = 0;
|
|
|
|
/*
|
|
* clean out all dead pids from the stlb;
|
|
*/
|
|
entry = m->stb;
|
|
for(etab = &entry[STLBSIZE]; entry < etab; entry++)
|
|
if(pidproc[TLBPID(entry->virt)] == 0)
|
|
entry->virt = 0;
|
|
|
|
/*
|
|
* clean up the hardware
|
|
*/
|
|
for(i=tlbroff; i<NTLB; i++)
|
|
if(pidproc[TLBPID(gettlbvirt(i))] == 0)
|
|
TLBINVAL(i, pid);
|
|
}
|
|
|
|
void
|
|
flushmmu(void)
|
|
{
|
|
int s;
|
|
|
|
s = splhi();
|
|
up->newtlb = 1;
|
|
mmuswitch(up);
|
|
splx(s);
|
|
}
|
|
|
|
void
|
|
checkmmu(ulong, ulong)
|
|
{
|
|
}
|
|
|
|
/*
|
|
* Return the number of bytes that can be accessed via KADDR(pa).
|
|
* If pa is not a valid argument to KADDR, return 0.
|
|
*/
|
|
ulong
|
|
cankaddr(ulong pa)
|
|
{
|
|
if(pa >= KZERO)
|
|
return 0;
|
|
return -KZERO - pa;
|
|
}
|