From 2eadf1fa1703e5da4a98ba9c1a2bdc23bd140d8c Mon Sep 17 00:00:00 2001 From: Noam Preil Date: Wed, 21 Jul 2021 05:06:05 +0000 Subject: [PATCH] venti: fix memory layers --- sys/include/venti.h | 8 ++-- sys/src/cmd/venti/srv/arenas.c | 2 +- sys/src/cmd/venti/srv/buildindex.c | 9 ++-- sys/src/cmd/venti/srv/config.c | 10 ++--- sys/src/cmd/venti/srv/dcache.c | 15 +++---- sys/src/cmd/venti/srv/fns.h | 16 ++++---- sys/src/cmd/venti/srv/icache.c | 8 ++-- sys/src/cmd/venti/srv/icachewrite.c | 2 +- sys/src/cmd/venti/srv/lumpcache.c | 8 ++-- sys/src/cmd/venti/srv/part.c | 6 +-- sys/src/cmd/venti/srv/printindex.c | 2 +- sys/src/cmd/venti/srv/utils.c | 64 ----------------------------- sys/src/cmd/venti/srv/venti.c | 2 +- sys/src/libventi/mem.c | 52 +++++++++++++---------- 14 files changed, 73 insertions(+), 131 deletions(-) diff --git a/sys/include/venti.h b/sys/include/venti.h index e4677c1a3..dff553b4c 100644 --- a/sys/include/venti.h +++ b/sys/include/venti.h @@ -207,10 +207,10 @@ int vtscorefmt(Fmt*); * error-checking malloc et al. */ void vtfree(void *); -void* vtmalloc(int); -void* vtmallocz(int); -void* vtrealloc(void *p, int); -void* vtbrk(int n); +void* vtmalloc(ulong); +void* vtmallocz(ulong); +void* vtrealloc(void *p, ulong); +void* vtbrk(ulong); char* vtstrdup(char *); /* diff --git a/sys/src/cmd/venti/srv/arenas.c b/sys/src/cmd/venti/srv/arenas.c index 0316c4c86..96fa7d668 100644 --- a/sys/src/cmd/venti/srv/arenas.c +++ b/sys/src/cmd/venti/srv/arenas.c @@ -375,7 +375,7 @@ parseamap(IFile *f, AMapN *amn) for(i = 0; i < n; i++){ s = ifileline(f); if(s) - t = estrdup(s); + t = vtstrdup(s); else t = nil; if(s == nil || getfields(s, flds, 4, 0, "\t") != 3){ diff --git a/sys/src/cmd/venti/srv/buildindex.c b/sys/src/cmd/venti/srv/buildindex.c index 97ec3855b..8ce5a77d6 100644 --- a/sys/src/cmd/venti/srv/buildindex.c +++ b/sys/src/cmd/venti/srv/buildindex.c @@ -446,7 +446,7 @@ mkipool(ISect *isect, Minibuf *mbuf, u32int nmbuf, IEntryLink *l; nentry = (nmbuf+1)*bufsize / IEntrySize; - p = ezmalloc(sizeof(IPool) + p = vtmallocz(sizeof(IPool) +nentry*sizeof(IEntry) +nmbuf*sizeof(IEntryLink*) +nmbuf*sizeof(u32int) @@ -676,7 +676,7 @@ sortminibuffer(ISect *is, Minibuf *mb, uchar *buf, u32int nbuf, u32int bufsize) Part *part; part = is->part; - buckdata = emalloc(is->blocksize); + buckdata = vtmalloc(is->blocksize); if(mb->nwentry == 0) return; @@ -691,7 +691,6 @@ sortminibuffer(ISect *is, Minibuf *mb, uchar *buf, u32int nbuf, u32int bufsize) errors = 1; return; } - assert(*(uint*)buf != 0xa5a5a5a5); /* * remove fragmentation due to IEntrySize @@ -820,7 +819,7 @@ isectproc(void *v) bufsize = MinBufSize; while(bufsize*2*nbuf <= isectmem && bufsize < MaxBufSize) bufsize *= 2; - data = emalloc(nbuf*bufsize); + data = vtmalloc(nbuf*bufsize); epbuf = bufsize/IEntrySize; fprint(2, "%T %s: %,ud buckets, %,ud groups, %,ud minigroups, %,ud buffer\n", is->part->name, nbucket, nbuf, nminibuf, bufsize); @@ -951,7 +950,7 @@ isectproc(void *v) if(space < mbuf[j].woffset - mbuf[j].boffset) space = mbuf[j].woffset - mbuf[j].boffset; - data = emalloc(space); + data = vtmalloc(space); for(j=0; jnsects + 1); @@ -122,7 +122,7 @@ runconfig(char *file, Config *config) seterr(EAdmin, "duplicate indices in config file %s", file); break; } - config->index = estrdup(flds[1]); + config->index = vtstrdup(flds[1]); }else if(i == 2 && strcmp(flds[0], "bcmem") == 0){ if(numok(flds[1]) < 0){ seterr(EAdmin, "illegal size %s in config file %s", @@ -163,19 +163,19 @@ runconfig(char *file, Config *config) seterr(EAdmin, "duplicate httpaddr lines in configuration file %s", file); break; } - config->haddr = estrdup(flds[1]); + config->haddr = vtstrdup(flds[1]); }else if(i == 2 && strcmp(flds[0], "webroot") == 0){ if(config->webroot){ seterr(EAdmin, "duplicate webroot lines in configuration file %s", file); break; } - config->webroot = estrdup(flds[1]); + config->webroot = vtstrdup(flds[1]); }else if(i == 2 && strcmp(flds[0], "addr") == 0){ if(config->vaddr){ seterr(EAdmin, "duplicate addr lines in configuration file %s", file); break; } - config->vaddr = estrdup(flds[1]); + config->vaddr = vtstrdup(flds[1]); }else{ seterr(EAdmin, "illegal line '%s' in configuration file %s", line, file); break; diff --git a/sys/src/cmd/venti/srv/dcache.c b/sys/src/cmd/venti/srv/dcache.c index a50ef0c5c..44e530e43 100644 --- a/sys/src/cmd/venti/srv/dcache.c +++ b/sys/src/cmd/venti/srv/dcache.c @@ -82,10 +82,11 @@ initdcache(u32int mem) int i; u8int *p; - if(mem < maxblocksize * 2) - sysfatal("need at least %d bytes for the disk cache", maxblocksize * 2); if(maxblocksize == 0) sysfatal("no max. block size given for disk cache"); + if(mem < maxblocksize * 2) + sysfatal("need at least 2 max-size blocks (%d bytes) for the disk cache", maxblocksize * 2); + blocksize = maxblocksize; nblocks = mem / blocksize; dcache.full.l = &dcache.lock; @@ -94,11 +95,11 @@ initdcache(u32int mem) trace(TraceProc, "initialize disk cache with %d blocks of %d bytes, maximum %d dirty blocks\n", nblocks, blocksize, dcache.maxdirty); dcache.size = blocksize; - dcache.heads = MKNZ(DBlock*, HashSize); - dcache.heap = MKNZ(DBlock*, nblocks); - dcache.blocks = MKNZ(DBlock, nblocks); - dcache.write = MKNZ(DBlock*, nblocks); - dcache.mem = MKNZ(u8int, (nblocks+1+128) * blocksize); + dcache.heads = vtbrk(sizeof(DBlock*) * HashSize); + dcache.heap = vtbrk(sizeof(DBlock*) * nblocks); + dcache.blocks = vtbrk(sizeof(DBlock) * nblocks); + dcache.write = vtbrk(sizeof(DBlock*) * nblocks); + dcache.mem = vtbrk((nblocks+1+128) * blocksize); last = nil; p = (u8int*)(((uintptr)dcache.mem+blocksize-1)&~(uintptr)(blocksize-1)); diff --git a/sys/src/cmd/venti/srv/fns.h b/sys/src/cmd/venti/srv/fns.h index 398562c27..a1181b8cb 100644 --- a/sys/src/cmd/venti/srv/fns.h +++ b/sys/src/cmd/venti/srv/fns.h @@ -29,13 +29,9 @@ void delaykickroundproc(void*); void dirtydblock(DBlock*, int); void diskaccess(int); void disksched(void); -void *emalloc(ulong); void emptydcache(void); void emptyicache(void); void emptylumpcache(void); -void *erealloc(void *, ulong); -char *estrdup(char*); -void *ezmalloc(ulong); Arena *findarena(char *name); int flushciblocks(Arena *arena); void flushdcache(void); @@ -151,6 +147,8 @@ int readclumpinfos(Arena *arena, int clump, ClumpInfo *cis, int n); ZBlock *readfile(char *name); int readifile(IFile *f, char *name); Packet *readlump(u8int *score, int type, u32int size, int *cached); +// If the return value is not negative one, n bytes were successfully read. +// If n == -1, this will ALWAYS report failure, even when the read succeeded. int readpart(Part *part, u64int addr, u8int *buf, u32int n); int resetbloom(Bloom*); int runconfig(char *config, Config*); @@ -221,8 +219,8 @@ void zeropart(Part *part, int blocksize); #define scorecmp(h1,h2) memcmp((h1),(h2),VtScoreSize) #define scorecp(h1,h2) memmove((h1),(h2),VtScoreSize) -#define MK(t) ((t*)emalloc(sizeof(t))) -#define MKZ(t) ((t*)ezmalloc(sizeof(t))) -#define MKN(t,n) ((t*)emalloc((n)*sizeof(t))) -#define MKNZ(t,n) ((t*)ezmalloc((n)*sizeof(t))) -#define MKNA(t,at,n) ((t*)emalloc(sizeof(t) + (n)*sizeof(at))) +#define MK(t) ((t*)vtmalloc(sizeof(t))) +#define MKZ(t) ((t*)vtmallocz(sizeof(t))) +#define MKN(t,n) ((t*)vtmalloc((n)*sizeof(t))) +#define MKNZ(t,n) ((t*)vtmallocz((n)*sizeof(t))) +#define MKNA(t,at,n) ((t*)vtmalloc(sizeof(t) + (n)*sizeof(at))) diff --git a/sys/src/cmd/venti/srv/icache.c b/sys/src/cmd/venti/srv/icache.c index 67faba209..4c7a71327 100644 --- a/sys/src/cmd/venti/srv/icache.c +++ b/sys/src/cmd/venti/srv/icache.c @@ -278,14 +278,12 @@ scachemiss(u64int addr) */ void -initicache(u32int mem0) +initicache(u32int mem) { - u32int mem; int i, entries, scache; icache.full.l = &icache.lock; - mem = mem0; entries = mem / (sizeof(IEntry)+sizeof(IEntry*)); scache = (entries/8) / ArenaCIGSize; entries -= entries/8; @@ -295,7 +293,7 @@ initicache(u32int mem0) scache = 16; if(entries < 1000) entries = 1000; -fprint(2, "icache %,d bytes = %,d entries; %d scache\n", mem0, entries, scache); +fprint(2, "icache %,lud bytes = %,lud entries; %lud scache\n", mem, entries, scache); icache.clean.prev = icache.clean.next = &icache.clean; icache.dirty.prev = icache.dirty.next = &icache.dirty; @@ -304,7 +302,7 @@ fprint(2, "icache %,d bytes = %,d entries; %d scache\n", mem0, entries, scache); icache.hash = mkihash(entries); icache.nentries = entries; setstat(StatIcacheSize, entries); - icache.entries = vtmallocz(entries*sizeof icache.entries[0]); + icache.entries = vtbrk(entries*sizeof icache.entries[0]); icache.maxdirty = entries / 2; for(i=0; ipart->name); bsize = 1<blocklog; - buf = emalloc(Bufsize+bsize); + buf = vtmalloc(Bufsize+bsize); buf = (u8int*)(((uintptr)buf+bsize-1)&~(uintptr)(bsize-1)); for(;;){ diff --git a/sys/src/cmd/venti/srv/lumpcache.c b/sys/src/cmd/venti/srv/lumpcache.c index d9a6b954e..a67aff4ea 100644 --- a/sys/src/cmd/venti/srv/lumpcache.c +++ b/sys/src/cmd/venti/srv/lumpcache.c @@ -47,9 +47,9 @@ initlumpcache(u32int size, u32int nblocks) lumpcache.nblocks = nblocks; lumpcache.allowed = size; lumpcache.avail = size; - lumpcache.heads = MKNZ(Lump*, HashSize); - lumpcache.heap = MKNZ(Lump*, nblocks); - lumpcache.blocks = MKNZ(Lump, nblocks); + lumpcache.heads = vtbrk(sizeof(Lump*) * HashSize); + lumpcache.heap = vtbrk(sizeof(Lump*) * nblocks); + lumpcache.blocks = vtbrk(sizeof(Lump) * nblocks); setstat(StatLcacheSize, lumpcache.nblocks); last = nil; @@ -413,7 +413,7 @@ checklumpcache(void) } if(lumpcache.avail != lumpcache.allowed - size){ fprint(2, "mismatched available=%d and allowed=%d - used=%d space", lumpcache.avail, lumpcache.allowed, size); - *(int*)0=0; + abort(); } nfree = 0; diff --git a/sys/src/cmd/venti/srv/part.c b/sys/src/cmd/venti/srv/part.c index 9f112cf62..d5c778742 100644 --- a/sys/src/cmd/venti/srv/part.c +++ b/sys/src/cmd/venti/srv/part.c @@ -47,7 +47,7 @@ parsepart(char *name, char **file, u64int *lo, u64int *hi) { char *p; - *file = estrdup(name); + *file = vtstrdup(name); if((p = strrchr(*file, ':')) == nil){ *lo = 0; *hi = 0; @@ -87,8 +87,8 @@ initpart(char *name, int mode) return nil; trace(TraceDisk, "initpart %s file %s lo 0x%llx hi 0x%llx", name, file, lo, hi); part = MKZ(Part); - part->name = estrdup(name); - part->filename = estrdup(file); + part->name = vtstrdup(name); + part->filename = vtstrdup(file); if(readonly){ mode &= ~(OREAD|OWRITE|ORDWR); mode |= OREAD; diff --git a/sys/src/cmd/venti/srv/printindex.c b/sys/src/cmd/venti/srv/printindex.c index edbcf7934..895f11407 100644 --- a/sys/src/cmd/venti/srv/printindex.c +++ b/sys/src/cmd/venti/srv/printindex.c @@ -44,7 +44,7 @@ dumpisect(ISect *is) IBucket ib; IEntry ie; - buf = emalloc(is->blocksize); + buf = vtmalloc(is->blocksize); for(i=0; iblocks; i++){ off = is->blockbase+(u64int)is->blocksize*i; if(readpart(is->part, off, buf, is->blocksize) < 0) diff --git a/sys/src/cmd/venti/srv/utils.c b/sys/src/cmd/venti/srv/utils.c index d810c53d8..0c4cb4517 100644 --- a/sys/src/cmd/venti/srv/utils.c +++ b/sys/src/cmd/venti/srv/utils.c @@ -136,70 +136,6 @@ now(void) return time(nil); } -int abortonmem = 1; - -void * -emalloc(ulong n) -{ - void *p; - - p = malloc(n); - if(p == nil){ - if(abortonmem) - abort(); - sysfatal("out of memory allocating %lud", n); - } - memset(p, 0xa5, n); - setmalloctag(p, getcallerpc(&n)); -if(0)print("emalloc %p-%p by %#p\n", p, (char*)p+n, getcallerpc(&n)); - return p; -} - -void * -ezmalloc(ulong n) -{ - void *p; - - p = malloc(n); - if(p == nil){ - if(abortonmem) - abort(); - sysfatal("out of memory allocating %lud", n); - } - memset(p, 0, n); - setmalloctag(p, getcallerpc(&n)); -if(0)print("ezmalloc %p-%p by %#p\n", p, (char*)p+n, getcallerpc(&n)); - return p; -} - -void * -erealloc(void *p, ulong n) -{ - p = realloc(p, n); - if(p == nil){ - if(abortonmem) - abort(); - sysfatal("out of memory allocating %lud", n); - } - setrealloctag(p, getcallerpc(&p)); -if(0)print("erealloc %p-%p by %#p\n", p, (char*)p+n, getcallerpc(&p)); - return p; -} - -char * -estrdup(char *s) -{ - char *t; - int n; - - n = strlen(s) + 1; - t = emalloc(n); - memmove(t, s, n); - setmalloctag(t, getcallerpc(&s)); -if(0)print("estrdup %p-%p by %#p\n", t, (char*)t+n, getcallerpc(&s)); - return t; -} - /* * return floor(log2(v)) */ diff --git a/sys/src/cmd/venti/srv/venti.c b/sys/src/cmd/venti/srv/venti.c index 87361a0e1..da166515b 100755 --- a/sys/src/cmd/venti/srv/venti.c +++ b/sys/src/cmd/venti/srv/venti.c @@ -359,7 +359,7 @@ static void vtrerror(VtReq *r, char *error) { r->rx.msgtype = VtRerror; - r->rx.error = estrdup(error); + r->rx.error = vtstrdup(error); } static void diff --git a/sys/src/libventi/mem.c b/sys/src/libventi/mem.c index dea99a9d1..57e108a5a 100644 --- a/sys/src/libventi/mem.c +++ b/sys/src/libventi/mem.c @@ -7,55 +7,55 @@ enum { ChunkSize = 128*1024 }; - void vtfree(void *p) { - if(p == 0) - return; free(p); } void * -vtmalloc(int size) +vtmalloc(ulong size) { - void *p; - - p = malloc(size); - if(p == 0) - sysfatal("vtmalloc: out of memory"); + void *p = mallocz(size, 0); + if(p == nil){ + fprint(2, "vtmalloc: out of memory allocating %lud", size); + abort(); + } setmalloctag(p, getcallerpc(&size)); return p; } void * -vtmallocz(int size) +vtmallocz(ulong size) { - void *p = vtmalloc(size); - memset(p, 0, size); + void *p = mallocz(size, 1); + if(p == nil){ + fprint(2, "vtmallocz: out of memory allocating %lud", size); + abort(); + } setmalloctag(p, getcallerpc(&size)); return p; } void * -vtrealloc(void *p, int size) +vtrealloc(void *p, ulong size) { - if(p == nil) - return vtmalloc(size); p = realloc(p, size); - if(p == 0) - sysfatal("vtMemRealloc: out of memory"); + if(p == 0 && size != 0){ + fprint(2, "vtrealloc: out of memory allocating %lud", size); + abort(); + } setrealloctag(p, getcallerpc(&size)); return p; } void * -vtbrk(int n) +vtbrk(ulong n) { static Lock lk; static uchar *buf; - static int nbuf, nchunk; - int align, pad; + static ulong nbuf, nchunk; + ulong align, pad; void *p; if(n >= IdealAlignment) @@ -65,10 +65,20 @@ vtbrk(int n) else align = 4; + if(n > ChunkSize){ + p = sbrk(n); + if(p == (void*)-1) + sysfatal("Failed to allocate permanent chunk size %lud", n); + memset(p, 0, n); + return (uchar*)p; + } lock(&lk); pad = (align - (uintptr)buf) & (align-1); if(n + pad > nbuf) { - buf = vtmallocz(ChunkSize); + buf = sbrk(ChunkSize); + if(buf == (void*)-1) + sysfatal("Failed to allocate permanent chunk size %ud", ChunkSize); + memset(buf, 0, ChunkSize); nbuf = ChunkSize; pad = (align - (uintptr)buf) & (align-1); nchunk++;