From 8113195f3209ca35a557aa02611f7053870717e0 Mon Sep 17 00:00:00 2001 From: ReactOS Portable Systems Group Date: Sun, 3 Jan 2010 05:24:30 +0000 Subject: [PATCH] - Bring back old paged pool allocator and use it for paged pool, until ARM3 pool issues are solved. - Should fix recent regressions. svn path=/trunk/; revision=44904 --- reactos/ntoskrnl/mm/ARM3/expool.c | 18 + reactos/ntoskrnl/mm/mminit.c | 33 +- reactos/ntoskrnl/mm/ppool.c | 1023 ++++++++++++++++++++++ reactos/ntoskrnl/ntoskrnl-generic.rbuild | 1 + 4 files changed, 1072 insertions(+), 3 deletions(-) create mode 100644 reactos/ntoskrnl/mm/ppool.c diff --git a/reactos/ntoskrnl/mm/ARM3/expool.c b/reactos/ntoskrnl/mm/ARM3/expool.c index d8cf8642eca..4c5f46b0ccc 100644 --- a/reactos/ntoskrnl/mm/ARM3/expool.c +++ b/reactos/ntoskrnl/mm/ARM3/expool.c @@ -193,6 +193,11 @@ ExAllocatePoolWithTag(IN POOL_TYPE PoolType, KIRQL OldIrql; ULONG BlockSize, i; + // + // Check for paged pool + // + if (PoolType == PagedPool) return ExAllocatePagedPoolWithTag(PagedPool, NumberOfBytes, Tag); + // // Some sanity checks // @@ -480,6 +485,19 @@ ExFreePoolWithTag(IN PVOID P, POOL_TYPE PoolType; PPOOL_DESCRIPTOR PoolDesc; BOOLEAN Combined = FALSE; + + // + // Check for paged pool + // + if ((P >= MmPagedPoolBase) && + (P <= (PVOID)((ULONG_PTR)MmPagedPoolBase + MmPagedPoolSize))) + { + // + // Use old allocator + // + ExFreePagedPool(P); + return; + } // // Quickly deal with big page allocations diff --git a/reactos/ntoskrnl/mm/mminit.c b/reactos/ntoskrnl/mm/mminit.c index 50b93ea04b7..f8c8574af9f 100644 --- a/reactos/ntoskrnl/mm/mminit.c +++ b/reactos/ntoskrnl/mm/mminit.c @@ -198,6 +198,21 @@ MiInitSystemMemoryAreas() BoundaryAddressMultiple); ASSERT(Status == STATUS_SUCCESS); + // + // And now, ReactOS paged pool + // + BaseAddress = MmPagedPoolBase; + Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), + MEMORY_AREA_PAGED_POOL | MEMORY_AREA_STATIC, + &BaseAddress, + MmPagedPoolSize, + PAGE_READWRITE, + &MArea, + TRUE, + 0, + BoundaryAddressMultiple); + ASSERT(Status == STATUS_SUCCESS); + // // Next, the KPCR // @@ -272,6 +287,10 @@ MiDbgDumpAddressSpace(VOID) MmSystemRangeStart, (ULONG_PTR)MmSystemRangeStart + MmBootImageSize, "Boot Loaded Image"); + DPRINT1(" 0x%p - 0x%p\t%s\n", + MmPagedPoolBase, + (ULONG_PTR)MmPagedPoolBase + MmPagedPoolSize, + "Paged Pool"); DPRINT1(" 0x%p - 0x%p\t%s\n", MmPfnDatabase, (ULONG_PTR)MmPfnDatabase + (MxPfnAllocation << PAGE_SHIFT), @@ -370,13 +389,21 @@ MmInitSystem(IN ULONG Phase, // MiDbgReadyForPhysical = TRUE; #endif - + + /* Put the paged pool after the loaded modules */ + MmPagedPoolBase = (PVOID)PAGE_ROUND_UP((ULONG_PTR)MmSystemRangeStart + + MmBootImageSize); + MmPagedPoolSize = MM_PAGED_POOL_SIZE; + /* Intialize system memory areas */ MiInitSystemMemoryAreas(); - + /* Dump the address space */ MiDbgDumpAddressSpace(); - + + /* Initialize paged pool */ + MmInitializePagedPool(); + /* Initialize working sets */ MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory); diff --git a/reactos/ntoskrnl/mm/ppool.c b/reactos/ntoskrnl/mm/ppool.c new file mode 100644 index 00000000000..973736d759a --- /dev/null +++ b/reactos/ntoskrnl/mm/ppool.c @@ -0,0 +1,1023 @@ +/* + * COPYRIGHT: See COPYING in the top level directory + * PROJECT: ReactOS kernel + * FILE: ntoskrnl/mm/ppool.c + * PURPOSE: Implements the paged pool + * + * PROGRAMMERS: David Welch (welch@mcmail.com) + * Royce Mitchell III + */ + +/* INCLUDES *****************************************************************/ + +#include +#define NDEBUG +#include + +#if defined (ALLOC_PRAGMA) +#pragma alloc_text(INIT, MmInitializePagedPool) +#endif + +#undef ASSERT +#define ASSERT(x) if (!(x)) {DbgPrint("Assertion "#x" failed at %s:%d\n", __FILE__,__LINE__); DbgBreakPoint(); } + +// enable "magic" +//#define R_MAGIC +#define R_MUTEX FAST_MUTEX +#define R_ACQUIRE_MUTEX(pool) /*DPRINT1("Acquiring PPool Mutex\n");*/ ExAcquireFastMutex(&pool->Mutex) +#define R_RELEASE_MUTEX(pool) /*DPRINT1("Releasing PPool Mutex\n");*/ ExReleaseFastMutex(&pool->Mutex) +#define R_PRINT_ADDRESS(addr) KeRosPrintAddress(addr) +#define R_PANIC() KeBugCheck(MEMORY_MANAGEMENT) +#define R_DEBUG DbgPrint + +#ifdef _ARM_ +#define R_GET_STACK_FRAMES(ptr,cnt) +#else +#define R_GET_STACK_FRAMES(ptr,cnt) RtlWalkFrameChain((PVOID*)ptr,cnt, 0) +#endif + +/* GLOBALS ********************************************************************/ + +typedef unsigned long rulong; + +#define R_IS_POOL_PTR(pool,ptr) (((void*)(ULONG_PTR)(ptr) >= pool->UserBase) && ((ULONG_PTR)(ptr) < ((ULONG_PTR)pool->UserBase + pool->UserSize))) +#define R_ASSERT_PTR(pool,ptr) ASSERT( R_IS_POOL_PTR(pool,ptr) ) +#define R_ASSERT_SIZE(pool,sz) ASSERT( sz > (sizeof(R_USED)+2*R_RZ) && sz >= sizeof(R_FREE) && sz < pool->UserSize ) + +#ifndef R_ROUND_UP +#define R_ROUND_UP(x,s) ((PVOID)(((ULONG_PTR)(x)+(s)-1) & ~((ULONG_PTR)(s)-1))) +#endif//R_ROUND_UP + +#ifndef R_ROUND_DOWN +#define R_ROUND_DOWN(x,s) ((PVOID)(((ULONG_PTR)(x)) & ~((ULONG_PTR)(s)-1))) +#endif//R_ROUND_DOWN + +#ifndef R_QUEMIN +// R_QUEMIN is the minimum number of entries to keep in a que +#define R_QUEMIN 0 +#endif//R_QUEMIN + +#ifndef R_QUECOUNT +// 16, 32, 64, 128, 256, 512 +#define R_QUECOUNT 6 +#endif//R_QUECOUNT + +#ifndef R_RZ +// R_RZ is the redzone size +#define R_RZ 4 +#endif//R_RZ + +#ifndef R_RZ_LOVALUE +#define R_RZ_LOVALUE 0x87 +#endif//R_RZ_LOVALUE + +#ifndef R_RZ_HIVALUE +#define R_RZ_HIVALUE 0xA5 +#endif//R_RZ_HIVALUE + +#ifndef R_STACK +// R_STACK is the number of stack entries to store in blocks for debug purposes +#define R_STACK 6 +#else // R_STACK +#if R_STACK > 0 && R_STACK < 6 +/* Increase the frame depth to get a reasonable back trace */ +#undef R_STACK +#define R_STACK 6 +#endif // R_STACK > 0 && R_STACK < 6 +#endif//R_STACK + +#ifndef R_TAG +// R_TAG do we keep track of tags on a per-memory block basis? +#define R_TAG 0 +#endif//R_TAG + +#ifdef R_MAGIC +# ifndef R_FREE_MAGIC +# define R_FREE_MAGIC (rulong)(('F'<<0) + ('r'<<8) + ('E'<<16) + ('e'<<24)) +# endif//R_FREE_MAGIC +# ifndef R_USED_MAGIC +# define R_USED_MAGIC (rulong)(('u'<<0) + ('S'<<8) + ('e'<<16) + ('D'<<24)) +# endif//R_USED_MAGIC +#endif//R_MAGIC + +// **IMPORTANT NOTE** Magic, PrevSize, Size and Status must be at same offset +// in both the R_FREE and R_USED structures + +typedef struct _R_FREE +{ +#ifdef R_MAGIC + rulong FreeMagic; +#endif//R_MAGIC + rulong PrevSize : 30; + rulong Status : 2; + rulong Size; +#if R_STACK + ULONG_PTR LastOwnerStack[R_STACK]; +#endif//R_STACK + struct _R_FREE* NextFree; + struct _R_FREE* PrevFree; +} +R_FREE, *PR_FREE; + +typedef struct _R_USED +{ +#ifdef R_MAGIC + rulong UsedMagic; +#endif//R_MAGIC + rulong PrevSize : 30; + rulong Status : 2; + rulong Size; +#if R_STACK + ULONG_PTR LastOwnerStack[R_STACK]; +#endif//R_STACK + struct _R_USED* NextUsed; +#if R_RZ + rulong UserSize; // how many bytes the user actually asked for... +#endif//R_RZ + rulong Tag; +} +R_USED, *PR_USED; + +typedef struct _R_QUE +{ + rulong Count; + PR_USED First, Last; +} +R_QUE, *PR_QUE; + +typedef struct _R_POOL +{ + void* PoolBase; + rulong PoolSize; + void* UserBase; + rulong UserSize; + rulong Alignments[3]; + PR_FREE FirstFree, LastFree; + R_QUE Que[R_QUECOUNT][3]; + R_MUTEX Mutex; +} +R_POOL, *PR_POOL; + +PVOID MmPagedPoolBase; +ULONG MmPagedPoolSize; +ULONG MmTotalPagedPoolQuota = 0; // TODO FIXME commented out until we use it +static PR_POOL MmPagedPool = NULL; + +/* FUNCTIONS*******************************************************************/ + +#if !R_STACK +#define RiPrintLastOwner(Block) +#else +static void +RiPrintLastOwner ( PR_USED Block ) +{ + int i; + for ( i = 0; i < R_STACK; i++ ) + { + if ( Block->LastOwnerStack[i] != 0xDEADBEEF ) + { + R_DEBUG(" "); + //if (!R_PRINT_ADDRESS ((PVOID)Block->LastOwnerStack[i]) ) + { + R_DEBUG("<%X>", Block->LastOwnerStack[i] ); + } + } + } +} +#endif//R_STACK + +static int +RQueWhich ( rulong size ) +{ + rulong que, quesize; + for ( que=0, quesize=16; que < R_QUECOUNT; que++, quesize<<=1 ) + { + if ( quesize >= size ) + { + return que; + } + } + return -1; +} + +static void +RQueInit ( PR_QUE que ) +{ + que->Count = 0; + que->First = NULL; + que->Last = NULL; +} + +static void +RQueAdd ( PR_QUE que, PR_USED Item ) +{ + ASSERT(Item); + Item->Status = 2; + Item->NextUsed = NULL; + ++que->Count; + if ( !que->Last ) + { + que->First = que->Last = Item; + return; + } + ASSERT(!que->Last->NextUsed); + que->Last->NextUsed = Item; + que->Last = Item; +} + +static PR_USED +RQueRemove ( PR_QUE que ) +{ + PR_USED Item; +#if R_QUEMIN + if ( que->count < R_QUEMIN ) + return NULL; +#endif + if ( !que->First ) + return NULL; + Item = que->First; + que->First = Item->NextUsed; + if ( !--que->Count ) + { + ASSERT ( !que->First ); + que->Last = NULL; + } + Item->Status = 0; + return Item; +} + +static void +RPoolAddFree ( PR_POOL pool, PR_FREE Item ) +{ + ASSERT(pool); + ASSERT(Item); + if ( !pool->FirstFree ) + { + pool->FirstFree = pool->LastFree = Item; + Item->NextFree = NULL; + } + else + { + pool->FirstFree->PrevFree = Item; + Item->NextFree = pool->FirstFree; + pool->FirstFree = Item; + } + Item->PrevFree = NULL; +} + +static void +RPoolRemoveFree ( PR_POOL pool, PR_FREE Item ) +{ + ASSERT(pool); + ASSERT(Item); + if ( Item->NextFree ) + Item->NextFree->PrevFree = Item->PrevFree; + else + { + ASSERT ( pool->LastFree == Item ); + pool->LastFree = Item->PrevFree; + } + if ( Item->PrevFree ) + Item->PrevFree->NextFree = Item->NextFree; + else + { + ASSERT ( pool->FirstFree == Item ); + pool->FirstFree = Item->NextFree; + } +#if DBG + Item->NextFree = Item->PrevFree = (PR_FREE)(ULONG_PTR)0xDEADBEEF; +#endif//DBG +} + +#if !R_STACK +#define RFreeFillStack(free) +#define RUsedFillStack(used) +#else +static void +RFreeFillStack ( PR_FREE free ) +{ + int i; + ULONG stack[R_STACK+3]; // need to skip 3 known levels of stack trace + memset ( stack, 0xCD, sizeof(stack) ); + R_GET_STACK_FRAMES ( stack, R_STACK+3 ); + for ( i = 0; i < R_STACK; i++ ) + free->LastOwnerStack[i] = stack[i+3]; +} + +static void +RUsedFillStack ( PR_USED used ) +{ + int i; + ULONG stack[R_STACK+2]; // need to skip 2 known levels of stack trace + memset ( stack, 0xCD, sizeof(stack) ); + R_GET_STACK_FRAMES ( stack, R_STACK+2 ); + for ( i = 0; i < R_STACK; i++ ) + used->LastOwnerStack[i] = stack[i+2]; +} +#endif + +static PR_FREE +RFreeInit ( void* memory ) +{ + PR_FREE block = (PR_FREE)memory; +#if R_FREEMAGIC + block->FreeMagic = R_FREE_MAGIC; +#endif//R_FREEMAGIC + block->Status = 0; + RFreeFillStack ( block ); +#if DBG + block->PrevFree = block->NextFree = (PR_FREE)(ULONG_PTR)0xDEADBEEF; +#endif//DBG + return block; +} + +PR_POOL +RPoolInit ( void* PoolBase, rulong PoolSize, int align1, int align2, int align3 ) +{ + int align, que; + PR_POOL pool = (PR_POOL)PoolBase; + + pool->PoolBase = PoolBase; + pool->PoolSize = PoolSize; + pool->UserBase = (char*)pool->PoolBase + sizeof(R_POOL); + pool->UserSize = PoolSize - sizeof(R_POOL); + pool->Alignments[0] = align1; + pool->Alignments[1] = align2; + pool->Alignments[2] = align3; + pool->FirstFree = pool->LastFree = NULL; + + RPoolAddFree ( pool, + RFreeInit ( pool->UserBase )); + + pool->FirstFree->PrevSize = 0; + pool->FirstFree->Size = pool->UserSize; + + for ( que = 0; que < R_QUECOUNT; que++ ) + { + for ( align = 0; align < 3; align++ ) + { + RQueInit ( &pool->Que[que][align] ); + } + } + return pool; +} + +#if R_RZ +static const char* +RFormatTag ( rulong Tag, char* buf ) +{ + int i; + *(rulong*)&buf[0] = Tag; + buf[4] = 0; + for ( i = 0; i < 4; i++ ) + { + if ( !buf[i] ) + buf[i] = ' '; + } + return buf; +} +#endif + +#if !R_RZ +#define RUsedRedZoneCheck(pUsed,Addr,file,line, printzone) +#else//R_RZ +static void +RiBadBlock ( PR_USED pUsed, char* Addr, const char* violation, const char* file, int line, int printzone ) +{ + char tag[5]; + unsigned int i; + + R_DEBUG("%s(%i): %s detected for paged pool address 0x%x\n", + file, line, violation, Addr ); + +#ifdef R_MAGIC + R_DEBUG ( "UsedMagic 0x%x, ", pUsed->UsedMagic ); +#endif//R_MAGIC + R_DEBUG ( "Tag %s(%X), Size %i, UserSize %i", + RFormatTag(pUsed->Tag,tag), + pUsed->Tag, + pUsed->Size, + pUsed->UserSize ); + + if ( printzone ) + { + unsigned char* HiZone = (unsigned char*)Addr + pUsed->UserSize; + unsigned char* LoZone = (unsigned char*)Addr - R_RZ; // this is to simplify indexing below... + R_DEBUG ( ", LoZone " ); + for ( i = 0; i < R_RZ; i++ ) + R_DEBUG ( "%02x", LoZone[i] ); + R_DEBUG ( ", HiZone " ); + for ( i = 0; i < R_RZ; i++ ) + R_DEBUG ( "%02x", HiZone[i] ); + } + R_DEBUG ( "\n" ); + + R_DEBUG ( "First few Stack Frames:" ); + RiPrintLastOwner ( pUsed ); + R_DEBUG ( "\n" ); + + R_DEBUG ( "Contents of Block:\n" ); + for ( i = 0; i < 8*16 && i < pUsed->UserSize; i += 16 ) + { + int j; + R_DEBUG ( "%04X ", i ); + for ( j = 0; j < 16; j++ ) + { + if ( i+j < pUsed->UserSize ) + { + R_DEBUG ( "%02X ", (unsigned)(unsigned char)Addr[i+j] ); + } + else + { + R_DEBUG ( " " ); + } + } + R_DEBUG(" "); + for ( j = 0; j < 16; j++ ) + { + if ( i+j < pUsed->UserSize ) + { + char c = Addr[i+j]; + if ( c < 0x20 || c > 0x7E ) + c = '.'; + R_DEBUG ( "%c", c ); + } + else + { + R_DEBUG ( " " ); + } + } + R_DEBUG("\n"); + } + R_PANIC(); +} +static void +RUsedRedZoneCheck ( PR_POOL pool, PR_USED pUsed, char* Addr, const char* file, int line ) +{ + int i; + unsigned char *LoZone, *HiZone; + int bLow = 1; + int bHigh = 1; + + ASSERT ( Addr >= (char*)pool->UserBase && Addr < ((char*)pool->UserBase + pool->UserSize - 16) ); +#ifdef R_MAGIC + if ( pUsed->UsedMagic == R_FREE_MAGIC ) + { + pUsed->UserSize = 0; // just to keep from confusion, MmpBadBlock() doesn't return... + RiBadBlock ( pUsed, Addr, "double-free", file, line, 0 ); + } + if ( pUsed->UsedMagic != R_USED_MAGIC ) + { + RiBadBlock ( pUsed, Addr, "bad magic", file, line, 0 ); + } +#endif//R_MAGIC + switch ( pUsed->Status ) + { + case 0: // freed into main pool + case 2: // in ques + RiBadBlock ( pUsed, Addr, "double-free", file, line, 0 ); + // no need for break here - RiBadBlock doesn't return + case 1: // allocated - this is okay + break; + default: + RiBadBlock ( pUsed, Addr, "corrupt status", file, line, 0 ); + } + if ( pUsed->Status != 1 ) + { + RiBadBlock ( pUsed, Addr, "double-free", file, line, 0 ); + } + if ( pUsed->Size > pool->PoolSize || pUsed->Size == 0 ) + { + RiBadBlock ( pUsed, Addr, "invalid size", file, line, 0 ); + } + if ( pUsed->UserSize > pool->PoolSize || pUsed->UserSize == 0 ) + { + RiBadBlock ( pUsed, Addr, "invalid user size", file, line, 0 ); + } + HiZone = (unsigned char*)Addr + pUsed->UserSize; + LoZone = (unsigned char*)Addr - R_RZ; // this is to simplify indexing below... + for ( i = 0; i < R_RZ && bLow && bHigh; i++ ) + { + bLow = bLow && ( LoZone[i] == R_RZ_LOVALUE ); + bHigh = bHigh && ( HiZone[i] == R_RZ_HIVALUE ); + } + if ( !bLow || !bHigh ) + { + const char* violation = "High and Low-side redzone overwrite"; + if ( bHigh ) // high is okay, so it was just low failed + violation = "Low-side redzone overwrite"; + else if ( bLow ) // low side is okay, so it was just high failed + violation = "High-side redzone overwrite"; + RiBadBlock ( pUsed, Addr, violation, file, line, 1 ); + } +} +#endif//R_RZ + +PR_FREE +RPreviousBlock ( PR_FREE Block ) +{ + if ( Block->PrevSize > 0 ) + return (PR_FREE)( (char*)Block - Block->PrevSize ); + return NULL; +} + +PR_FREE +RNextBlock ( PR_POOL pool, PR_FREE Block ) +{ + PR_FREE NextBlock = (PR_FREE)( (char*)Block + Block->Size ); + if ( (char*)NextBlock >= (char*)pool->UserBase + pool->UserSize ) + NextBlock = NULL; + return NextBlock; +} + +static __inline void* +RHdrToBody ( void* blk ) +/* + * FUNCTION: Translate a block header address to the corresponding block + * address (internal) + */ +{ + return ( (void *) ((char*)blk + sizeof(R_USED) + R_RZ) ); +} + +static __inline PR_USED +RBodyToHdr ( void* addr ) +{ + return (PR_USED) + ( ((char*)addr) - sizeof(R_USED) - R_RZ ); +} + +static int +RiInFreeChain ( PR_POOL pool, PR_FREE Block ) +{ + PR_FREE Free; + Free = pool->FirstFree; + if ( Free == Block ) + return 1; + while ( Free != Block ) + { + Free = Free->NextFree; + if ( !Free ) + return 0; + } + return 1; +} + +static void +RPoolRedZoneCheck ( PR_POOL pool, const char* file, int line ) +{ + { + PR_USED Block = (PR_USED)pool->UserBase; + PR_USED NextBlock; + + for ( ;; ) + { + switch ( Block->Status ) + { + case 0: // block is in chain + ASSERT ( RiInFreeChain ( pool, (PR_FREE)Block ) ); + break; + case 1: // block is allocated + RUsedRedZoneCheck ( pool, Block, RHdrToBody(Block), file, line ); + break; + case 2: // block is in que + // nothing to verify here yet + break; + default: + ASSERT ( !"invalid status in memory block found in pool!" ); + } + NextBlock = (PR_USED)RNextBlock(pool,(PR_FREE)Block); + if ( !NextBlock ) + break; + ASSERT ( NextBlock->PrevSize == Block->Size ); + Block = NextBlock; + } + } + { + // now let's step through the list of free pointers and verify + // each one can be found by size-jumping... + PR_FREE Free = (PR_FREE)pool->FirstFree; + while ( Free ) + { + PR_FREE NextFree = (PR_FREE)pool->UserBase; + if ( Free != NextFree ) + { + while ( NextFree != Free ) + { + NextFree = RNextBlock ( pool, NextFree ); + ASSERT(NextFree); + } + } + Free = Free->NextFree; + } + } +} + +static void +RSetSize ( PR_POOL pool, PR_FREE Block, rulong NewSize, PR_FREE NextBlock ) +{ + R_ASSERT_PTR(pool,Block); + ASSERT ( NewSize < pool->UserSize ); + ASSERT ( NewSize >= sizeof(R_FREE) ); + Block->Size = NewSize; + if ( !NextBlock ) + NextBlock = RNextBlock ( pool, Block ); + if ( NextBlock ) + NextBlock->PrevSize = NewSize; +} + +static PR_FREE +RFreeSplit ( PR_POOL pool, PR_FREE Block, rulong NewSize ) +{ + PR_FREE NewBlock = (PR_FREE)((char*)Block + NewSize); + RSetSize ( pool, NewBlock, Block->Size - NewSize, NULL ); + RSetSize ( pool, Block, NewSize, NewBlock ); + RFreeInit ( NewBlock ); + RPoolAddFree ( pool, NewBlock ); + return NewBlock; +} + +static void +RFreeMerge ( PR_POOL pool, PR_FREE First, PR_FREE Second ) +{ + ASSERT ( RPreviousBlock(Second) == First ); + ASSERT ( First->Size == Second->PrevSize ); + RPoolRemoveFree ( pool, Second ); + RSetSize ( pool, First, First->Size + Second->Size, NULL ); +} + +static void +RPoolReclaim ( PR_POOL pool, PR_FREE FreeBlock ) +{ + PR_FREE NextBlock, PreviousBlock; + + RFreeInit ( FreeBlock ); + RPoolAddFree ( pool, FreeBlock ); + + // TODO FIXME - don't merge and always insert freed blocks at the end for debugging purposes... + + /* + * If the next block is immediately adjacent to the newly freed one then + * merge them. + * PLEASE DO NOT WIPE OUT 'MAGIC' OR 'LASTOWNER' DATA FOR MERGED FREE BLOCKS + */ + NextBlock = RNextBlock ( pool, FreeBlock ); + if ( NextBlock != NULL && !NextBlock->Status ) + { + RFreeMerge ( pool, FreeBlock, NextBlock ); + } + + /* + * If the previous block is adjacent to the newly freed one then + * merge them. + * PLEASE DO NOT WIPE OUT 'MAGIC' OR 'LASTOWNER' DATA FOR MERGED FREE BLOCKS + */ + PreviousBlock = RPreviousBlock ( FreeBlock ); + if ( PreviousBlock != NULL && !PreviousBlock->Status ) + { + RFreeMerge ( pool, PreviousBlock, FreeBlock ); + } +} + +static void +RiUsedInit ( PR_USED Block, rulong Tag ) +{ + Block->Status = 1; + RUsedFillStack ( Block ); +#ifdef R_MAGIC + Block->UsedMagic = R_USED_MAGIC; +#endif//R_MAGIC + //ASSERT_SIZE ( Block->Size ); + + // now add the block to the used block list +#if DBG + Block->NextUsed = (PR_USED)(ULONG_PTR)0xDEADBEEF; +#endif//R_USED_LIST + + Block->Tag = Tag; +} + +#if !R_RZ +#define RiUsedInitRedZone(Block,UserSize) +#else//R_RZ +static void +RiUsedInitRedZone ( PR_USED Block, rulong UserSize ) +{ + // write out buffer-overrun detection bytes + char* Addr = (char*)RHdrToBody(Block); + Block->UserSize = UserSize; + memset ( Addr - R_RZ, R_RZ_LOVALUE, R_RZ ); + memset ( Addr + Block->UserSize, R_RZ_HIVALUE, R_RZ ); +#if DBG + memset ( Addr, 0xCD, UserSize ); +#endif//DBG +} +#endif//R_RZ + +static void* +RPoolAlloc ( PR_POOL pool, rulong NumberOfBytes, rulong Tag, rulong align ) +{ + PR_USED NewBlock; + PR_FREE BestBlock, + NextBlock, + PreviousBlock, + BestPreviousBlock, + CurrentBlock; + void* BestAlignedAddr; + int que, + queBytes = NumberOfBytes; + rulong BlockSize, + Alignment; + int que_reclaimed = 0; + + ASSERT ( pool ); + ASSERT ( align < 3 ); + + R_ACQUIRE_MUTEX(pool); + + if ( !NumberOfBytes ) + { + R_DEBUG("0 bytes requested - initiating pool verification\n"); + RPoolRedZoneCheck ( pool, __FILE__, __LINE__ ); + R_RELEASE_MUTEX(pool); + return NULL; + } + if ( NumberOfBytes > pool->PoolSize ) + { + if ( R_IS_POOL_PTR(pool,NumberOfBytes) ) + { + R_DEBUG("red zone verification requested for block 0x%X\n", NumberOfBytes ); + RUsedRedZoneCheck(pool,RBodyToHdr((void*)(ULONG_PTR)NumberOfBytes), (char*)(ULONG_PTR)NumberOfBytes, __FILE__, __LINE__ ); + R_RELEASE_MUTEX(pool); + return NULL; + } + R_DEBUG("Invalid allocation request: %i bytes\n", NumberOfBytes ); + R_RELEASE_MUTEX(pool); + return NULL; + } + + que = RQueWhich ( NumberOfBytes ); + if ( que >= 0 ) + { + if ( (NewBlock = RQueRemove ( &pool->Que[que][align] )) ) + { + RiUsedInit ( NewBlock, Tag ); + RiUsedInitRedZone ( NewBlock, NumberOfBytes ); + R_RELEASE_MUTEX(pool); + return RHdrToBody(NewBlock); + } + queBytes = 16 << que; + } + + /* + * Calculate the total number of bytes we will need. + */ + BlockSize = queBytes + sizeof(R_USED) + 2*R_RZ; + if (BlockSize < sizeof(R_FREE)) + { + /* At least we need the size of the free block header. */ + BlockSize = sizeof(R_FREE); + } + +try_again: + /* + * Find the best-fitting block. + */ + BestBlock = NULL; + Alignment = pool->Alignments[align]; + PreviousBlock = NULL; + BestPreviousBlock = NULL, + CurrentBlock = pool->FirstFree; + BestAlignedAddr = NULL; + + while ( CurrentBlock != NULL ) + { + PVOID Addr = RHdrToBody(CurrentBlock); + PVOID CurrentBlockEnd = (char*)CurrentBlock + CurrentBlock->Size; + /* calculate last size-aligned address available within this block */ + PVOID AlignedAddr = R_ROUND_DOWN((char*)CurrentBlockEnd-queBytes-R_RZ, Alignment); + ASSERT ( (char*)AlignedAddr+queBytes+R_RZ <= (char*)CurrentBlockEnd ); + + /* special case, this address is already size-aligned, and the right size */ + if ( Addr == AlignedAddr ) + { + BestAlignedAddr = AlignedAddr; + BestPreviousBlock = PreviousBlock; + BestBlock = CurrentBlock; + break; + } + // if we carve out a size-aligned block... is it still past the end of this + // block's free header? + else if ( (char*)RBodyToHdr(AlignedAddr) + >= (char*)CurrentBlock+sizeof(R_FREE) ) + { + /* + * there's enough room to allocate our size-aligned memory out + * of this block, see if it's a better choice than any previous + * finds + */ + if ( BestBlock == NULL + || BestBlock->Size > CurrentBlock->Size ) + { + BestAlignedAddr = AlignedAddr; + BestPreviousBlock = PreviousBlock; + BestBlock = CurrentBlock; + } + } + + PreviousBlock = CurrentBlock; + CurrentBlock = CurrentBlock->NextFree; + } + + /* + * We didn't find anything suitable at all. + */ + if (BestBlock == NULL) + { + if ( !que_reclaimed ) + { + // reclaim que + int i, j; + for ( i = 0; i < R_QUECOUNT; i++ ) + { + for ( j = 0; j < 3; j++ ) + { + while ( (BestBlock = (PR_FREE)RQueRemove ( &pool->Que[i][j] )) ) + { + RPoolReclaim ( pool, BestBlock ); + } + } + } + + que_reclaimed = 1; + goto try_again; + } + DPRINT1("Trying to allocate %lu bytes from paged pool - nothing suitable found, returning NULL\n", + queBytes ); + R_RELEASE_MUTEX(pool); + return NULL; + } + /* + * we found a best block. If Addr isn't already aligned, we've pre-qualified that + * there's room at the beginning of the block for a free block... + */ + { + void* Addr = RHdrToBody(BestBlock); + if ( BestAlignedAddr != Addr ) + { + PR_FREE NewFreeBlock = RFreeSplit ( + pool, + BestBlock, + (char*)RBodyToHdr(BestAlignedAddr) - (char*)BestBlock ); + ASSERT ( BestAlignedAddr > Addr ); + + //DPRINT ( "breaking off preceding bytes into their own block...\n" ); + /*DPRINT ( "NewFreeBlock 0x%x Size %lu (Old Block's new size %lu) NextFree 0x%x\n", + NewFreeBlock, NewFreeBlock->Size, BestBlock->Size, BestBlock->NextFree );*/ + + /* we want the following code to use our size-aligned block */ + BestPreviousBlock = BestBlock; + BestBlock = NewFreeBlock; + + //VerifyPagedPool(); + } + } + /* + * Is there enough space to create a second block from the unused portion. + */ + if ( (BestBlock->Size - BlockSize) > sizeof(R_FREE) ) + { + /*DPRINT("BestBlock 0x%x Size 0x%x BlockSize 0x%x NewSize 0x%x\n", + BestBlock, BestBlock->Size, BlockSize, NewSize );*/ + + /* + * Create the new free block. + */ + NextBlock = RFreeSplit ( pool, BestBlock, BlockSize ); + //ASSERT_SIZE ( NextBlock->Size ); + } + /* + * Remove the selected block from the list of free blocks. + */ + //DPRINT ( "Removing selected block from free block list\n" ); + RPoolRemoveFree ( pool, BestBlock ); + /* + * Create the new used block header. + */ + NewBlock = (PR_USED)BestBlock; + RiUsedInit ( NewBlock, Tag ); + + /* RtlZeroMemory(RHdrToBody(NewBlock), NumberOfBytes);*/ + + RiUsedInitRedZone ( NewBlock, NumberOfBytes ); + R_RELEASE_MUTEX(pool); + + return RHdrToBody(NewBlock); +} + +static void +RPoolFree ( PR_POOL pool, void* Addr ) +{ + PR_USED UsedBlock; + rulong UsedSize; + PR_FREE FreeBlock; + rulong UserSize; + int que; + + ASSERT(pool); + if ( !Addr ) + { + R_DEBUG("Attempt to free NULL ptr, initiating Red Zone Check\n" ); + R_ACQUIRE_MUTEX(pool); + RPoolRedZoneCheck ( pool, __FILE__, __LINE__ ); + R_RELEASE_MUTEX(pool); + return; + } + R_ASSERT_PTR(pool,Addr); + + UsedBlock = RBodyToHdr(Addr); + UsedSize = UsedBlock->Size; + FreeBlock = (PR_FREE)UsedBlock; +#if R_RZ + UserSize = UsedBlock->UserSize; +#else + UserSize = UsedSize - sizeof(R_USED) - 2*R_RZ; +#endif//R_RZ + + RUsedRedZoneCheck ( pool, UsedBlock, Addr, __FILE__, __LINE__ ); + +#if R_RZ + memset ( Addr, 0xCD, UsedBlock->UserSize ); +#endif + + que = RQueWhich ( UserSize ); + if ( que >= 0 ) + { + int queBytes = 16 << que; + ASSERT( (rulong)queBytes >= UserSize ); + if ( que >= 0 ) + { + int align = 0; + if ( R_ROUND_UP(Addr,pool->Alignments[2]) == Addr ) + align = 2; + else if ( R_ROUND_UP(Addr,pool->Alignments[1]) == Addr ) + align = 1; + R_ACQUIRE_MUTEX(pool); + RQueAdd ( &pool->Que[que][align], UsedBlock ); + R_RELEASE_MUTEX(pool); + return; + } + } + + R_ACQUIRE_MUTEX(pool); + RPoolReclaim ( pool, FreeBlock ); + R_RELEASE_MUTEX(pool); +} + +VOID +INIT_FUNCTION +NTAPI +MmInitializePagedPool(VOID) +{ + /* + * We are still at a high IRQL level at this point so explicitly commit + * the first page of the paged pool before writing the first block header. + */ + MmCommitPagedPoolAddress ( (PVOID)MmPagedPoolBase, FALSE ); + + MmPagedPool = RPoolInit ( MmPagedPoolBase, + MmPagedPoolSize, + MM_POOL_ALIGNMENT, + MM_CACHE_LINE_SIZE, + PAGE_SIZE ); + + ExInitializeFastMutex(&MmPagedPool->Mutex); +} + +PVOID NTAPI +ExAllocatePagedPoolWithTag (IN POOL_TYPE PoolType, + IN ULONG NumberOfBytes, + IN ULONG Tag) +{ + int align; + + if ( NumberOfBytes >= PAGE_SIZE ) + align = 2; + else if ( PoolType & CACHE_ALIGNED_POOL_MASK ) + align = 1; + else + align = 0; + + ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); + + return RPoolAlloc ( MmPagedPool, NumberOfBytes, Tag, align ); +} + +VOID NTAPI +ExFreePagedPool(IN PVOID Block) +{ + ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); + RPoolFree ( MmPagedPool, Block ); +} + +/* EOF */ diff --git a/reactos/ntoskrnl/ntoskrnl-generic.rbuild b/reactos/ntoskrnl/ntoskrnl-generic.rbuild index b20bf9b6a76..c89d8915c99 100644 --- a/reactos/ntoskrnl/ntoskrnl-generic.rbuild +++ b/reactos/ntoskrnl/ntoskrnl-generic.rbuild @@ -414,6 +414,7 @@ pagefile.c pageop.c pe.c + ppool.c procsup.c region.c rmap.c