/* * PROJECT: ReactOS Kernel * LICENSE: BSD - See COPYING.ARM in the top level directory * FILE: ntoskrnl/mm/ARM3/expool.c * PURPOSE: ARM Memory Manager Executive Pool Manager * PROGRAMMERS: ReactOS Portable Systems Group */ /* INCLUDES *******************************************************************/ #include #define NDEBUG #include #define MODULE_INVOLVED_IN_ARM3 #include #undef ExAllocatePoolWithQuota #undef ExAllocatePoolWithQuotaTag /* GLOBALS ********************************************************************/ #define POOL_BIG_TABLE_ENTRY_FREE 0x1 /* * This defines when we shrink or expand the table. * 3 --> keep the number of used entries in the 33%-66% of the table capacity. * 4 --> 25% - 75% * etc. */ #define POOL_BIG_TABLE_USE_RATE 4 typedef struct _POOL_DPC_CONTEXT { PPOOL_TRACKER_TABLE PoolTrackTable; SIZE_T PoolTrackTableSize; PPOOL_TRACKER_TABLE PoolTrackTableExpansion; SIZE_T PoolTrackTableSizeExpansion; } POOL_DPC_CONTEXT, *PPOOL_DPC_CONTEXT; ULONG ExpNumberOfPagedPools; POOL_DESCRIPTOR NonPagedPoolDescriptor; PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1]; PPOOL_DESCRIPTOR PoolVector[2]; PKGUARDED_MUTEX ExpPagedPoolMutex; SIZE_T PoolTrackTableSize, PoolTrackTableMask; SIZE_T PoolBigPageTableSize, PoolBigPageTableHash; ULONG ExpBigTableExpansionFailed; PPOOL_TRACKER_TABLE PoolTrackTable; PPOOL_TRACKER_BIG_PAGES PoolBigPageTable; KSPIN_LOCK ExpTaggedPoolLock; ULONG PoolHitTag; BOOLEAN ExStopBadTags; KSPIN_LOCK ExpLargePoolTableLock; ULONG ExpPoolBigEntriesInUse; ULONG ExpPoolFlags; ULONG ExPoolFailures; ULONGLONG MiLastPoolDumpTime; /* Pool block/header/list access macros */ #define POOL_ENTRY(x) (PPOOL_HEADER)((ULONG_PTR)(x) - sizeof(POOL_HEADER)) #define POOL_FREE_BLOCK(x) (PLIST_ENTRY)((ULONG_PTR)(x) + sizeof(POOL_HEADER)) #define POOL_BLOCK(x, i) (PPOOL_HEADER)((ULONG_PTR)(x) + ((i) * POOL_BLOCK_SIZE)) #define POOL_NEXT_BLOCK(x) POOL_BLOCK((x), (x)->BlockSize) #define POOL_PREV_BLOCK(x) POOL_BLOCK((x), -((x)->PreviousSize)) /* * Pool list access debug macros, similar to Arthur's pfnlist.c work. * Microsoft actually implements similar checks in the Windows Server 2003 SP1 * pool code, but only for checked builds. * * As of Vista, however, an MSDN Blog entry by a Security Team Manager indicates * that these checks are done even on retail builds, due to the increasing * number of kernel-mode attacks which depend on dangling list pointers and other * kinds of list-based attacks. * * For now, I will leave these checks on all the time, but later they are likely * to be DBG-only, at least until there are enough kernel-mode security attacks * against ReactOS to warrant the performance hit. * * For now, these are not made inline, so we can get good stack traces. */ PLIST_ENTRY NTAPI ExpDecodePoolLink(IN PLIST_ENTRY Link) { return (PLIST_ENTRY)((ULONG_PTR)Link & ~1); } PLIST_ENTRY NTAPI ExpEncodePoolLink(IN PLIST_ENTRY Link) { return (PLIST_ENTRY)((ULONG_PTR)Link | 1); } VOID NTAPI ExpCheckPoolLinks(IN PLIST_ENTRY ListHead) { if ((ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Flink)->Blink) != ListHead) || (ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Blink)->Flink) != ListHead)) { KeBugCheckEx(BAD_POOL_HEADER, 3, (ULONG_PTR)ListHead, (ULONG_PTR)ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Flink)->Blink), (ULONG_PTR)ExpDecodePoolLink(ExpDecodePoolLink(ListHead->Blink)->Flink)); } } VOID NTAPI ExpInitializePoolListHead(IN PLIST_ENTRY ListHead) { ListHead->Flink = ListHead->Blink = ExpEncodePoolLink(ListHead); } BOOLEAN NTAPI ExpIsPoolListEmpty(IN PLIST_ENTRY ListHead) { return (ExpDecodePoolLink(ListHead->Flink) == ListHead); } VOID NTAPI ExpRemovePoolEntryList(IN PLIST_ENTRY Entry) { PLIST_ENTRY Blink, Flink; Flink = ExpDecodePoolLink(Entry->Flink); Blink = ExpDecodePoolLink(Entry->Blink); Flink->Blink = ExpEncodePoolLink(Blink); Blink->Flink = ExpEncodePoolLink(Flink); } PLIST_ENTRY NTAPI ExpRemovePoolHeadList(IN PLIST_ENTRY ListHead) { PLIST_ENTRY Entry, Flink; Entry = ExpDecodePoolLink(ListHead->Flink); Flink = ExpDecodePoolLink(Entry->Flink); ListHead->Flink = ExpEncodePoolLink(Flink); Flink->Blink = ExpEncodePoolLink(ListHead); return Entry; } PLIST_ENTRY NTAPI ExpRemovePoolTailList(IN PLIST_ENTRY ListHead) { PLIST_ENTRY Entry, Blink; Entry = ExpDecodePoolLink(ListHead->Blink); Blink = ExpDecodePoolLink(Entry->Blink); ListHead->Blink = ExpEncodePoolLink(Blink); Blink->Flink = ExpEncodePoolLink(ListHead); return Entry; } VOID NTAPI ExpInsertPoolTailList(IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry) { PLIST_ENTRY Blink; ExpCheckPoolLinks(ListHead); Blink = ExpDecodePoolLink(ListHead->Blink); Entry->Flink = ExpEncodePoolLink(ListHead); Entry->Blink = ExpEncodePoolLink(Blink); Blink->Flink = ExpEncodePoolLink(Entry); ListHead->Blink = ExpEncodePoolLink(Entry); ExpCheckPoolLinks(ListHead); } VOID NTAPI ExpInsertPoolHeadList(IN PLIST_ENTRY ListHead, IN PLIST_ENTRY Entry) { PLIST_ENTRY Flink; ExpCheckPoolLinks(ListHead); Flink = ExpDecodePoolLink(ListHead->Flink); Entry->Flink = ExpEncodePoolLink(Flink); Entry->Blink = ExpEncodePoolLink(ListHead); Flink->Blink = ExpEncodePoolLink(Entry); ListHead->Flink = ExpEncodePoolLink(Entry); ExpCheckPoolLinks(ListHead); } VOID NTAPI ExpCheckPoolHeader(IN PPOOL_HEADER Entry) { PPOOL_HEADER PreviousEntry, NextEntry; /* Is there a block before this one? */ if (Entry->PreviousSize) { /* Get it */ PreviousEntry = POOL_PREV_BLOCK(Entry); /* The two blocks must be on the same page! */ if (PAGE_ALIGN(Entry) != PAGE_ALIGN(PreviousEntry)) { /* Something is awry */ KeBugCheckEx(BAD_POOL_HEADER, 6, (ULONG_PTR)PreviousEntry, __LINE__, (ULONG_PTR)Entry); } /* This block should also indicate that it's as large as we think it is */ if (PreviousEntry->BlockSize != Entry->PreviousSize) { /* Otherwise, someone corrupted one of the sizes */ DPRINT1("PreviousEntry BlockSize %lu, tag %.4s. Entry PreviousSize %lu, tag %.4s\n", PreviousEntry->BlockSize, (char *)&PreviousEntry->PoolTag, Entry->PreviousSize, (char *)&Entry->PoolTag); KeBugCheckEx(BAD_POOL_HEADER, 5, (ULONG_PTR)PreviousEntry, __LINE__, (ULONG_PTR)Entry); } } else if (PAGE_ALIGN(Entry) != Entry) { /* If there's no block before us, we are the first block, so we should be on a page boundary */ KeBugCheckEx(BAD_POOL_HEADER, 7, 0, __LINE__, (ULONG_PTR)Entry); } /* This block must have a size */ if (!Entry->BlockSize) { /* Someone must've corrupted this field */ if (Entry->PreviousSize) { PreviousEntry = POOL_PREV_BLOCK(Entry); DPRINT1("PreviousEntry tag %.4s. Entry tag %.4s\n", (char *)&PreviousEntry->PoolTag, (char *)&Entry->PoolTag); } else { DPRINT1("Entry tag %.4s\n", (char *)&Entry->PoolTag); } KeBugCheckEx(BAD_POOL_HEADER, 8, 0, __LINE__, (ULONG_PTR)Entry); } /* Okay, now get the next block */ NextEntry = POOL_NEXT_BLOCK(Entry); /* If this is the last block, then we'll be page-aligned, otherwise, check this block */ if (PAGE_ALIGN(NextEntry) != NextEntry) { /* The two blocks must be on the same page! */ if (PAGE_ALIGN(Entry) != PAGE_ALIGN(NextEntry)) { /* Something is messed up */ KeBugCheckEx(BAD_POOL_HEADER, 9, (ULONG_PTR)NextEntry, __LINE__, (ULONG_PTR)Entry); } /* And this block should think we are as large as we truly are */ if (NextEntry->PreviousSize != Entry->BlockSize) { /* Otherwise, someone corrupted the field */ DPRINT1("Entry BlockSize %lu, tag %.4s. NextEntry PreviousSize %lu, tag %.4s\n", Entry->BlockSize, (char *)&Entry->PoolTag, NextEntry->PreviousSize, (char *)&NextEntry->PoolTag); KeBugCheckEx(BAD_POOL_HEADER, 5, (ULONG_PTR)NextEntry, __LINE__, (ULONG_PTR)Entry); } } } VOID NTAPI ExpCheckPoolAllocation( PVOID P, POOL_TYPE PoolType, ULONG Tag) { PPOOL_HEADER Entry; ULONG i; KIRQL OldIrql; POOL_TYPE RealPoolType; /* Get the pool header */ Entry = ((PPOOL_HEADER)P) - 1; /* Check if this is a large allocation */ if (PAGE_ALIGN(P) == P) { /* Lock the pool table */ KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql); /* Find the pool tag */ for (i = 0; i < PoolBigPageTableSize; i++) { /* Check if this is our allocation */ if (PoolBigPageTable[i].Va == P) { /* Make sure the tag is ok */ if (PoolBigPageTable[i].Key != Tag) { KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, PoolBigPageTable[i].Key, Tag); } break; } } /* Release the lock */ KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); if (i == PoolBigPageTableSize) { /* Did not find the allocation */ //ASSERT(FALSE); } /* Get Pool type by address */ RealPoolType = MmDeterminePoolType(P); } else { /* Verify the tag */ if (Entry->PoolTag != Tag) { DPRINT1("Allocation has wrong pool tag! Expected '%.4s', got '%.4s' (0x%08lx)\n", &Tag, &Entry->PoolTag, Entry->PoolTag); KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Entry->PoolTag, Tag); } /* Check the rest of the header */ ExpCheckPoolHeader(Entry); /* Get Pool type from entry */ RealPoolType = (Entry->PoolType - 1); } /* Should we check the pool type? */ if (PoolType != -1) { /* Verify the pool type */ if (RealPoolType != PoolType) { DPRINT1("Wrong pool type! Expected %s, got %s\n", PoolType & BASE_POOL_TYPE_MASK ? "PagedPool" : "NonPagedPool", (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK ? "PagedPool" : "NonPagedPool"); KeBugCheckEx(BAD_POOL_CALLER, 0xCC, (ULONG_PTR)P, Entry->PoolTag, Tag); } } } VOID NTAPI ExpCheckPoolBlocks(IN PVOID Block) { BOOLEAN FoundBlock = FALSE; SIZE_T Size = 0; PPOOL_HEADER Entry; /* Get the first entry for this page, make sure it really is the first */ Entry = PAGE_ALIGN(Block); ASSERT(Entry->PreviousSize == 0); /* Now scan each entry */ while (TRUE) { /* When we actually found our block, remember this */ if (Entry == Block) FoundBlock = TRUE; /* Now validate this block header */ ExpCheckPoolHeader(Entry); /* And go to the next one, keeping track of our size */ Size += Entry->BlockSize; Entry = POOL_NEXT_BLOCK(Entry); /* If we hit the last block, stop */ if (Size >= (PAGE_SIZE / POOL_BLOCK_SIZE)) break; /* If we hit the end of the page, stop */ if (PAGE_ALIGN(Entry) == Entry) break; } /* We must've found our block, and we must have hit the end of the page */ if ((PAGE_ALIGN(Entry) != Entry) || !(FoundBlock)) { /* Otherwise, the blocks are messed up */ KeBugCheckEx(BAD_POOL_HEADER, 10, (ULONG_PTR)Block, __LINE__, (ULONG_PTR)Entry); } } FORCEINLINE VOID ExpCheckPoolIrqlLevel(IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN PVOID Entry) { // // Validate IRQL: It must be APC_LEVEL or lower for Paged Pool, and it must // be DISPATCH_LEVEL or lower for Non Paged Pool // if (((PoolType & BASE_POOL_TYPE_MASK) == PagedPool) ? (KeGetCurrentIrql() > APC_LEVEL) : (KeGetCurrentIrql() > DISPATCH_LEVEL)) { // // Take the system down // KeBugCheckEx(BAD_POOL_CALLER, !Entry ? POOL_ALLOC_IRQL_INVALID : POOL_FREE_IRQL_INVALID, KeGetCurrentIrql(), PoolType, !Entry ? NumberOfBytes : (ULONG_PTR)Entry); } } FORCEINLINE ULONG ExpComputeHashForTag(IN ULONG Tag, IN SIZE_T BucketMask) { // // Compute the hash by multiplying with a large prime number and then XORing // with the HIDWORD of the result. // // Finally, AND with the bucket mask to generate a valid index/bucket into // the table // ULONGLONG Result = (ULONGLONG)40543 * Tag; return (ULONG)BucketMask & ((ULONG)Result ^ (Result >> 32)); } FORCEINLINE ULONG ExpComputePartialHashForAddress(IN PVOID BaseAddress) { ULONG Result; // // Compute the hash by converting the address into a page number, and then // XORing each nibble with the next one. // // We do *NOT* AND with the bucket mask at this point because big table expansion // might happen. Therefore, the final step of the hash must be performed // while holding the expansion pushlock, and this is why we call this a // "partial" hash only. // Result = (ULONG)((ULONG_PTR)BaseAddress >> PAGE_SHIFT); return (Result >> 24) ^ (Result >> 16) ^ (Result >> 8) ^ Result; } #if DBG /* * FORCEINLINE * BOOLEAN * ExpTagAllowPrint(CHAR Tag); */ #define ExpTagAllowPrint(Tag) \ ((Tag) >= 0x20 /* Space */ && (Tag) <= 0x7E /* Tilde */) #ifdef KDBG #include #endif #ifdef KDBG #define MiDumperPrint(dbg, fmt, ...) \ if (dbg) KdbpPrint(fmt, ##__VA_ARGS__); \ else DPRINT1(fmt, ##__VA_ARGS__) #else #define MiDumperPrint(dbg, fmt, ...) \ DPRINT1(fmt, ##__VA_ARGS__) #endif VOID MiDumpPoolConsumers(BOOLEAN CalledFromDbg, ULONG Tag, ULONG Mask, ULONG Flags) { SIZE_T i; BOOLEAN Verbose; // // Only print header if called from OOM situation // if (!CalledFromDbg) { DPRINT1("---------------------\n"); DPRINT1("Out of memory dumper!\n"); } #ifdef KDBG else { KdbpPrint("Pool Used:\n"); } #endif // // Remember whether we'll have to be verbose // This is the only supported flag! // Verbose = BooleanFlagOn(Flags, 1); // // Print table header // if (Verbose) { MiDumperPrint(CalledFromDbg, "\t\t\t\tNonPaged\t\t\t\t\t\t\tPaged\n"); MiDumperPrint(CalledFromDbg, "Tag\t\tAllocs\t\tFrees\t\tDiff\t\tUsed\t\tAllocs\t\tFrees\t\tDiff\t\tUsed\n"); } else { MiDumperPrint(CalledFromDbg, "\t\tNonPaged\t\t\tPaged\n"); MiDumperPrint(CalledFromDbg, "Tag\t\tAllocs\t\tUsed\t\tAllocs\t\tUsed\n"); } // // We'll extract allocations for all the tracked pools // for (i = 0; i < PoolTrackTableSize; ++i) { PPOOL_TRACKER_TABLE TableEntry; TableEntry = &PoolTrackTable[i]; // // We only care about tags which have allocated memory // if (TableEntry->NonPagedBytes != 0 || TableEntry->PagedBytes != 0) { // // If there's a tag, attempt to do a pretty print // only if it matches the caller's tag, or if // any tag is allowed // For checking whether it matches caller's tag, // use the mask to make sure not to mess with the wildcards // if (TableEntry->Key != 0 && TableEntry->Key != TAG_NONE && (Tag == 0 || (TableEntry->Key & Mask) == (Tag & Mask))) { CHAR Tag[4]; // // Extract each 'component' and check whether they are printable // Tag[0] = TableEntry->Key & 0xFF; Tag[1] = TableEntry->Key >> 8 & 0xFF; Tag[2] = TableEntry->Key >> 16 & 0xFF; Tag[3] = TableEntry->Key >> 24 & 0xFF; if (ExpTagAllowPrint(Tag[0]) && ExpTagAllowPrint(Tag[1]) && ExpTagAllowPrint(Tag[2]) && ExpTagAllowPrint(Tag[3])) { // // Print in direct order to make !poolused TAG usage easier // if (Verbose) { MiDumperPrint(CalledFromDbg, "'%c%c%c%c'\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\n", Tag[0], Tag[1], Tag[2], Tag[3], TableEntry->NonPagedAllocs, TableEntry->NonPagedFrees, (TableEntry->NonPagedAllocs - TableEntry->NonPagedFrees), TableEntry->NonPagedBytes, TableEntry->PagedAllocs, TableEntry->PagedFrees, (TableEntry->PagedAllocs - TableEntry->PagedFrees), TableEntry->PagedBytes); } else { MiDumperPrint(CalledFromDbg, "'%c%c%c%c'\t\t%ld\t\t%ld\t\t%ld\t\t%ld\n", Tag[0], Tag[1], Tag[2], Tag[3], TableEntry->NonPagedAllocs, TableEntry->NonPagedBytes, TableEntry->PagedAllocs, TableEntry->PagedBytes); } } else { if (Verbose) { MiDumperPrint(CalledFromDbg, "0x%08x\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\n", TableEntry->Key, TableEntry->NonPagedAllocs, TableEntry->NonPagedFrees, (TableEntry->NonPagedAllocs - TableEntry->NonPagedFrees), TableEntry->NonPagedBytes, TableEntry->PagedAllocs, TableEntry->PagedFrees, (TableEntry->PagedAllocs - TableEntry->PagedFrees), TableEntry->PagedBytes); } else { MiDumperPrint(CalledFromDbg, "0x%08x\t%ld\t\t%ld\t\t%ld\t\t%ld\n", TableEntry->Key, TableEntry->NonPagedAllocs, TableEntry->NonPagedBytes, TableEntry->PagedAllocs, TableEntry->PagedBytes); } } } else if (Tag == 0 || (Tag & Mask) == (TAG_NONE & Mask)) { if (Verbose) { MiDumperPrint(CalledFromDbg, "Anon\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\t\t%ld\n", TableEntry->NonPagedAllocs, TableEntry->NonPagedFrees, (TableEntry->NonPagedAllocs - TableEntry->NonPagedFrees), TableEntry->NonPagedBytes, TableEntry->PagedAllocs, TableEntry->PagedFrees, (TableEntry->PagedAllocs - TableEntry->PagedFrees), TableEntry->PagedBytes); } else { MiDumperPrint(CalledFromDbg, "Anon\t\t%ld\t\t%ld\t\t%ld\t\t%ld\n", TableEntry->NonPagedAllocs, TableEntry->NonPagedBytes, TableEntry->PagedAllocs, TableEntry->PagedBytes); } } } } if (!CalledFromDbg) { DPRINT1("---------------------\n"); } } #endif /* PRIVATE FUNCTIONS **********************************************************/ CODE_SEG("INIT") VOID NTAPI ExpSeedHotTags(VOID) { ULONG i, Key, Hash, Index; PPOOL_TRACKER_TABLE TrackTable = PoolTrackTable; ULONG TagList[] = { ' oI', ' laH', 'PldM', 'LooP', 'tSbO', ' prI', 'bdDN', 'LprI', 'pOoI', ' ldM', 'eliF', 'aVMC', 'dSeS', 'CFtN', 'looP', 'rPCT', 'bNMC', 'dTeS', 'sFtN', 'TPCT', 'CPCT', ' yeK', 'qSbO', 'mNoI', 'aEoI', 'cPCT', 'aFtN', '0ftN', 'tceS', 'SprI', 'ekoT', ' eS', 'lCbO', 'cScC', 'lFtN', 'cAeS', 'mfSF', 'kWcC', 'miSF', 'CdfA', 'EdfA', 'orSF', 'nftN', 'PRIU', 'rFpN', 'RFpN', 'aPeS', 'sUeS', 'FpcA', 'MpcA', 'cSeS', 'mNbO', 'sFpN', 'uLeS', 'DPcS', 'nevE', 'vrqR', 'ldaV', ' pP', 'SdaV', ' daV', 'LdaV', 'FdaV', ' GIB', }; // // Loop all 64 hot tags // ASSERT((sizeof(TagList) / sizeof(ULONG)) == 64); for (i = 0; i < sizeof(TagList) / sizeof(ULONG); i++) { // // Get the current tag, and compute its hash in the tracker table // Key = TagList[i]; Hash = ExpComputeHashForTag(Key, PoolTrackTableMask); // // Loop all the hashes in this index/bucket // Index = Hash; while (TRUE) { // // Find an empty entry, and make sure this isn't the last hash that // can fit. // // On checked builds, also make sure this is the first time we are // seeding this tag. // ASSERT(TrackTable[Hash].Key != Key); if (!(TrackTable[Hash].Key) && (Hash != PoolTrackTableSize - 1)) { // // It has been seeded, move on to the next tag // TrackTable[Hash].Key = Key; break; } // // This entry was already taken, compute the next possible hash while // making sure we're not back at our initial index. // ASSERT(TrackTable[Hash].Key != Key); Hash = (Hash + 1) & PoolTrackTableMask; if (Hash == Index) break; } } } VOID NTAPI ExpRemovePoolTracker(IN ULONG Key, IN SIZE_T NumberOfBytes, IN POOL_TYPE PoolType) { ULONG Hash, Index; PPOOL_TRACKER_TABLE Table, TableEntry; SIZE_T TableMask, TableSize; // // Remove the PROTECTED_POOL flag which is not part of the tag // Key &= ~PROTECTED_POOL; // // With WinDBG you can set a tag you want to break on when an allocation is // attempted // if (Key == PoolHitTag) DbgBreakPoint(); // // Why the double indirection? Because normally this function is also used // when doing session pool allocations, which has another set of tables, // sizes, and masks that live in session pool. Now we don't support session // pool so we only ever use the regular tables, but I'm keeping the code this // way so that the day we DO support session pool, it won't require that // many changes // Table = PoolTrackTable; TableMask = PoolTrackTableMask; TableSize = PoolTrackTableSize; DBG_UNREFERENCED_LOCAL_VARIABLE(TableSize); // // Compute the hash for this key, and loop all the possible buckets // Hash = ExpComputeHashForTag(Key, TableMask); Index = Hash; while (TRUE) { // // Have we found the entry for this tag? */ // TableEntry = &Table[Hash]; if (TableEntry->Key == Key) { // // Decrement the counters depending on if this was paged or nonpaged // pool // if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { InterlockedIncrement(&TableEntry->NonPagedFrees); InterlockedExchangeAddSizeT(&TableEntry->NonPagedBytes, -(SSIZE_T)NumberOfBytes); return; } InterlockedIncrement(&TableEntry->PagedFrees); InterlockedExchangeAddSizeT(&TableEntry->PagedBytes, -(SSIZE_T)NumberOfBytes); return; } // // We should have only ended up with an empty entry if we've reached // the last bucket // if (!TableEntry->Key) { DPRINT1("Empty item reached in tracker table. Hash=0x%lx, TableMask=0x%lx, Tag=0x%08lx, NumberOfBytes=%lu, PoolType=%d\n", Hash, TableMask, Key, (ULONG)NumberOfBytes, PoolType); ASSERT(Hash == TableMask); } // // This path is hit when we don't have an entry, and the current bucket // is full, so we simply try the next one // Hash = (Hash + 1) & TableMask; if (Hash == Index) break; } // // And finally this path is hit when all the buckets are full, and we need // some expansion. This path is not yet supported in ReactOS and so we'll // ignore the tag // DPRINT1("Out of pool tag space, ignoring...\n"); } VOID NTAPI ExpInsertPoolTracker(IN ULONG Key, IN SIZE_T NumberOfBytes, IN POOL_TYPE PoolType) { ULONG Hash, Index; KIRQL OldIrql; PPOOL_TRACKER_TABLE Table, TableEntry; SIZE_T TableMask, TableSize; // // Remove the PROTECTED_POOL flag which is not part of the tag // Key &= ~PROTECTED_POOL; // // With WinDBG you can set a tag you want to break on when an allocation is // attempted // if (Key == PoolHitTag) DbgBreakPoint(); // // There is also an internal flag you can set to break on malformed tags // if (ExStopBadTags) ASSERT(Key & 0xFFFFFF00); // // ASSERT on ReactOS features not yet supported // ASSERT(!(PoolType & SESSION_POOL_MASK)); ASSERT(KeGetCurrentProcessorNumber() == 0); // // Why the double indirection? Because normally this function is also used // when doing session pool allocations, which has another set of tables, // sizes, and masks that live in session pool. Now we don't support session // pool so we only ever use the regular tables, but I'm keeping the code this // way so that the day we DO support session pool, it won't require that // many changes // Table = PoolTrackTable; TableMask = PoolTrackTableMask; TableSize = PoolTrackTableSize; DBG_UNREFERENCED_LOCAL_VARIABLE(TableSize); // // Compute the hash for this key, and loop all the possible buckets // Hash = ExpComputeHashForTag(Key, TableMask); Index = Hash; while (TRUE) { // // Do we already have an entry for this tag? */ // TableEntry = &Table[Hash]; if (TableEntry->Key == Key) { // // Increment the counters depending on if this was paged or nonpaged // pool // if ((PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { InterlockedIncrement(&TableEntry->NonPagedAllocs); InterlockedExchangeAddSizeT(&TableEntry->NonPagedBytes, NumberOfBytes); return; } InterlockedIncrement(&TableEntry->PagedAllocs); InterlockedExchangeAddSizeT(&TableEntry->PagedBytes, NumberOfBytes); return; } // // We don't have an entry yet, but we've found a free bucket for it // if (!(TableEntry->Key) && (Hash != PoolTrackTableSize - 1)) { // // We need to hold the lock while creating a new entry, since other // processors might be in this code path as well // ExAcquireSpinLock(&ExpTaggedPoolLock, &OldIrql); if (!PoolTrackTable[Hash].Key) { // // We've won the race, so now create this entry in the bucket // ASSERT(Table[Hash].Key == 0); PoolTrackTable[Hash].Key = Key; TableEntry->Key = Key; } ExReleaseSpinLock(&ExpTaggedPoolLock, OldIrql); // // Now we force the loop to run again, and we should now end up in // the code path above which does the interlocked increments... // continue; } // // This path is hit when we don't have an entry, and the current bucket // is full, so we simply try the next one // Hash = (Hash + 1) & TableMask; if (Hash == Index) break; } // // And finally this path is hit when all the buckets are full, and we need // some expansion. This path is not yet supported in ReactOS and so we'll // ignore the tag // DPRINT1("Out of pool tag space, ignoring...\n"); } CODE_SEG("INIT") VOID NTAPI ExInitializePoolDescriptor(IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock) { PLIST_ENTRY NextEntry, LastEntry; // // Setup the descriptor based on the caller's request // PoolDescriptor->PoolType = PoolType; PoolDescriptor->PoolIndex = PoolIndex; PoolDescriptor->Threshold = Threshold; PoolDescriptor->LockAddress = PoolLock; // // Initialize accounting data // PoolDescriptor->RunningAllocs = 0; PoolDescriptor->RunningDeAllocs = 0; PoolDescriptor->TotalPages = 0; PoolDescriptor->TotalBytes = 0; PoolDescriptor->TotalBigPages = 0; // // Nothing pending for now // PoolDescriptor->PendingFrees = NULL; PoolDescriptor->PendingFreeDepth = 0; // // Loop all the descriptor's allocation lists and initialize them // NextEntry = PoolDescriptor->ListHeads; LastEntry = NextEntry + POOL_LISTS_PER_PAGE; while (NextEntry < LastEntry) { ExpInitializePoolListHead(NextEntry); NextEntry++; } // // Note that ReactOS does not support Session Pool Yet // ASSERT(PoolType != PagedPoolSession); } CODE_SEG("INIT") VOID NTAPI InitializePool(IN POOL_TYPE PoolType, IN ULONG Threshold) { PPOOL_DESCRIPTOR Descriptor; SIZE_T TableSize; ULONG i; // // Check what kind of pool this is // if (PoolType == NonPagedPool) { // // Compute the track table size and convert it from a power of two to an // actual byte size // // NOTE: On checked builds, we'll assert if the registry table size was // invalid, while on retail builds we'll just break out of the loop at // that point. // TableSize = min(PoolTrackTableSize, MmSizeOfNonPagedPoolInBytes >> 8); for (i = 0; i < 32; i++) { if (TableSize & 1) { ASSERT((TableSize & ~1) == 0); if (!(TableSize & ~1)) break; } TableSize >>= 1; } // // If we hit bit 32, than no size was defined in the registry, so // we'll use the default size of 2048 entries. // // Otherwise, use the size from the registry, as long as it's not // smaller than 64 entries. // if (i == 32) { PoolTrackTableSize = 2048; } else { PoolTrackTableSize = max(1 << i, 64); } // // Loop trying with the biggest specified size first, and cut it down // by a power of two each iteration in case not enough memory exist // while (TRUE) { // // Do not allow overflow // if ((PoolTrackTableSize + 1) > (MAXULONG_PTR / sizeof(POOL_TRACKER_TABLE))) { PoolTrackTableSize >>= 1; continue; } // // Allocate the tracker table and exit the loop if this worked // PoolTrackTable = MiAllocatePoolPages(NonPagedPool, (PoolTrackTableSize + 1) * sizeof(POOL_TRACKER_TABLE)); if (PoolTrackTable) break; // // Otherwise, as long as we're not down to the last bit, keep // iterating // if (PoolTrackTableSize == 1) { KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY, TableSize, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); } PoolTrackTableSize >>= 1; } // // Add one entry, compute the hash, and zero the table // PoolTrackTableSize++; PoolTrackTableMask = PoolTrackTableSize - 2; RtlZeroMemory(PoolTrackTable, PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE)); // // Finally, add the most used tags to speed up those allocations // ExpSeedHotTags(); // // We now do the exact same thing with the tracker table for big pages // TableSize = min(PoolBigPageTableSize, MmSizeOfNonPagedPoolInBytes >> 8); for (i = 0; i < 32; i++) { if (TableSize & 1) { ASSERT((TableSize & ~1) == 0); if (!(TableSize & ~1)) break; } TableSize >>= 1; } // // For big pages, the default tracker table is 4096 entries, while the // minimum is still 64 // if (i == 32) { PoolBigPageTableSize = 4096; } else { PoolBigPageTableSize = max(1 << i, 64); } // // Again, run the exact same loop we ran earlier, but this time for the // big pool tracker instead // while (TRUE) { if ((PoolBigPageTableSize + 1) > (MAXULONG_PTR / sizeof(POOL_TRACKER_BIG_PAGES))) { PoolBigPageTableSize >>= 1; continue; } PoolBigPageTable = MiAllocatePoolPages(NonPagedPool, PoolBigPageTableSize * sizeof(POOL_TRACKER_BIG_PAGES)); if (PoolBigPageTable) break; if (PoolBigPageTableSize == 1) { KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY, TableSize, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF); } PoolBigPageTableSize >>= 1; } // // An extra entry is not needed for for the big pool tracker, so just // compute the hash and zero it // PoolBigPageTableHash = PoolBigPageTableSize - 1; RtlZeroMemory(PoolBigPageTable, PoolBigPageTableSize * sizeof(POOL_TRACKER_BIG_PAGES)); for (i = 0; i < PoolBigPageTableSize; i++) { PoolBigPageTable[i].Va = (PVOID)POOL_BIG_TABLE_ENTRY_FREE; } // // During development, print this out so we can see what's happening // DPRINT("EXPOOL: Pool Tracker Table at: 0x%p with 0x%lx bytes\n", PoolTrackTable, PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE)); DPRINT("EXPOOL: Big Pool Tracker Table at: 0x%p with 0x%lx bytes\n", PoolBigPageTable, PoolBigPageTableSize * sizeof(POOL_TRACKER_BIG_PAGES)); // // Insert the generic tracker for all of big pool // ExpInsertPoolTracker('looP', ROUND_TO_PAGES(PoolBigPageTableSize * sizeof(POOL_TRACKER_BIG_PAGES)), NonPagedPool); // // No support for NUMA systems at this time // ASSERT(KeNumberNodes == 1); // // Initialize the tag spinlock // KeInitializeSpinLock(&ExpTaggedPoolLock); // // Initialize the nonpaged pool descriptor // PoolVector[NonPagedPool] = &NonPagedPoolDescriptor; ExInitializePoolDescriptor(PoolVector[NonPagedPool], NonPagedPool, 0, Threshold, NULL); } else { // // No support for NUMA systems at this time // ASSERT(KeNumberNodes == 1); // // Allocate the pool descriptor // Descriptor = ExAllocatePoolWithTag(NonPagedPool, sizeof(KGUARDED_MUTEX) + sizeof(POOL_DESCRIPTOR), 'looP'); if (!Descriptor) { // // This is really bad... // KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY, 0, -1, -1, -1); } // // Setup the vector and guarded mutex for paged pool // PoolVector[PagedPool] = Descriptor; ExpPagedPoolMutex = (PKGUARDED_MUTEX)(Descriptor + 1); ExpPagedPoolDescriptor[0] = Descriptor; KeInitializeGuardedMutex(ExpPagedPoolMutex); ExInitializePoolDescriptor(Descriptor, PagedPool, 0, Threshold, ExpPagedPoolMutex); // // Insert the generic tracker for all of nonpaged pool // ExpInsertPoolTracker('looP', ROUND_TO_PAGES(PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE)), NonPagedPool); } } FORCEINLINE KIRQL ExLockPool(IN PPOOL_DESCRIPTOR Descriptor) { // // Check if this is nonpaged pool // if ((Descriptor->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { // // Use the queued spin lock // return KeAcquireQueuedSpinLock(LockQueueNonPagedPoolLock); } else { // // Use the guarded mutex // KeAcquireGuardedMutex(Descriptor->LockAddress); return APC_LEVEL; } } FORCEINLINE VOID ExUnlockPool(IN PPOOL_DESCRIPTOR Descriptor, IN KIRQL OldIrql) { // // Check if this is nonpaged pool // if ((Descriptor->PoolType & BASE_POOL_TYPE_MASK) == NonPagedPool) { // // Use the queued spin lock // KeReleaseQueuedSpinLock(LockQueueNonPagedPoolLock, OldIrql); } else { // // Use the guarded mutex // KeReleaseGuardedMutex(Descriptor->LockAddress); } } VOID NTAPI ExpGetPoolTagInfoTarget(IN PKDPC Dpc, IN PVOID DeferredContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2) { PPOOL_DPC_CONTEXT Context = DeferredContext; UNREFERENCED_PARAMETER(Dpc); ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); // // Make sure we win the race, and if we did, copy the data atomically // if (KeSignalCallDpcSynchronize(SystemArgument2)) { RtlCopyMemory(Context->PoolTrackTable, PoolTrackTable, Context->PoolTrackTableSize * sizeof(POOL_TRACKER_TABLE)); // // This is here because ReactOS does not yet support expansion // ASSERT(Context->PoolTrackTableSizeExpansion == 0); } // // Regardless of whether we won or not, we must now synchronize and then // decrement the barrier since this is one more processor that has completed // the callback. // KeSignalCallDpcSynchronize(SystemArgument2); KeSignalCallDpcDone(SystemArgument1); } NTSTATUS NTAPI ExGetPoolTagInfo(IN PSYSTEM_POOLTAG_INFORMATION SystemInformation, IN ULONG SystemInformationLength, IN OUT PULONG ReturnLength OPTIONAL) { ULONG TableSize, CurrentLength; ULONG EntryCount; NTSTATUS Status = STATUS_SUCCESS; PSYSTEM_POOLTAG TagEntry; PPOOL_TRACKER_TABLE Buffer, TrackerEntry; POOL_DPC_CONTEXT Context; ASSERT(KeGetCurrentIrql() == PASSIVE_LEVEL); // // Keep track of how much data the caller's buffer must hold // CurrentLength = FIELD_OFFSET(SYSTEM_POOLTAG_INFORMATION, TagInfo); // // Initialize the caller's buffer // TagEntry = &SystemInformation->TagInfo[0]; SystemInformation->Count = 0; // // Capture the number of entries, and the total size needed to make a copy // of the table // EntryCount = (ULONG)PoolTrackTableSize; TableSize = EntryCount * sizeof(POOL_TRACKER_TABLE); // // Allocate the "Generic DPC" temporary buffer // Buffer = ExAllocatePoolWithTag(NonPagedPool, TableSize, 'ofnI'); if (!Buffer) return STATUS_INSUFFICIENT_RESOURCES; // // Do a "Generic DPC" to atomically retrieve the tag and allocation data // Context.PoolTrackTable = Buffer; Context.PoolTrackTableSize = PoolTrackTableSize; Context.PoolTrackTableExpansion = NULL; Context.PoolTrackTableSizeExpansion = 0; KeGenericCallDpc(ExpGetPoolTagInfoTarget, &Context); // // Now parse the results // for (TrackerEntry = Buffer; TrackerEntry < (Buffer + EntryCount); TrackerEntry++) { // // If the entry is empty, skip it // if (!TrackerEntry->Key) continue; // // Otherwise, add one more entry to the caller's buffer, and ensure that // enough space has been allocated in it // SystemInformation->Count++; CurrentLength += sizeof(*TagEntry); if (SystemInformationLength < CurrentLength) { // // The caller's buffer is too small, so set a failure code. The // caller will know the count, as well as how much space is needed. // // We do NOT break out of the loop, because we want to keep incrementing // the Count as well as CurrentLength so that the caller can know the // final numbers // Status = STATUS_INFO_LENGTH_MISMATCH; } else { // // Small sanity check that our accounting is working correctly // ASSERT(TrackerEntry->PagedAllocs >= TrackerEntry->PagedFrees); ASSERT(TrackerEntry->NonPagedAllocs >= TrackerEntry->NonPagedFrees); // // Return the data into the caller's buffer // TagEntry->TagUlong = TrackerEntry->Key; TagEntry->PagedAllocs = TrackerEntry->PagedAllocs; TagEntry->PagedFrees = TrackerEntry->PagedFrees; TagEntry->PagedUsed = TrackerEntry->PagedBytes; TagEntry->NonPagedAllocs = TrackerEntry->NonPagedAllocs; TagEntry->NonPagedFrees = TrackerEntry->NonPagedFrees; TagEntry->NonPagedUsed = TrackerEntry->NonPagedBytes; TagEntry++; } } // // Free the "Generic DPC" temporary buffer, return the buffer length and status // ExFreePoolWithTag(Buffer, 'ofnI'); if (ReturnLength) *ReturnLength = CurrentLength; return Status; } _IRQL_requires_(DISPATCH_LEVEL) static BOOLEAN ExpReallocateBigPageTable( _In_ _IRQL_restores_ KIRQL OldIrql, _In_ BOOLEAN Shrink) { SIZE_T OldSize = PoolBigPageTableSize; SIZE_T NewSize, NewSizeInBytes; PPOOL_TRACKER_BIG_PAGES NewTable; PPOOL_TRACKER_BIG_PAGES OldTable; ULONG i; ULONG PagesFreed; ULONG Hash; ULONG HashMask; /* Must be holding ExpLargePoolTableLock */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); /* Make sure we don't overflow */ if (Shrink) { NewSize = OldSize / 2; /* Make sure we don't shrink too much. */ ASSERT(NewSize >= ExpPoolBigEntriesInUse); NewSize = ALIGN_UP_BY(NewSize, PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES)); ASSERT(NewSize <= OldSize); /* If there is only one page left, then keep it around. Not a failure either. */ if (NewSize == OldSize) { ASSERT(NewSize == (PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES))); KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); return TRUE; } } else { if (!NT_SUCCESS(RtlSIZETMult(2, OldSize, &NewSize))) { DPRINT1("Overflow expanding big page table. Size=%lu\n", OldSize); KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); return FALSE; } /* Make sure we don't stupidly waste pages */ NewSize = ALIGN_DOWN_BY(NewSize, PAGE_SIZE / sizeof(POOL_TRACKER_BIG_PAGES)); ASSERT(NewSize > OldSize); } if (!NT_SUCCESS(RtlSIZETMult(sizeof(POOL_TRACKER_BIG_PAGES), NewSize, &NewSizeInBytes))) { DPRINT1("Overflow while calculating big page table size. Size=%lu\n", OldSize); KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); return FALSE; } NewTable = MiAllocatePoolPages(NonPagedPool, NewSizeInBytes); if (NewTable == NULL) { DPRINT("Could not allocate %lu bytes for new big page table\n", NewSizeInBytes); KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); return FALSE; } DPRINT("%s big pool tracker table to %lu entries\n", Shrink ? "Shrinking" : "Expanding", NewSize); /* Initialize the new table */ RtlZeroMemory(NewTable, NewSizeInBytes); for (i = 0; i < NewSize; i++) { NewTable[i].Va = (PVOID)POOL_BIG_TABLE_ENTRY_FREE; } /* Copy over all items */ OldTable = PoolBigPageTable; HashMask = NewSize - 1; for (i = 0; i < OldSize; i++) { /* Skip over empty items */ if ((ULONG_PTR)OldTable[i].Va & POOL_BIG_TABLE_ENTRY_FREE) { continue; } /* Recalculate the hash due to the new table size */ Hash = ExpComputePartialHashForAddress(OldTable[i].Va) % HashMask; /* Find the location in the new table */ while (!((ULONG_PTR)NewTable[Hash].Va & POOL_BIG_TABLE_ENTRY_FREE)) { if (++Hash == NewSize) Hash = 0; } /* We must have space */ ASSERT((ULONG_PTR)NewTable[Hash].Va & POOL_BIG_TABLE_ENTRY_FREE); /* Finally, copy the item */ NewTable[Hash] = OldTable[i]; } /* Activate the new table */ PoolBigPageTable = NewTable; PoolBigPageTableSize = NewSize; PoolBigPageTableHash = PoolBigPageTableSize - 1; /* Release the lock, we're done changing global state */ KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); /* Free the old table and update our tracker */ PagesFreed = MiFreePoolPages(OldTable); ExpRemovePoolTracker('looP', PagesFreed << PAGE_SHIFT, 0); ExpInsertPoolTracker('looP', ALIGN_UP_BY(NewSizeInBytes, PAGE_SIZE), 0); return TRUE; } BOOLEAN NTAPI ExpAddTagForBigPages(IN PVOID Va, IN ULONG Key, IN ULONG NumberOfPages, IN POOL_TYPE PoolType) { ULONG Hash, i = 0; PVOID OldVa; KIRQL OldIrql; SIZE_T TableSize; PPOOL_TRACKER_BIG_PAGES Entry, EntryEnd, EntryStart; ASSERT(((ULONG_PTR)Va & POOL_BIG_TABLE_ENTRY_FREE) == 0); ASSERT(!(PoolType & SESSION_POOL_MASK)); // // As the table is expandable, these values must only be read after acquiring // the lock to avoid a teared access during an expansion // NOTE: Windows uses a special reader/writer SpinLock to improve // performance in the common case (add/remove a tracker entry) // Retry: Hash = ExpComputePartialHashForAddress(Va); KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql); Hash &= PoolBigPageTableHash; TableSize = PoolBigPageTableSize; // // We loop from the current hash bucket to the end of the table, and then // rollover to hash bucket 0 and keep going from there. If we return back // to the beginning, then we attempt expansion at the bottom of the loop // EntryStart = Entry = &PoolBigPageTable[Hash]; EntryEnd = &PoolBigPageTable[TableSize]; do { // // Make sure that this is a free entry and attempt to atomically make the // entry busy now // NOTE: the Interlocked operation cannot fail with an exclusive SpinLock // OldVa = Entry->Va; if (((ULONG_PTR)OldVa & POOL_BIG_TABLE_ENTRY_FREE) && (NT_VERIFY(InterlockedCompareExchangePointer(&Entry->Va, Va, OldVa) == OldVa))) { // // We now own this entry, write down the size and the pool tag // Entry->Key = Key; Entry->NumberOfPages = NumberOfPages; // // Add one more entry to the count, and see if we're getting within // 75% of the table size, at which point we'll do an expansion now // to avoid blocking too hard later on. // // Note that we only do this if it's also been the 16th time that we // keep losing the race or that we are not finding a free entry anymore, // which implies a massive number of concurrent big pool allocations. // ExpPoolBigEntriesInUse++; if ((i >= 16) && (ExpPoolBigEntriesInUse > (TableSize * (POOL_BIG_TABLE_USE_RATE - 1) / POOL_BIG_TABLE_USE_RATE))) { DPRINT("Attempting expansion since we now have %lu entries\n", ExpPoolBigEntriesInUse); ASSERT(TableSize == PoolBigPageTableSize); ExpReallocateBigPageTable(OldIrql, FALSE); return TRUE; } // // We have our entry, return // KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); return TRUE; } // // We don't have our entry yet, so keep trying, making the entry list // circular if we reach the last entry. We'll eventually break out of // the loop once we've rolled over and returned back to our original // hash bucket // i++; if (++Entry >= EntryEnd) Entry = &PoolBigPageTable[0]; } while (Entry != EntryStart); // // This means there's no free hash buckets whatsoever, so we now have // to attempt expanding the table // ASSERT(TableSize == PoolBigPageTableSize); if (ExpReallocateBigPageTable(OldIrql, FALSE)) { goto Retry; } ExpBigTableExpansionFailed++; DPRINT1("Big pool table expansion failed\n"); return FALSE; } ULONG NTAPI ExpFindAndRemoveTagBigPages(IN PVOID Va, OUT PULONG_PTR BigPages, IN POOL_TYPE PoolType) { BOOLEAN FirstTry = TRUE; SIZE_T TableSize; KIRQL OldIrql; ULONG PoolTag, Hash; PPOOL_TRACKER_BIG_PAGES Entry; ASSERT(((ULONG_PTR)Va & POOL_BIG_TABLE_ENTRY_FREE) == 0); ASSERT(!(PoolType & SESSION_POOL_MASK)); // // As the table is expandable, these values must only be read after acquiring // the lock to avoid a teared access during an expansion // Hash = ExpComputePartialHashForAddress(Va); KeAcquireSpinLock(&ExpLargePoolTableLock, &OldIrql); Hash &= PoolBigPageTableHash; TableSize = PoolBigPageTableSize; // // Loop while trying to find this big page allocation // while (PoolBigPageTable[Hash].Va != Va) { // // Increment the size until we go past the end of the table // if (++Hash >= TableSize) { // // Is this the second time we've tried? // if (!FirstTry) { // // This means it was never inserted into the pool table and it // received the special "BIG" tag -- return that and return 0 // so that the code can ask Mm for the page count instead // KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); *BigPages = 0; return ' GIB'; } // // The first time this happens, reset the hash index and try again // Hash = 0; FirstTry = FALSE; } } // // Now capture all the information we need from the entry, since after we // release the lock, the data can change // Entry = &PoolBigPageTable[Hash]; *BigPages = Entry->NumberOfPages; PoolTag = Entry->Key; // // Set the free bit, and decrement the number of allocations. Finally, release // the lock and return the tag that was located // Entry->Va = (PVOID)((ULONG_PTR)Entry->Va | POOL_BIG_TABLE_ENTRY_FREE); ExpPoolBigEntriesInUse--; /* If reaching 12.5% of the size (or whatever integer rounding gets us to), * halve the allocation size, which will get us to 25% of space used. */ if (ExpPoolBigEntriesInUse < (PoolBigPageTableSize / (POOL_BIG_TABLE_USE_RATE * 2))) { /* Shrink the table. */ ExpReallocateBigPageTable(OldIrql, TRUE); } else { KeReleaseSpinLock(&ExpLargePoolTableLock, OldIrql); } return PoolTag; } VOID NTAPI ExQueryPoolUsage(OUT PULONG PagedPoolPages, OUT PULONG NonPagedPoolPages, OUT PULONG PagedPoolAllocs, OUT PULONG PagedPoolFrees, OUT PULONG PagedPoolLookasideHits, OUT PULONG NonPagedPoolAllocs, OUT PULONG NonPagedPoolFrees, OUT PULONG NonPagedPoolLookasideHits) { ULONG i; PPOOL_DESCRIPTOR PoolDesc; // // Assume all failures // *PagedPoolPages = 0; *PagedPoolAllocs = 0; *PagedPoolFrees = 0; // // Tally up the totals for all the apged pool // for (i = 0; i < ExpNumberOfPagedPools + 1; i++) { PoolDesc = ExpPagedPoolDescriptor[i]; *PagedPoolPages += PoolDesc->TotalPages + PoolDesc->TotalBigPages; *PagedPoolAllocs += PoolDesc->RunningAllocs; *PagedPoolFrees += PoolDesc->RunningDeAllocs; } // // The first non-paged pool has a hardcoded well-known descriptor name // PoolDesc = &NonPagedPoolDescriptor; *NonPagedPoolPages = PoolDesc->TotalPages + PoolDesc->TotalBigPages; *NonPagedPoolAllocs = PoolDesc->RunningAllocs; *NonPagedPoolFrees = PoolDesc->RunningDeAllocs; // // If the system has more than one non-paged pool, copy the other descriptor // totals as well // #if 0 if (ExpNumberOfNonPagedPools > 1) { for (i = 0; i < ExpNumberOfNonPagedPools; i++) { PoolDesc = ExpNonPagedPoolDescriptor[i]; *NonPagedPoolPages += PoolDesc->TotalPages + PoolDesc->TotalBigPages; *NonPagedPoolAllocs += PoolDesc->RunningAllocs; *NonPagedPoolFrees += PoolDesc->RunningDeAllocs; } } #endif // // Get the amount of hits in the system lookaside lists // if (!IsListEmpty(&ExPoolLookasideListHead)) { PLIST_ENTRY ListEntry; for (ListEntry = ExPoolLookasideListHead.Flink; ListEntry != &ExPoolLookasideListHead; ListEntry = ListEntry->Flink) { PGENERAL_LOOKASIDE Lookaside; Lookaside = CONTAINING_RECORD(ListEntry, GENERAL_LOOKASIDE, ListEntry); if (Lookaside->Type == NonPagedPool) { *NonPagedPoolLookasideHits += Lookaside->AllocateHits; } else { *PagedPoolLookasideHits += Lookaside->AllocateHits; } } } } VOID NTAPI ExReturnPoolQuota(IN PVOID P) { PPOOL_HEADER Entry; POOL_TYPE PoolType; USHORT BlockSize; PEPROCESS Process; if ((ExpPoolFlags & POOL_FLAG_SPECIAL_POOL) && (MmIsSpecialPoolAddress(P))) { return; } Entry = P; Entry--; ASSERT((ULONG_PTR)Entry % POOL_BLOCK_SIZE == 0); PoolType = Entry->PoolType - 1; BlockSize = Entry->BlockSize; if (PoolType & QUOTA_POOL_MASK) { Process = ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1]; ASSERT(Process != NULL); if (Process) { if (Process->Pcb.Header.Type != ProcessObject) { DPRINT1("Object %p is not a process. Type %u, pool type 0x%x, block size %u\n", Process, Process->Pcb.Header.Type, Entry->PoolType, BlockSize); KeBugCheckEx(BAD_POOL_CALLER, POOL_BILLED_PROCESS_INVALID, (ULONG_PTR)P, Entry->PoolTag, (ULONG_PTR)Process); } ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = NULL; PsReturnPoolQuota(Process, PoolType & BASE_POOL_TYPE_MASK, BlockSize * POOL_BLOCK_SIZE); ObDereferenceObject(Process); } } } /* PUBLIC FUNCTIONS ***********************************************************/ /* * @implemented */ PVOID NTAPI ExAllocatePoolWithTag(IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag) { PPOOL_DESCRIPTOR PoolDesc; PLIST_ENTRY ListHead; PPOOL_HEADER Entry, NextEntry, FragmentEntry; KIRQL OldIrql; USHORT BlockSize, i; ULONG OriginalType; PKPRCB Prcb = KeGetCurrentPrcb(); PGENERAL_LOOKASIDE LookasideList; // // Some sanity checks // ASSERT(Tag != 0); ASSERT(Tag != ' GIB'); ASSERT(NumberOfBytes != 0); ExpCheckPoolIrqlLevel(PoolType, NumberOfBytes, NULL); // // Not supported in ReactOS // ASSERT(!(PoolType & SESSION_POOL_MASK)); // // Check if verifier or special pool is enabled // if (ExpPoolFlags & (POOL_FLAG_VERIFIER | POOL_FLAG_SPECIAL_POOL)) { // // For verifier, we should call the verification routine // if (ExpPoolFlags & POOL_FLAG_VERIFIER) { DPRINT1("Driver Verifier is not yet supported\n"); } // // For special pool, we check if this is a suitable allocation and do // the special allocation if needed // if (ExpPoolFlags & POOL_FLAG_SPECIAL_POOL) { // // Check if this is a special pool allocation // if (MmUseSpecialPool(NumberOfBytes, Tag)) { // // Try to allocate using special pool // Entry = MmAllocateSpecialPool(NumberOfBytes, Tag, PoolType, 2); if (Entry) return Entry; } } } // // Get the pool type and its corresponding vector for this request // OriginalType = PoolType; PoolType = PoolType & BASE_POOL_TYPE_MASK; PoolDesc = PoolVector[PoolType]; ASSERT(PoolDesc != NULL); // // Check if this is a big page allocation // if (NumberOfBytes > POOL_MAX_ALLOC) { // // Allocate pages for it // Entry = MiAllocatePoolPages(OriginalType, NumberOfBytes); if (!Entry) { #if DBG // // Out of memory, display current consumption // Let's consider that if the caller wanted more // than a hundred pages, that's a bogus caller // and we are not out of memory. Dump at most // once a second to avoid spamming the log. // if (NumberOfBytes < 100 * PAGE_SIZE && KeQueryInterruptTime() >= MiLastPoolDumpTime + 10000000) { MiDumpPoolConsumers(FALSE, 0, 0, 0); MiLastPoolDumpTime = KeQueryInterruptTime(); } #endif // // Must succeed pool is deprecated, but still supported. These allocation // failures must cause an immediate bugcheck // if (OriginalType & MUST_SUCCEED_POOL_MASK) { KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY, NumberOfBytes, NonPagedPoolDescriptor.TotalPages, NonPagedPoolDescriptor.TotalBigPages, 0); } // // Internal debugging // ExPoolFailures++; // // This flag requests printing failures, and can also further specify // breaking on failures // if (ExpPoolFlags & POOL_FLAG_DBGPRINT_ON_FAILURE) { DPRINT1("EX: ExAllocatePool (%lu, 0x%x) returning NULL\n", NumberOfBytes, OriginalType); if (ExpPoolFlags & POOL_FLAG_CRASH_ON_FAILURE) DbgBreakPoint(); } // // Finally, this flag requests an exception, which we are more than // happy to raise! // if (OriginalType & POOL_RAISE_IF_ALLOCATION_FAILURE) { ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); } return NULL; } // // Increment required counters // InterlockedExchangeAdd((PLONG)&PoolDesc->TotalBigPages, (LONG)BYTES_TO_PAGES(NumberOfBytes)); InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, NumberOfBytes); InterlockedIncrement((PLONG)&PoolDesc->RunningAllocs); // // Add a tag for the big page allocation and switch to the generic "BIG" // tag if we failed to do so, then insert a tracker for this alloation. // if (!ExpAddTagForBigPages(Entry, Tag, (ULONG)BYTES_TO_PAGES(NumberOfBytes), OriginalType)) { Tag = ' GIB'; } ExpInsertPoolTracker(Tag, ROUND_TO_PAGES(NumberOfBytes), OriginalType); return Entry; } // // Should never request 0 bytes from the pool, but since so many drivers do // it, we'll just assume they want 1 byte, based on NT's similar behavior // if (!NumberOfBytes) NumberOfBytes = 1; // // A pool allocation is defined by its data, a linked list to connect it to // the free list (if necessary), and a pool header to store accounting info. // Calculate this size, then convert it into a block size (units of pool // headers) // // Note that i cannot overflow (past POOL_LISTS_PER_PAGE) because any such // request would've been treated as a POOL_MAX_ALLOC earlier and resulted in // the direct allocation of pages. // i = (USHORT)((NumberOfBytes + sizeof(POOL_HEADER) + (POOL_BLOCK_SIZE - 1)) / POOL_BLOCK_SIZE); ASSERT(i < POOL_LISTS_PER_PAGE); // // Handle lookaside list optimization for both paged and nonpaged pool // if (i <= NUMBER_POOL_LOOKASIDE_LISTS) { // // Try popping it from the per-CPU lookaside list // LookasideList = (PoolType == PagedPool) ? Prcb->PPPagedLookasideList[i - 1].P : Prcb->PPNPagedLookasideList[i - 1].P; LookasideList->TotalAllocates++; Entry = (PPOOL_HEADER)InterlockedPopEntrySList(&LookasideList->ListHead); if (!Entry) { // // We failed, try popping it from the global list // LookasideList = (PoolType == PagedPool) ? Prcb->PPPagedLookasideList[i - 1].L : Prcb->PPNPagedLookasideList[i - 1].L; LookasideList->TotalAllocates++; Entry = (PPOOL_HEADER)InterlockedPopEntrySList(&LookasideList->ListHead); } // // If we were able to pop it, update the accounting and return the block // if (Entry) { LookasideList->AllocateHits++; // // Get the real entry, write down its pool type, and track it // Entry--; Entry->PoolType = OriginalType + 1; ExpInsertPoolTracker(Tag, Entry->BlockSize * POOL_BLOCK_SIZE, OriginalType); // // Return the pool allocation // Entry->PoolTag = Tag; (POOL_FREE_BLOCK(Entry))->Flink = NULL; (POOL_FREE_BLOCK(Entry))->Blink = NULL; return POOL_FREE_BLOCK(Entry); } } // // Loop in the free lists looking for a block if this size. Start with the // list optimized for this kind of size lookup // ListHead = &PoolDesc->ListHeads[i]; do { // // Are there any free entries available on this list? // if (!ExpIsPoolListEmpty(ListHead)) { // // Acquire the pool lock now // OldIrql = ExLockPool(PoolDesc); // // And make sure the list still has entries // if (ExpIsPoolListEmpty(ListHead)) { // // Someone raced us (and won) before we had a chance to acquire // the lock. // // Try again! // ExUnlockPool(PoolDesc, OldIrql); continue; } // // Remove a free entry from the list // Note that due to the way we insert free blocks into multiple lists // there is a guarantee that any block on this list will either be // of the correct size, or perhaps larger. // ExpCheckPoolLinks(ListHead); Entry = POOL_ENTRY(ExpRemovePoolHeadList(ListHead)); ExpCheckPoolLinks(ListHead); ExpCheckPoolBlocks(Entry); ASSERT(Entry->BlockSize >= i); ASSERT(Entry->PoolType == 0); // // Check if this block is larger that what we need. The block could // not possibly be smaller, due to the reason explained above (and // we would've asserted on a checked build if this was the case). // if (Entry->BlockSize != i) { // // Is there an entry before this one? // if (Entry->PreviousSize == 0) { // // There isn't anyone before us, so take the next block and // turn it into a fragment that contains the leftover data // that we don't need to satisfy the caller's request // FragmentEntry = POOL_BLOCK(Entry, i); FragmentEntry->BlockSize = Entry->BlockSize - i; // // And make it point back to us // FragmentEntry->PreviousSize = i; // // Now get the block that follows the new fragment and check // if it's still on the same page as us (and not at the end) // NextEntry = POOL_NEXT_BLOCK(FragmentEntry); if (PAGE_ALIGN(NextEntry) != NextEntry) { // // Adjust this next block to point to our newly created // fragment block // NextEntry->PreviousSize = FragmentEntry->BlockSize; } } else { // // There is a free entry before us, which we know is smaller // so we'll make this entry the fragment instead // FragmentEntry = Entry; // // And then we'll remove from it the actual size required. // Now the entry is a leftover free fragment // Entry->BlockSize -= i; // // Now let's go to the next entry after the fragment (which // used to point to our original free entry) and make it // reference the new fragment entry instead. // // This is the entry that will actually end up holding the // allocation! // Entry = POOL_NEXT_BLOCK(Entry); Entry->PreviousSize = FragmentEntry->BlockSize; // // And now let's go to the entry after that one and check if // it's still on the same page, and not at the end // NextEntry = POOL_BLOCK(Entry, i); if (PAGE_ALIGN(NextEntry) != NextEntry) { // // Make it reference the allocation entry // NextEntry->PreviousSize = i; } } // // Now our (allocation) entry is the right size // Entry->BlockSize = i; // // And the next entry is now the free fragment which contains // the remaining difference between how big the original entry // was, and the actual size the caller needs/requested. // FragmentEntry->PoolType = 0; BlockSize = FragmentEntry->BlockSize; // // Now check if enough free bytes remained for us to have a // "full" entry, which contains enough bytes for a linked list // and thus can be used for allocations (up to 8 bytes...) // ExpCheckPoolLinks(&PoolDesc->ListHeads[BlockSize - 1]); if (BlockSize != 1) { // // Insert the free entry into the free list for this size // ExpInsertPoolTailList(&PoolDesc->ListHeads[BlockSize - 1], POOL_FREE_BLOCK(FragmentEntry)); ExpCheckPoolLinks(POOL_FREE_BLOCK(FragmentEntry)); } } // // We have found an entry for this allocation, so set the pool type // and release the lock since we're done // Entry->PoolType = OriginalType + 1; ExpCheckPoolBlocks(Entry); ExUnlockPool(PoolDesc, OldIrql); // // Increment required counters // InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, Entry->BlockSize * POOL_BLOCK_SIZE); InterlockedIncrement((PLONG)&PoolDesc->RunningAllocs); // // Track this allocation // ExpInsertPoolTracker(Tag, Entry->BlockSize * POOL_BLOCK_SIZE, OriginalType); // // Return the pool allocation // Entry->PoolTag = Tag; (POOL_FREE_BLOCK(Entry))->Flink = NULL; (POOL_FREE_BLOCK(Entry))->Blink = NULL; return POOL_FREE_BLOCK(Entry); } } while (++ListHead != &PoolDesc->ListHeads[POOL_LISTS_PER_PAGE]); // // There were no free entries left, so we have to allocate a new fresh page // Entry = MiAllocatePoolPages(OriginalType, PAGE_SIZE); if (!Entry) { #if DBG // // Out of memory, display current consumption // Let's consider that if the caller wanted more // than a hundred pages, that's a bogus caller // and we are not out of memory. Dump at most // once a second to avoid spamming the log. // if (NumberOfBytes < 100 * PAGE_SIZE && KeQueryInterruptTime() >= MiLastPoolDumpTime + 10000000) { MiDumpPoolConsumers(FALSE, 0, 0, 0); MiLastPoolDumpTime = KeQueryInterruptTime(); } #endif // // Must succeed pool is deprecated, but still supported. These allocation // failures must cause an immediate bugcheck // if (OriginalType & MUST_SUCCEED_POOL_MASK) { KeBugCheckEx(MUST_SUCCEED_POOL_EMPTY, PAGE_SIZE, NonPagedPoolDescriptor.TotalPages, NonPagedPoolDescriptor.TotalBigPages, 0); } // // Internal debugging // ExPoolFailures++; // // This flag requests printing failures, and can also further specify // breaking on failures // if (ExpPoolFlags & POOL_FLAG_DBGPRINT_ON_FAILURE) { DPRINT1("EX: ExAllocatePool (%lu, 0x%x) returning NULL\n", NumberOfBytes, OriginalType); if (ExpPoolFlags & POOL_FLAG_CRASH_ON_FAILURE) DbgBreakPoint(); } // // Finally, this flag requests an exception, which we are more than // happy to raise! // if (OriginalType & POOL_RAISE_IF_ALLOCATION_FAILURE) { ExRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); } // // Return NULL to the caller in all other cases // return NULL; } // // Setup the entry data // Entry->Ulong1 = 0; Entry->BlockSize = i; Entry->PoolType = OriginalType + 1; // // This page will have two entries -- one for the allocation (which we just // created above), and one for the remaining free bytes, which we're about // to create now. The free bytes are the whole page minus what was allocated // and then converted into units of block headers. // BlockSize = (PAGE_SIZE / POOL_BLOCK_SIZE) - i; FragmentEntry = POOL_BLOCK(Entry, i); FragmentEntry->Ulong1 = 0; FragmentEntry->BlockSize = BlockSize; FragmentEntry->PreviousSize = i; // // Increment required counters // InterlockedIncrement((PLONG)&PoolDesc->TotalPages); InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, Entry->BlockSize * POOL_BLOCK_SIZE); // // Now check if enough free bytes remained for us to have a "full" entry, // which contains enough bytes for a linked list and thus can be used for // allocations (up to 8 bytes...) // if (FragmentEntry->BlockSize != 1) { // // Excellent -- acquire the pool lock // OldIrql = ExLockPool(PoolDesc); // // And insert the free entry into the free list for this block size // ExpCheckPoolLinks(&PoolDesc->ListHeads[BlockSize - 1]); ExpInsertPoolTailList(&PoolDesc->ListHeads[BlockSize - 1], POOL_FREE_BLOCK(FragmentEntry)); ExpCheckPoolLinks(POOL_FREE_BLOCK(FragmentEntry)); // // Release the pool lock // ExpCheckPoolBlocks(Entry); ExUnlockPool(PoolDesc, OldIrql); } else { // // Simply do a sanity check // ExpCheckPoolBlocks(Entry); } // // Increment performance counters and track this allocation // InterlockedIncrement((PLONG)&PoolDesc->RunningAllocs); ExpInsertPoolTracker(Tag, Entry->BlockSize * POOL_BLOCK_SIZE, OriginalType); // // And return the pool allocation // ExpCheckPoolBlocks(Entry); Entry->PoolTag = Tag; return POOL_FREE_BLOCK(Entry); } /* * @implemented */ PVOID NTAPI ExAllocatePool(POOL_TYPE PoolType, SIZE_T NumberOfBytes) { ULONG Tag = TAG_NONE; #if 0 && DBG PLDR_DATA_TABLE_ENTRY LdrEntry; /* Use the first four letters of the driver name, or "None" if unavailable */ LdrEntry = KeGetCurrentIrql() <= APC_LEVEL ? MiLookupDataTableEntry(_ReturnAddress()) : NULL; if (LdrEntry) { ULONG i; Tag = 0; for (i = 0; i < min(4, LdrEntry->BaseDllName.Length / sizeof(WCHAR)); i++) Tag = Tag >> 8 | (LdrEntry->BaseDllName.Buffer[i] & 0xff) << 24; for (; i < 4; i++) Tag = Tag >> 8 | ' ' << 24; } #endif return ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag); } /* * @implemented */ VOID NTAPI ExFreePoolWithTag(IN PVOID P, IN ULONG TagToFree) { PPOOL_HEADER Entry, NextEntry; USHORT BlockSize; KIRQL OldIrql; POOL_TYPE PoolType; PPOOL_DESCRIPTOR PoolDesc; ULONG Tag; BOOLEAN Combined = FALSE; PFN_NUMBER PageCount, RealPageCount; PKPRCB Prcb = KeGetCurrentPrcb(); PGENERAL_LOOKASIDE LookasideList; PEPROCESS Process; // // Check if any of the debug flags are enabled // if (ExpPoolFlags & (POOL_FLAG_CHECK_TIMERS | POOL_FLAG_CHECK_WORKERS | POOL_FLAG_CHECK_RESOURCES | POOL_FLAG_VERIFIER | POOL_FLAG_CHECK_DEADLOCK | POOL_FLAG_SPECIAL_POOL)) { // // Check if special pool is enabled // if (ExpPoolFlags & POOL_FLAG_SPECIAL_POOL) { // // Check if it was allocated from a special pool // if (MmIsSpecialPoolAddress(P)) { // // Was deadlock verification also enabled? We can do some extra // checks at this point // if (ExpPoolFlags & POOL_FLAG_CHECK_DEADLOCK) { DPRINT1("Verifier not yet supported\n"); } // // It is, so handle it via special pool free routine // MmFreeSpecialPool(P); return; } } // // For non-big page allocations, we'll do a bunch of checks in here // if (PAGE_ALIGN(P) != P) { // // Get the entry for this pool allocation // The pointer math here may look wrong or confusing, but it is quite right // Entry = P; Entry--; // // Get the pool type // PoolType = (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK; // // FIXME: Many other debugging checks go here // ExpCheckPoolIrqlLevel(PoolType, 0, P); } } // // Check if this is a big page allocation // if (PAGE_ALIGN(P) == P) { // // We need to find the tag for it, so first we need to find out what // kind of allocation this was (paged or nonpaged), then we can go // ahead and try finding the tag for it. Remember to get rid of the // PROTECTED_POOL tag if it's found. // // Note that if at insertion time, we failed to add the tag for a big // pool allocation, we used a special tag called 'BIG' to identify the // allocation, and we may get this tag back. In this scenario, we must // manually get the size of the allocation by actually counting through // the PFN database. // PoolType = MmDeterminePoolType(P); ExpCheckPoolIrqlLevel(PoolType, 0, P); Tag = ExpFindAndRemoveTagBigPages(P, &PageCount, PoolType); if (!Tag) { DPRINT1("We do not know the size of this allocation. This is not yet supported\n"); ASSERT(Tag == ' GIB'); PageCount = 1; // We are going to lie! This might screw up accounting? } else if (Tag & PROTECTED_POOL) { Tag &= ~PROTECTED_POOL; } // // Check block tag // if (TagToFree && TagToFree != Tag) { DPRINT1("Freeing pool - invalid tag specified: %.4s != %.4s\n", (char*)&TagToFree, (char*)&Tag); #if DBG /* Do not bugcheck in case this is a big allocation for which we didn't manage to insert the tag */ if (Tag != ' GIB') KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Tag, TagToFree); #endif } // // We have our tag and our page count, so we can go ahead and remove this // tracker now // ExpRemovePoolTracker(Tag, PageCount << PAGE_SHIFT, PoolType); // // Check if any of the debug flags are enabled // if (ExpPoolFlags & (POOL_FLAG_CHECK_TIMERS | POOL_FLAG_CHECK_WORKERS | POOL_FLAG_CHECK_RESOURCES | POOL_FLAG_CHECK_DEADLOCK)) { // // Was deadlock verification also enabled? We can do some extra // checks at this point // if (ExpPoolFlags & POOL_FLAG_CHECK_DEADLOCK) { DPRINT1("Verifier not yet supported\n"); } // // FIXME: Many debugging checks go here // } // // Update counters // PoolDesc = PoolVector[PoolType]; InterlockedIncrement((PLONG)&PoolDesc->RunningDeAllocs); InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, -(LONG_PTR)(PageCount << PAGE_SHIFT)); // // Do the real free now and update the last counter with the big page count // RealPageCount = MiFreePoolPages(P); ASSERT(RealPageCount == PageCount); InterlockedExchangeAdd((PLONG)&PoolDesc->TotalBigPages, -(LONG)RealPageCount); return; } // // Get the entry for this pool allocation // The pointer math here may look wrong or confusing, but it is quite right // Entry = P; Entry--; ASSERT((ULONG_PTR)Entry % POOL_BLOCK_SIZE == 0); // // Get the size of the entry, and it's pool type, then load the descriptor // for this pool type // BlockSize = Entry->BlockSize; PoolType = (Entry->PoolType - 1) & BASE_POOL_TYPE_MASK; PoolDesc = PoolVector[PoolType]; // // Make sure that the IRQL makes sense // ExpCheckPoolIrqlLevel(PoolType, 0, P); // // Get the pool tag and get rid of the PROTECTED_POOL flag // Tag = Entry->PoolTag; if (Tag & PROTECTED_POOL) Tag &= ~PROTECTED_POOL; // // Check block tag // if (TagToFree && TagToFree != Tag) { DPRINT1("Freeing pool - invalid tag specified: %.4s != %.4s\n", (char*)&TagToFree, (char*)&Tag); #if DBG KeBugCheckEx(BAD_POOL_CALLER, 0x0A, (ULONG_PTR)P, Tag, TagToFree); #endif } // // Track the removal of this allocation // ExpRemovePoolTracker(Tag, BlockSize * POOL_BLOCK_SIZE, Entry->PoolType - 1); // // Release pool quota, if any // if ((Entry->PoolType - 1) & QUOTA_POOL_MASK) { Process = ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1]; if (Process) { if (Process->Pcb.Header.Type != ProcessObject) { DPRINT1("Object %p is not a process. Type %u, pool type 0x%x, block size %u\n", Process, Process->Pcb.Header.Type, Entry->PoolType, BlockSize); KeBugCheckEx(BAD_POOL_CALLER, POOL_BILLED_PROCESS_INVALID, (ULONG_PTR)P, Tag, (ULONG_PTR)Process); } PsReturnPoolQuota(Process, PoolType, BlockSize * POOL_BLOCK_SIZE); ObDereferenceObject(Process); } } // // Is this allocation small enough to have come from a lookaside list? // if (BlockSize <= NUMBER_POOL_LOOKASIDE_LISTS) { // // Try pushing it into the per-CPU lookaside list // LookasideList = (PoolType == PagedPool) ? Prcb->PPPagedLookasideList[BlockSize - 1].P : Prcb->PPNPagedLookasideList[BlockSize - 1].P; LookasideList->TotalFrees++; if (ExQueryDepthSList(&LookasideList->ListHead) < LookasideList->Depth) { LookasideList->FreeHits++; InterlockedPushEntrySList(&LookasideList->ListHead, P); return; } // // We failed, try to push it into the global lookaside list // LookasideList = (PoolType == PagedPool) ? Prcb->PPPagedLookasideList[BlockSize - 1].L : Prcb->PPNPagedLookasideList[BlockSize - 1].L; LookasideList->TotalFrees++; if (ExQueryDepthSList(&LookasideList->ListHead) < LookasideList->Depth) { LookasideList->FreeHits++; InterlockedPushEntrySList(&LookasideList->ListHead, P); return; } } // // Get the pointer to the next entry // NextEntry = POOL_BLOCK(Entry, BlockSize); // // Update performance counters // InterlockedIncrement((PLONG)&PoolDesc->RunningDeAllocs); InterlockedExchangeAddSizeT(&PoolDesc->TotalBytes, -BlockSize * POOL_BLOCK_SIZE); // // Acquire the pool lock // OldIrql = ExLockPool(PoolDesc); // // Check if the next allocation is at the end of the page // ExpCheckPoolBlocks(Entry); if (PAGE_ALIGN(NextEntry) != NextEntry) { // // We may be able to combine the block if it's free // if (NextEntry->PoolType == 0) { // // The next block is free, so we'll do a combine // Combined = TRUE; // // Make sure there's actual data in the block -- anything smaller // than this means we only have the header, so there's no linked list // for us to remove // if ((NextEntry->BlockSize != 1)) { // // The block is at least big enough to have a linked list, so go // ahead and remove it // ExpCheckPoolLinks(POOL_FREE_BLOCK(NextEntry)); ExpRemovePoolEntryList(POOL_FREE_BLOCK(NextEntry)); ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry))->Flink)); ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry))->Blink)); } // // Our entry is now combined with the next entry // Entry->BlockSize = Entry->BlockSize + NextEntry->BlockSize; } } // // Now check if there was a previous entry on the same page as us // if (Entry->PreviousSize) { // // Great, grab that entry and check if it's free // NextEntry = POOL_PREV_BLOCK(Entry); if (NextEntry->PoolType == 0) { // // It is, so we can do a combine // Combined = TRUE; // // Make sure there's actual data in the block -- anything smaller // than this means we only have the header so there's no linked list // for us to remove // if ((NextEntry->BlockSize != 1)) { // // The block is at least big enough to have a linked list, so go // ahead and remove it // ExpCheckPoolLinks(POOL_FREE_BLOCK(NextEntry)); ExpRemovePoolEntryList(POOL_FREE_BLOCK(NextEntry)); ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry))->Flink)); ExpCheckPoolLinks(ExpDecodePoolLink((POOL_FREE_BLOCK(NextEntry))->Blink)); } // // Combine our original block (which might've already been combined // with the next block), into the previous block // NextEntry->BlockSize = NextEntry->BlockSize + Entry->BlockSize; // // And now we'll work with the previous block instead // Entry = NextEntry; } } // // By now, it may have been possible for our combined blocks to actually // have made up a full page (if there were only 2-3 allocations on the // page, they could've all been combined). // if ((PAGE_ALIGN(Entry) == Entry) && (PAGE_ALIGN(POOL_NEXT_BLOCK(Entry)) == POOL_NEXT_BLOCK(Entry))) { // // In this case, release the pool lock, update the performance counter, // and free the page // ExUnlockPool(PoolDesc, OldIrql); InterlockedExchangeAdd((PLONG)&PoolDesc->TotalPages, -1); MiFreePoolPages(Entry); return; } // // Otherwise, we now have a free block (or a combination of 2 or 3) // Entry->PoolType = 0; BlockSize = Entry->BlockSize; ASSERT(BlockSize != 1); // // Check if we actually did combine it with anyone // if (Combined) { // // Get the first combined block (either our original to begin with, or // the one after the original, depending if we combined with the previous) // NextEntry = POOL_NEXT_BLOCK(Entry); // // As long as the next block isn't on a page boundary, have it point // back to us // if (PAGE_ALIGN(NextEntry) != NextEntry) NextEntry->PreviousSize = BlockSize; } // // Insert this new free block, and release the pool lock // ExpInsertPoolHeadList(&PoolDesc->ListHeads[BlockSize - 1], POOL_FREE_BLOCK(Entry)); ExpCheckPoolLinks(POOL_FREE_BLOCK(Entry)); ExUnlockPool(PoolDesc, OldIrql); } /* * @implemented */ VOID NTAPI ExFreePool(PVOID P) { // // Just free without checking for the tag // ExFreePoolWithTag(P, 0); } /* * @unimplemented */ SIZE_T NTAPI ExQueryPoolBlockSize(IN PVOID PoolBlock, OUT PBOOLEAN QuotaCharged) { // // Not implemented // UNIMPLEMENTED; return FALSE; } /* * @implemented */ PVOID NTAPI ExAllocatePoolWithQuota(IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes) { // // Allocate the pool // return ExAllocatePoolWithQuotaTag(PoolType, NumberOfBytes, TAG_NONE); } /* * @implemented */ PVOID NTAPI ExAllocatePoolWithTagPriority(IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag, IN EX_POOL_PRIORITY Priority) { PVOID Buffer; // // Allocate the pool // Buffer = ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag); if (Buffer == NULL) { UNIMPLEMENTED; } return Buffer; } /* * @implemented */ PVOID NTAPI ExAllocatePoolWithQuotaTag(IN POOL_TYPE PoolType, IN SIZE_T NumberOfBytes, IN ULONG Tag) { BOOLEAN Raise = TRUE; PVOID Buffer; PPOOL_HEADER Entry; NTSTATUS Status; PEPROCESS Process = PsGetCurrentProcess(); // // Check if we should fail instead of raising an exception // if (PoolType & POOL_QUOTA_FAIL_INSTEAD_OF_RAISE) { Raise = FALSE; PoolType &= ~POOL_QUOTA_FAIL_INSTEAD_OF_RAISE; } // // Inject the pool quota mask // PoolType += QUOTA_POOL_MASK; // // Check if we have enough space to add the quota owner process, as long as // this isn't the system process, which never gets charged quota // ASSERT(NumberOfBytes != 0); if ((NumberOfBytes <= (PAGE_SIZE - POOL_BLOCK_SIZE - sizeof(PVOID))) && (Process != PsInitialSystemProcess)) { // // Add space for our EPROCESS pointer // NumberOfBytes += sizeof(PEPROCESS); } else { // // We won't be able to store the pointer, so don't use quota for this // PoolType -= QUOTA_POOL_MASK; } // // Allocate the pool buffer now // Buffer = ExAllocatePoolWithTag(PoolType, NumberOfBytes, Tag); // // If the buffer is page-aligned, this is a large page allocation and we // won't touch it // if (PAGE_ALIGN(Buffer) != Buffer) { // // Also if special pool is enabled, and this was allocated from there, // we won't touch it either // if ((ExpPoolFlags & POOL_FLAG_SPECIAL_POOL) && (MmIsSpecialPoolAddress(Buffer))) { return Buffer; } // // If it wasn't actually allocated with quota charges, ignore it too // if (!(PoolType & QUOTA_POOL_MASK)) return Buffer; // // If this is the system process, we don't charge quota, so ignore // if (Process == PsInitialSystemProcess) return Buffer; // // Actually go and charge quota for the process now // Entry = POOL_ENTRY(Buffer); Status = PsChargeProcessPoolQuota(Process, PoolType & BASE_POOL_TYPE_MASK, Entry->BlockSize * POOL_BLOCK_SIZE); if (!NT_SUCCESS(Status)) { // // Quota failed, back out the allocation, clear the owner, and fail // ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = NULL; ExFreePoolWithTag(Buffer, Tag); if (Raise) RtlRaiseStatus(Status); return NULL; } // // Quota worked, write the owner and then reference it before returning // ((PVOID *)POOL_NEXT_BLOCK(Entry))[-1] = Process; ObReferenceObject(Process); } else if (!(Buffer) && (Raise)) { // // The allocation failed, raise an error if we are in raise mode // RtlRaiseStatus(STATUS_INSUFFICIENT_RESOURCES); } // // Return the allocated buffer // return Buffer; } /* EOF */