From 14f8621042a0e9b05a356da7fa18fafe6a4e9ebd Mon Sep 17 00:00:00 2001 From: Sir Richard Date: Wed, 29 Sep 2010 01:10:28 +0000 Subject: [PATCH] [NTOS]: Add MiRemoveZeroPageSafe helper function, when a zero page is required, but the inline zeroing of MiRemoveZeroPage is not. This function will only try grabbing a zero page if one exists, otherwise a free page will be grabbed and zeroed with custom code of the caller's choosing. [NTOS]: Add concept of process color and system color. Compute correct color to use whenever requesting a page. [NTOS]: Uncondtionally enable the color code when inserting/removing pages. For now, when requesting a page, colors are still ignored, and the global PFN lists are scanned instead. If there are no regressions, we are one patch away from that. svn path=/trunk/; revision=48927 --- reactos/ntoskrnl/mm/ARM3/miarm.h | 27 ++++++++++++-- reactos/ntoskrnl/mm/ARM3/pagfault.c | 56 +++++++++++++++++++++-------- reactos/ntoskrnl/mm/ARM3/pfnlist.c | 47 +++++++++++------------- reactos/ntoskrnl/mm/ARM3/pool.c | 4 +-- reactos/ntoskrnl/mm/ARM3/procsup.c | 55 +++++++++++++++++++--------- reactos/ntoskrnl/mm/ARM3/zeropage.c | 5 ++- reactos/ntoskrnl/mm/freelist.c | 4 +-- 7 files changed, 133 insertions(+), 65 deletions(-) diff --git a/reactos/ntoskrnl/mm/ARM3/miarm.h b/reactos/ntoskrnl/mm/ARM3/miarm.h index 479af5bfefe..8385ea3bf7a 100644 --- a/reactos/ntoskrnl/mm/ARM3/miarm.h +++ b/reactos/ntoskrnl/mm/ARM3/miarm.h @@ -218,6 +218,13 @@ extern const ULONG MmProtectToPteMask[32]; // #define MM_NOIRQL (KIRQL)0xFFFFFFFF +// +// Returns the color of a page +// +#define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask) +#define MI_GET_NEXT_COLOR(x) (MI_GET_PAGE_COLOR(++MmSystemPageColor)) +#define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor)) + // // FIXFIX: These should go in ex.h after the pool merge // @@ -455,9 +462,8 @@ extern PMMPTE MmSharedUserDataPte; extern LIST_ENTRY MmProcessList; extern BOOLEAN MmZeroingPageThreadActive; extern KEVENT MmZeroingPageEvent; - -#define MI_PFN_TO_PFNENTRY(x) (&MmPfnDatabase[1][x]) -#define MI_PFNENTRY_TO_PFN(x) (x - MmPfnDatabase[1]) +extern ULONG MmSystemPageColor; +extern ULONG MmProcessColorSeed; // // Figures out the hardware bits for a PTE @@ -1093,4 +1099,19 @@ MiGetNextNode( IN PMMADDRESS_NODE Node ); +// +// MiRemoveZeroPage will use inline code to zero out the page manually if only +// free pages are available. In some scenarios, we don't/can't run that piece of +// code and would rather only have a real zero page. If we can't have a zero page, +// then we'd like to have our own code to grab a free page and zero it out, by +// using MiRemoveAnyPage. This macro implements this. +// +PFN_NUMBER +FORCEINLINE +MiRemoveZeroPageSafe(IN ULONG Color) +{ + if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color); + return 0; +} + /* EOF */ diff --git a/reactos/ntoskrnl/mm/ARM3/pagfault.c b/reactos/ntoskrnl/mm/ARM3/pagfault.c index c391a0ad389..81d81d503a8 100644 --- a/reactos/ntoskrnl/mm/ARM3/pagfault.c +++ b/reactos/ntoskrnl/mm/ARM3/pagfault.c @@ -179,9 +179,10 @@ MiResolveDemandZeroFault(IN PVOID Address, IN PEPROCESS Process, IN KIRQL OldIrql) { - PFN_NUMBER PageFrameNumber; + PFN_NUMBER PageFrameNumber = 0; MMPTE TempPte; BOOLEAN NeedZero = FALSE; + ULONG Color; DPRINT("ARM3 Demand Zero Page Fault Handler for address: %p in process: %p\n", Address, Process); @@ -196,9 +197,17 @@ MiResolveDemandZeroFault(IN PVOID Address, /* No forking yet */ ASSERT(Process->ForkInProgress == NULL); + /* Get process color */ + Color = MI_GET_NEXT_PROCESS_COLOR(Process); + /* We'll need a zero page */ NeedZero = TRUE; } + else + { + /* Get the next system page color */ + Color = MI_GET_NEXT_COLOR(); + } // // Lock the PFN database @@ -206,9 +215,21 @@ MiResolveDemandZeroFault(IN PVOID Address, OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); ASSERT(PointerPte->u.Hard.Valid == 0); - /* Get a page */ - PageFrameNumber = MiRemoveAnyPage(0); - DPRINT("New pool page: %lx\n", PageFrameNumber); + /* Do we need a zero page? */ + if (NeedZero) + { + /* Try to get one, if we couldn't grab a free page and zero it */ + PageFrameNumber = MiRemoveZeroPageSafe(Color); + if (PageFrameNumber) NeedZero = FALSE; + } + + /* Did we get a page? */ + if (!PageFrameNumber) + { + /* We either failed to find a zero page, or this is a system request */ + PageFrameNumber = MiRemoveAnyPage(Color); + DPRINT("New pool page: %lx\n", PageFrameNumber); + } /* Initialize it */ MiInitializePfn(PageFrameNumber, PointerPte, TRUE); @@ -463,6 +484,7 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction, ULONG ProtectionCode; PMMVAD Vad; PFN_NUMBER PageFrameIndex; + ULONG Color; DPRINT("ARM3 FAULT AT: %p\n", Address); // @@ -756,19 +778,25 @@ MmArmAccessFault(IN BOOLEAN StoreInstruction, /* Lock the PFN database since we're going to grab a page */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Try to get a zero page */ + Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess); + PageFrameIndex = MiRemoveZeroPageSafe(Color); + if (!PageFrameIndex) + { + /* Grab a page out of there. Later we should grab a colored zero page */ + PageFrameIndex = MiRemoveAnyPage(Color); + ASSERT(PageFrameIndex); - /* Grab a page out of there. Later we should grab a colored zero page */ - PageFrameIndex = MiRemoveAnyPage(0); - ASSERT(PageFrameIndex); + /* Release the lock since we need to do some zeroing */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - /* Release the lock since we need to do some zeroing */ - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + /* Zero out the page, since it's for user-mode */ + MiZeroPfn(PageFrameIndex); - /* Zero out the page, since it's for user-mode */ - MiZeroPfn(PageFrameIndex); - - /* Grab the lock again so we can initialize the PFN entry */ - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + /* Grab the lock again so we can initialize the PFN entry */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + } /* Initialize the PFN entry now */ MiInitializePfn(PageFrameIndex, PointerPte, 1); diff --git a/reactos/ntoskrnl/mm/ARM3/pfnlist.c b/reactos/ntoskrnl/mm/ARM3/pfnlist.c index 9d4601392c0..e966ad1677e 100644 --- a/reactos/ntoskrnl/mm/ARM3/pfnlist.c +++ b/reactos/ntoskrnl/mm/ARM3/pfnlist.c @@ -30,12 +30,11 @@ do { \ #define ASSERT_LIST_INVARIANT(x) #endif -#define ARM3_COLORS 1 - /* GLOBALS ********************************************************************/ BOOLEAN MmDynamicPfn; BOOLEAN MmMirroring; +ULONG MmSystemPageColor; MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD}; MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD}; @@ -80,11 +79,9 @@ MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry) PFN_NUMBER OldFlink, OldBlink; PMMPFNLIST ListHead; MMLISTS ListName; -#ifdef ARM3_COLORS ULONG Color; PMMCOLOR_TABLES ColorTable; PMMPFN Pfn1; -#endif /* Make sure the PFN lock is held */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); @@ -131,7 +128,7 @@ MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry) /* Set the list head's backlink instead */ ListHead->Flink = OldFlink; } -#ifdef ARM3_COLORS + /* Get the page color */ OldBlink = MiGetPfnEntryIndex(Entry); Color = OldBlink & MmSecondaryColorMask; @@ -185,7 +182,7 @@ MiUnlinkFreeOrZeroedPage(IN PMMPFN Entry) /* ReactOS Hack */ Entry->OriginalPte.u.Long = 0; -#endif + /* We are not on a list anymore */ Entry->u1.Flink = Entry->u2.Blink = 0; ASSERT_LIST_INVARIANT(ListHead); @@ -219,9 +216,8 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex, MMLISTS ListName; PFN_NUMBER OldFlink, OldBlink; ULONG OldColor, OldCache; -#ifdef ARM3_COLORS PMMCOLOR_TABLES ColorTable; -#endif + /* Make sure PFN lock is held */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); ASSERT(Color < MmSecondaryColors); @@ -280,7 +276,7 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex, Pfn1->u3.e2.ShortFlags = 0; Pfn1->u3.e1.PageColor = OldColor; Pfn1->u3.e1.CacheAttribute = OldCache; -#ifdef ARM3_COLORS + /* Get the first page on the color list */ ASSERT(Color < MmSecondaryColors); ColorTable = &MmFreePagesByColor[ListName][Color]; @@ -306,7 +302,7 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex, /* ReactOS Hack */ Pfn1->OriginalPte.u.Long = 0; -#endif + /* See if we hit any thresholds */ if (MmAvailablePages == MmHighMemoryThreshold) { @@ -340,9 +336,8 @@ MiRemoveAnyPage(IN ULONG Color) ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); ASSERT(MmAvailablePages != 0); ASSERT(Color < MmSecondaryColors); - +#if 0 /* Check the colored free list */ -#if 0 // Enable when using ARM3 database */ PageIndex = MmFreePagesByColor[FreePageList][Color].Flink; if (PageIndex == LIST_HEAD) { @@ -368,11 +363,10 @@ MiRemoveAnyPage(IN ULONG Color) ASSERT(MmZeroedPageListHead.Total == 0); } } -#if 0 // Enable when using ARM3 database */ +#if 0 } } #endif - /* Remove the page from its list */ PageIndex = MiRemovePageByColor(PageIndex, Color); @@ -403,7 +397,7 @@ MiRemoveZeroPage(IN ULONG Color) ASSERT(Color < MmSecondaryColors); /* Check the colored zero list */ -#if 0 // Enable when using ARM3 database */ +#if 0 PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink; if (PageIndex == LIST_HEAD) { @@ -414,9 +408,10 @@ MiRemoveZeroPage(IN ULONG Color) Color = PageIndex & MmSecondaryColorMask; if (PageIndex == LIST_HEAD) { + /* This means there's no zero pages, we have to look for free ones */ ASSERT(MmZeroedPageListHead.Total == 0); Zero = TRUE; -#if 0 // Enable when using ARM3 database */ +#if 0 /* Check the colored free list */ PageIndex = MmFreePagesByColor[ZeroedPageList][Color].Flink; if (PageIndex == LIST_HEAD) @@ -432,13 +427,14 @@ MiRemoveZeroPage(IN ULONG Color) /* FIXME: Should check the standby list */ ASSERT(MmZeroedPageListHead.Total == 0); } -#if 0 // Enable when using ARM3 database */ +#if 0 } #endif } -#if 0 // Enable when using ARM3 database */ +#if 0 } #endif + /* Sanity checks */ Pfn1 = MiGetPfnEntry(PageIndex); ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) || @@ -468,11 +464,10 @@ MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex) PMMPFNLIST ListHead; PFN_NUMBER LastPage; PMMPFN Pfn1; -#ifdef ARM3_COLORS ULONG Color; PMMPFN Blink; PMMCOLOR_TABLES ColorTable; -#endif + /* Make sure the page index is valid */ ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); ASSERT((PageFrameIndex != 0) && @@ -537,7 +532,7 @@ MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex) /* Otherwise check if we reached the high threshold and signal the event */ KeSetEvent(MiHighMemoryEvent, 0, FALSE); } -#ifdef ARM3_COLORS + /* Get the page color */ Color = PageFrameIndex & MmSecondaryColorMask; @@ -571,7 +566,7 @@ MiInsertPageInFreeList(IN PFN_NUMBER PageFrameIndex) /* And increase the count in the colored list */ ColorTable->Count++; -#endif + /* Notify zero page thread if enough pages are on the free list now */ if ((ListHead->Total >= 8) && !(MmZeroingPageThreadActive)) { @@ -590,10 +585,9 @@ MiInsertPageInList(IN PMMPFNLIST ListHead, PFN_NUMBER Flink; PMMPFN Pfn1, Pfn2; MMLISTS ListName; -#ifdef ARM3_COLORS PMMCOLOR_TABLES ColorHead; ULONG Color; -#endif + /* For free pages, use MiInsertPageInFreeList */ ASSERT(ListHead != &MmFreePageListHead); @@ -657,7 +651,7 @@ MiInsertPageInList(IN PMMPFNLIST ListHead, KeSetEvent(MiHighMemoryEvent, 0, FALSE); } -#ifdef ARM3_COLORS + /* Sanity checks */ ASSERT(ListName == ZeroedPageList); ASSERT(Pfn1->u4.InPageError == 0); @@ -695,7 +689,6 @@ MiInsertPageInList(IN PMMPFNLIST ListHead, /* One more paged on the colored list */ ColorHead->Count++; -#endif } VOID @@ -783,7 +776,7 @@ MiAllocatePfn(IN PMMPTE PointerPte, /* Grab a page */ ASSERT_LIST_INVARIANT(&MmFreePageListHead); ASSERT_LIST_INVARIANT(&MmZeroedPageListHead); - PageFrameIndex = MiRemoveAnyPage(0); + PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Write the software PTE */ MI_WRITE_INVALID_PTE(PointerPte, TempPte); diff --git a/reactos/ntoskrnl/mm/ARM3/pool.c b/reactos/ntoskrnl/mm/ARM3/pool.c index 1124dfb03a0..c221a53aebd 100644 --- a/reactos/ntoskrnl/mm/ARM3/pool.c +++ b/reactos/ntoskrnl/mm/ARM3/pool.c @@ -473,7 +473,7 @@ MiAllocatePoolPages(IN POOL_TYPE PoolType, ASSERT(PointerPte->u.Hard.Valid == 0); /* Request a page */ - PageFrameNumber = MiRemoveAnyPage(0); + PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); TempPte.u.Hard.PageFrameNumber = PageFrameNumber; #if (_MI_PAGING_LEVELS >= 3) @@ -768,7 +768,7 @@ MiAllocatePoolPages(IN POOL_TYPE PoolType, do { /* Allocate a page */ - PageFrameNumber = MiRemoveAnyPage(0); + PageFrameNumber = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); /* Get the PFN entry for it and fill it out */ Pfn1 = MiGetPfnEntry(PageFrameNumber); diff --git a/reactos/ntoskrnl/mm/ARM3/procsup.c b/reactos/ntoskrnl/mm/ARM3/procsup.c index 8def3be2f2c..e0b840df203 100644 --- a/reactos/ntoskrnl/mm/ARM3/procsup.c +++ b/reactos/ntoskrnl/mm/ARM3/procsup.c @@ -16,7 +16,9 @@ #define MODULE_INVOLVED_IN_ARM3 #include "../ARM3/miarm.h" -extern MM_SYSTEMSIZE MmSystemSize; +/* GLOBALS ********************************************************************/ + +ULONG MmProcessColorSeed = 0x12345678; /* PRIVATE FUNCTIONS **********************************************************/ @@ -357,7 +359,7 @@ MmCreateKernelStack(IN BOOLEAN GuiStack, PointerPte++; /* Get a page and write the current invalid PTE */ - PageFrameIndex = MiRemoveAnyPage(0); + PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); MI_WRITE_INVALID_PTE(PointerPte, InvalidPte); /* Initialize the PFN entry for this page */ @@ -444,7 +446,7 @@ MmGrowKernelStackEx(IN PVOID StackPointer, while (LimitPte >= NewLimitPte) { /* Get a page and write the current invalid PTE */ - PageFrameIndex = MiRemoveAnyPage(0); + PageFrameIndex = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); MI_WRITE_INVALID_PTE(LimitPte, InvalidPte); /* Initialize the PFN entry for this page */ @@ -1058,9 +1060,10 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, MMPTE TempPte, PdePte; ULONG PdeOffset; PMMPTE SystemTable; + ULONG Color; - /* No page colors yet */ - Process->NextPageColor = 0; + /* Choose a process color */ + Process->NextPageColor = RtlRandom(&MmProcessColorSeed); /* Setup the hyperspace lock */ KeInitializeSpinLock(&Process->HyperSpaceLock); @@ -1068,16 +1071,37 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, /* Lock PFN database */ OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - /* Get a page for the PDE */ - PdeIndex = MiRemoveAnyPage(0); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - MiZeroPhysicalPage(PdeIndex); - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - - /* Get a page for hyperspace */ - HyperIndex = MiRemoveAnyPage(0); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - MiZeroPhysicalPage(HyperIndex); + /* Get a zero page for the PDE, if possible */ + Color = MI_GET_NEXT_PROCESS_COLOR(Process); + PdeIndex = MiRemoveZeroPageSafe(Color); + if (!PdeIndex) + { + /* No zero pages, grab a free one */ + PdeIndex = MiRemoveAnyPage(Color); + + /* Zero it outside the PFN lock */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + MiZeroPhysicalPage(PdeIndex); + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + } + + /* Get a zero page for hyperspace, if possible */ + Color = MI_GET_NEXT_PROCESS_COLOR(Process); + HyperIndex = MiRemoveZeroPageSafe(Color); + if (!HyperIndex) + { + /* No zero pages, grab a free one */ + HyperIndex = MiRemoveAnyPage(Color); + + /* Zero it outside the PFN lock */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + MiZeroPhysicalPage(HyperIndex); + } + else + { + /* Release the PFN lock */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + } /* Switch to phase 1 initialization */ ASSERT(Process->AddressSpaceInitialized == 0); @@ -1112,7 +1136,6 @@ MmCreateProcessAddressSpace(IN ULONG MinWs, /* Copy all the kernel mappings */ PdeOffset = MiGetPdeOffset(MmSystemRangeStart); - RtlCopyMemory(&SystemTable[PdeOffset], MiAddressToPde(MmSystemRangeStart), PAGE_SIZE - PdeOffset * sizeof(MMPTE)); diff --git a/reactos/ntoskrnl/mm/ARM3/zeropage.c b/reactos/ntoskrnl/mm/ARM3/zeropage.c index 320fc95eace..4735cfbadf4 100644 --- a/reactos/ntoskrnl/mm/ARM3/zeropage.c +++ b/reactos/ntoskrnl/mm/ARM3/zeropage.c @@ -69,8 +69,11 @@ MmZeroPageThread(VOID) } PageIndex = MmFreePageListHead.Flink; + ASSERT(PageIndex != LIST_HEAD); Pfn1 = MiGetPfnEntry(PageIndex); - FreePage = MiRemoveAnyPage(0); // FIXME: Use real color + FreePage = MiRemoveAnyPage(MI_GET_PAGE_COLOR(PageIndex)); + + /* The first global free page should also be the first on its own list */ if (FreePage != PageIndex) { KeBugCheckEx(PFN_LIST_CORRUPT, diff --git a/reactos/ntoskrnl/mm/freelist.c b/reactos/ntoskrnl/mm/freelist.c index 268a6ed8a79..a4578e2ab80 100644 --- a/reactos/ntoskrnl/mm/freelist.c +++ b/reactos/ntoskrnl/mm/freelist.c @@ -604,11 +604,11 @@ MmAllocPage(ULONG Type) if (Type != MC_SYSTEM) { - PfnOffset = MiRemoveZeroPage(0); + PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR()); } else { - PfnOffset = MiRemoveAnyPage(0); + PfnOffset = MiRemoveAnyPage(MI_GET_NEXT_COLOR()); } if (!PfnOffset)