diff --git a/ntoskrnl/cache/section/newmm.h b/ntoskrnl/cache/section/newmm.h index edff99f3d85..3fb731c6ace 100644 --- a/ntoskrnl/cache/section/newmm.h +++ b/ntoskrnl/cache/section/newmm.h @@ -3,22 +3,6 @@ #include /* TYPES *********************************************************************/ - -#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT)) -#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001) -#define MM_IS_WAIT_PTE(E) \ - (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY) -#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT)) -#define SWAPENTRY_FROM_SSE(E) ((E) >> 1) -#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1) -#define DIRTY_SSE(E) ((E) | 2) -#define CLEAN_SSE(E) ((E) & ~2) -#define IS_DIRTY_SSE(E) ((E) & 2) -#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000) -#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2) -#define MAX_SHARE_COUNT 0x3FF -#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2))) - #define MM_SEGMENT_FINALIZE (0x40000000) #define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff) @@ -123,25 +107,6 @@ VOID NTAPI MiInitializeSectionPageTable(PMM_SECTION_SEGMENT Segment); -NTSTATUS -NTAPI -_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - PLARGE_INTEGER Offset, - ULONG_PTR Entry, - const char *file, - int line); - -ULONG_PTR -NTAPI -_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - PLARGE_INTEGER Offset, - const char *file, - int line); - -#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__) - -#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__) - typedef VOID (NTAPI *FREE_SECTION_PAGE_FUN)( PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset); @@ -151,12 +116,6 @@ NTAPI MmFreePageTablesSectionSegment(PMM_SECTION_SEGMENT Segment, FREE_SECTION_PAGE_FUN FreePage); -/* Yields a lock */ -PMM_SECTION_SEGMENT -NTAPI -MmGetSectionAssociation(PFN_NUMBER Page, - PLARGE_INTEGER Offset); - NTSTATUS NTAPI MmSetSectionAssociation(PFN_NUMBER Page, @@ -267,22 +226,6 @@ MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address); -VOID -NTAPI -_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, - const char *file, - int line); - -#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__) - -VOID -NTAPI -_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, - const char *file, - int line); - -#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__) - VOID MmFreeCacheSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 4db21f72199..bd9ea25e0b7 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -357,6 +357,8 @@ typedef struct _MMPFN // HACK until WS lists are supported MMWSLE Wsle; + struct _MMPFN* NextLRU; + struct _MMPFN* PreviousLRU; } MMPFN, *PMMPFN; extern PMMPFN MmPfnDatabase; @@ -877,6 +879,11 @@ NTSTATUS NTAPI MmPageOutPhysicalAddress(PFN_NUMBER Page); +PMM_SECTION_SEGMENT +NTAPI +MmGetSectionAssociation(PFN_NUMBER Page, + PLARGE_INTEGER Offset); + /* freelist.c **********************************************************/ FORCEINLINE @@ -950,20 +957,12 @@ MiGetPfnEntryIndex(IN PMMPFN Pfn1) PFN_NUMBER NTAPI -MmGetLRUNextUserPage(PFN_NUMBER PreviousPage); +MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast); PFN_NUMBER NTAPI MmGetLRUFirstUserPage(VOID); -VOID -NTAPI -MmInsertLRULastUserPage(PFN_NUMBER Page); - -VOID -NTAPI -MmRemoveLRUUserPage(PFN_NUMBER Page); - VOID NTAPI MmDumpArmPfnDatabase( @@ -1232,6 +1231,37 @@ MmFindRegion( /* section.c *****************************************************************/ +#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT)) +#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001) +#define MM_IS_WAIT_PTE(E) \ + (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY) +#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT)) +#define SWAPENTRY_FROM_SSE(E) ((E) >> 1) +#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1) +#define DIRTY_SSE(E) ((E) | 2) +#define CLEAN_SSE(E) ((E) & ~2) +#define IS_DIRTY_SSE(E) ((E) & 2) +#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000) +#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2) +#define MAX_SHARE_COUNT 0x3FF +#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2))) + +VOID +NTAPI +_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, + const char *file, + int line); + +#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__) + +VOID +NTAPI +_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, + const char *file, + int line); + +#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__) + VOID NTAPI MmGetImageInformation( @@ -1372,6 +1402,27 @@ MmExtendSection( _In_ PVOID Section, _Inout_ PLARGE_INTEGER NewSize); +/* sptab.c *******************************************************************/ + +NTSTATUS +NTAPI +_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + ULONG_PTR Entry, + const char *file, + int line); + +ULONG_PTR +NTAPI +_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + const char *file, + int line); + +#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__) + +#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__) + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index 34dddf5dbfa..f9067c58a13 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -80,11 +80,7 @@ MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page) KeBugCheck(MEMORY_MANAGEMENT); } - if (MmGetReferenceCountPage(Page) == 1) - { - if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); - (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - } + (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); MmDereferencePage(Page); @@ -142,7 +138,6 @@ NTSTATUS MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages) { PFN_NUMBER CurrentPage; - PFN_NUMBER NextPage; NTSTATUS Status; (*NrFreedPages) = 0; @@ -158,13 +153,14 @@ MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages) (*NrFreedPages)++; } - NextPage = MmGetLRUNextUserPage(CurrentPage); - if (NextPage <= CurrentPage) - { - /* We wrapped around, so we're done */ - break; - } - CurrentPage = NextPage; + CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE); + } + + if (CurrentPage) + { + KIRQL OldIrql = MiAcquirePfnLock(); + MmDereferencePage(CurrentPage); + MiReleasePfnLock(OldIrql); } return STATUS_SUCCESS; @@ -209,14 +205,13 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, /* * Allocate always memory for the non paged pool and for the pager thread. */ - if ((Consumer == MC_SYSTEM) /* || MiIsBalancerThread() */) + if (Consumer == MC_SYSTEM) { Page = MmAllocPage(Consumer); if (Page == 0) { KeBugCheck(NO_PAGES_AVAILABLE); } - if (Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; if (MmAvailablePages < MiMinimumAvailablePages) MmRebalanceMemoryConsumers(); @@ -257,7 +252,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, KeBugCheck(NO_PAGES_AVAILABLE); } - if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; if (MmAvailablePages < MiMinimumAvailablePages) @@ -276,7 +270,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, { KeBugCheck(NO_PAGES_AVAILABLE); } - if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; if (MmAvailablePages < MiMinimumAvailablePages) diff --git a/ntoskrnl/mm/freelist.c b/ntoskrnl/mm/freelist.c index 5cc22ce38b1..35c6336dcc6 100644 --- a/ntoskrnl/mm/freelist.c +++ b/ntoskrnl/mm/freelist.c @@ -35,95 +35,127 @@ SIZE_T MmPagedPoolCommit; SIZE_T MmPeakCommitment; SIZE_T MmtotalCommitLimitMaximum; -static RTL_BITMAP MiUserPfnBitMap; +PMMPFN FirstUserLRUPfn; +PMMPFN LastUserLRUPfn; /* FUNCTIONS *************************************************************/ -VOID -NTAPI -MiInitializeUserPfnBitmap(VOID) -{ - PVOID Bitmap; - - /* Allocate enough buffer for the PFN bitmap and align it on 32-bits */ - Bitmap = ExAllocatePoolWithTag(NonPagedPool, - (((MmHighestPhysicalPage + 1) + 31) / 32) * 4, - TAG_MM); - ASSERT(Bitmap); - - /* Initialize it and clear all the bits to begin with */ - RtlInitializeBitMap(&MiUserPfnBitMap, - Bitmap, - (ULONG)MmHighestPhysicalPage + 1); - RtlClearAllBits(&MiUserPfnBitMap); -} - PFN_NUMBER NTAPI MmGetLRUFirstUserPage(VOID) { - ULONG Position; + PFN_NUMBER Page; KIRQL OldIrql; /* Find the first user page */ OldIrql = MiAcquirePfnLock(); - Position = RtlFindSetBits(&MiUserPfnBitMap, 1, 0); - MiReleasePfnLock(OldIrql); - if (Position == 0xFFFFFFFF) return 0; - /* Return it */ - ASSERT(Position != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position)); - return Position; + if (FirstUserLRUPfn == NULL) + { + MiReleasePfnLock(OldIrql); + return 0; + } + + Page = MiGetPfnEntryIndex(FirstUserLRUPfn); + MmReferencePage(Page); + + MiReleasePfnLock(OldIrql); + + return Page; } +static VOID -NTAPI -MmInsertLRULastUserPage(PFN_NUMBER Pfn) +MmInsertLRULastUserPage(PFN_NUMBER Page) { - KIRQL OldIrql; + MI_ASSERT_PFN_LOCK_HELD(); - /* Set the page as a user page */ - ASSERT(Pfn != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Pfn)); - ASSERT(!RtlCheckBit(&MiUserPfnBitMap, (ULONG)Pfn)); - OldIrql = MiAcquirePfnLock(); - RtlSetBit(&MiUserPfnBitMap, (ULONG)Pfn); - MiReleasePfnLock(OldIrql); + PMMPFN Pfn = MiGetPfnEntry(Page); + + if (FirstUserLRUPfn == NULL) + FirstUserLRUPfn = Pfn; + + Pfn->PreviousLRU = LastUserLRUPfn; + + if (LastUserLRUPfn != NULL) + LastUserLRUPfn->NextLRU = Pfn; + LastUserLRUPfn = Pfn; +} + +static +VOID +MmRemoveLRUUserPage(PFN_NUMBER Page) +{ + MI_ASSERT_PFN_LOCK_HELD(); + + /* Unset the page as a user page */ + ASSERT(Page != 0); + + PMMPFN Pfn = MiGetPfnEntry(Page); + + ASSERT_IS_ROS_PFN(Pfn); + + if (Pfn->PreviousLRU) + { + ASSERT(Pfn->PreviousLRU->NextLRU == Pfn); + Pfn->PreviousLRU->NextLRU = Pfn->NextLRU; + } + else + { + ASSERT(FirstUserLRUPfn == Pfn); + FirstUserLRUPfn = Pfn->NextLRU; + } + + if (Pfn->NextLRU) + { + ASSERT(Pfn->NextLRU->PreviousLRU == Pfn); + Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU; + } + else + { + ASSERT(Pfn == LastUserLRUPfn); + LastUserLRUPfn = Pfn->PreviousLRU; + } + + Pfn->PreviousLRU = Pfn->NextLRU = NULL; } PFN_NUMBER NTAPI -MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn) +MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast) { - ULONG Position; + PFN_NUMBER Page = 0; KIRQL OldIrql; /* Find the next user page */ OldIrql = MiAcquirePfnLock(); - Position = RtlFindSetBits(&MiUserPfnBitMap, 1, (ULONG)PreviousPfn + 1); + + PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage); + PMMPFN NextPfn = PreviousPfn->NextLRU; + + /* + * Move this one at the end of the list. + * It may be freed by MmDereferencePage below. + * If it's not, then it means it is still hanging in some process address space. + * This avoids paging-out e.g. ntdll early just because it's mapped first time. + */ + if (MoveToLast) + { + MmRemoveLRUUserPage(PreviousPage); + MmInsertLRULastUserPage(PreviousPage); + } + + if (NextPfn) + { + Page = MiGetPfnEntryIndex(NextPfn); + MmReferencePage(Page); + } + + MmDereferencePage(PreviousPage); + MiReleasePfnLock(OldIrql); - if (Position == 0xFFFFFFFF) return 0; - /* Return it */ - ASSERT(Position != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position)); - return Position; -} - -VOID -NTAPI -MmRemoveLRUUserPage(PFN_NUMBER Page) -{ - KIRQL OldIrql; - - /* Unset the page as a user page */ - ASSERT(Page != 0); - ASSERT_IS_ROS_PFN(MiGetPfnEntry(Page)); - ASSERT(RtlCheckBit(&MiUserPfnBitMap, (ULONG)Page)); - OldIrql = MiAcquirePfnLock(); - RtlClearBit(&MiUserPfnBitMap, (ULONG)Page); - MiReleasePfnLock(OldIrql); + return Page; } BOOLEAN @@ -548,6 +580,13 @@ MmDereferencePage(PFN_NUMBER Pfn) Pfn1->u3.e2.ReferenceCount--; if (Pfn1->u3.e2.ReferenceCount == 0) { + /* Apply LRU hack */ + if (Pfn1->u4.MustBeCached) + { + MmRemoveLRUUserPage(Pfn); + Pfn1->u4.MustBeCached = 0; + } + /* Mark the page temporarily as valid, we're going to make it free soon */ Pfn1->u3.e1.PageLocation = ActiveAndValid; @@ -590,6 +629,15 @@ MmAllocPage(ULONG Type) Pfn1->u1.SwapEntry = 0; Pfn1->RmapListHead = NULL; + Pfn1->NextLRU = NULL; + Pfn1->PreviousLRU = NULL; + + if (Type == MC_USER) + { + Pfn1->u4.MustBeCached = 1; /* HACK again */ + MmInsertLRULastUserPage(PfnOffset); + } + MiReleasePfnLock(OldIrql); return PfnOffset; } diff --git a/ntoskrnl/mm/mminit.c b/ntoskrnl/mm/mminit.c index 6c1a5f274d6..3e58ad9bbc8 100644 --- a/ntoskrnl/mm/mminit.c +++ b/ntoskrnl/mm/mminit.c @@ -17,8 +17,6 @@ /* GLOBALS *******************************************************************/ -VOID NTAPI MiInitializeUserPfnBitmap(VOID); - BOOLEAN Mm64BitPhysicalAddress = FALSE; ULONG MmReadClusterSize; // @@ -235,7 +233,6 @@ MmInitSystem(IN ULONG Phase, MiDbgDumpAddressSpace(); MmInitGlobalKernelPageDirectory(); - MiInitializeUserPfnBitmap(); MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory); MmInitializeRmapList(); MmInitSectionImplementation(); diff --git a/ntoskrnl/mm/shutdown.c b/ntoskrnl/mm/shutdown.c index c80cc527a30..89beca88fac 100644 --- a/ntoskrnl/mm/shutdown.c +++ b/ntoskrnl/mm/shutdown.c @@ -21,6 +21,44 @@ VOID MiShutdownSystem(VOID) { ULONG i; + PFN_NUMBER Page; + BOOLEAN Dirty; + + /* Loop through all the pages owned by the legacy Mm and page them out, if needed. */ + /* We do it twice, since flushing can cause the FS to dirtify new pages */ + do + { + Dirty = FALSE; + + Page = MmGetLRUFirstUserPage(); + while (Page) + { + LARGE_INTEGER SegmentOffset; + PMM_SECTION_SEGMENT Segment = MmGetSectionAssociation(Page, &SegmentOffset); + + if (Segment) + { + if ((*Segment->Flags) & MM_DATAFILE_SEGMENT) + { + MmLockSectionSegment(Segment); + + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + if (!IS_SWAP_FROM_SSE(Entry) && IS_DIRTY_SSE(Entry)) + { + Dirty = TRUE; + MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE); + } + + MmUnlockSectionSegment(Segment); + } + + MmDereferenceSegment(Segment); + } + + Page = MmGetLRUNextUserPage(Page, FALSE); + } + } while (Dirty); /* Loop through all the paging files */ for (i = 0; i < MmNumberOfPagingFiles; i++)