Intravision/reactos
1
0
Fork 0
mirror of https://github.com/reactos/reactos.git synced 2024-06-26 07:51:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

[NTOS:MM] Make the page LRU list a real LRU list.

Browse source 
Also, implement flushing mapped sections to disk on shutdown.
This commit is contained in:
Jérôme Gardou 2021-01-27 10:29:07 +01:00
parent bde2ee571a
commit 8a8b4db447
6 changed files with 217 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
ntoskrnl/cache/section/newmm.h vendored
View file

@ -3,22 +3,6 @@
#include <internal/arch/mm.h>
/* TYPES *********************************************************************/
#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT))
#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001)
#define MM_IS_WAIT_PTE(E) \
(IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY)
#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT))
#define SWAPENTRY_FROM_SSE(E) ((E) >> 1)
#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1)
#define DIRTY_SSE(E) ((E) | 2)
#define CLEAN_SSE(E) ((E) & ~2)
#define IS_DIRTY_SSE(E) ((E) & 2)
#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000)
#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2)
#define MAX_SHARE_COUNT 0x3FF
#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2)))
#define MM_SEGMENT_FINALIZE (0x40000000)
#define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff)
@ -123,25 +107,6 @@ VOID
NTAPI
MiInitializeSectionPageTable(PMM_SECTION_SEGMENT Segment);
NTSTATUS
NTAPI
_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
PLARGE_INTEGER Offset,
ULONG_PTR Entry,
const char *file,
int line);
ULONG_PTR
NTAPI
_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
PLARGE_INTEGER Offset,
const char *file,
int line);
#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__)
#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__)
typedef VOID (NTAPI *FREE_SECTION_PAGE_FUN)(
PMM_SECTION_SEGMENT Segment,
PLARGE_INTEGER Offset);
@ -151,12 +116,6 @@ NTAPI
MmFreePageTablesSectionSegment(PMM_SECTION_SEGMENT Segment,
FREE_SECTION_PAGE_FUN FreePage);
/* Yields a lock */
PMM_SECTION_SEGMENT
NTAPI
MmGetSectionAssociation(PFN_NUMBER Page,
PLARGE_INTEGER Offset);
NTSTATUS
NTAPI
MmSetSectionAssociation(PFN_NUMBER Page,
@ -267,22 +226,6 @@ MmPageOutDeleteMapping(PVOID Context,
PEPROCESS Process,
PVOID Address);
VOID
NTAPI
_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment,
const char *file,
int line);
#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__)
VOID
NTAPI
_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment,
const char *file,
int line);
#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__)
VOID
MmFreeCacheSectionPage(PVOID Context,
MEMORY_AREA* MemoryArea,

69
ntoskrnl/include/internal/mm.h
View file

@ -357,6 +357,8 @@ typedef struct _MMPFN
// HACK until WS lists are supported
MMWSLE Wsle;
struct _MMPFN* NextLRU;
struct _MMPFN* PreviousLRU;
} MMPFN, *PMMPFN;
extern PMMPFN MmPfnDatabase;
@ -877,6 +879,11 @@ NTSTATUS
NTAPI
MmPageOutPhysicalAddress(PFN_NUMBER Page);
PMM_SECTION_SEGMENT
NTAPI
MmGetSectionAssociation(PFN_NUMBER Page,
PLARGE_INTEGER Offset);
/* freelist.c **********************************************************/
FORCEINLINE
@ -950,20 +957,12 @@ MiGetPfnEntryIndex(IN PMMPFN Pfn1)
PFN_NUMBER
NTAPI
MmGetLRUNextUserPage(PFN_NUMBER PreviousPage);
MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast);
PFN_NUMBER
NTAPI
MmGetLRUFirstUserPage(VOID);
VOID
NTAPI
MmInsertLRULastUserPage(PFN_NUMBER Page);
VOID
NTAPI
MmRemoveLRUUserPage(PFN_NUMBER Page);
VOID
NTAPI
MmDumpArmPfnDatabase(
@ -1232,6 +1231,37 @@ MmFindRegion(
/* section.c *****************************************************************/
#define PFN_FROM_SSE(E) ((PFN_NUMBER)((E) >> PAGE_SHIFT))
#define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001)
#define MM_IS_WAIT_PTE(E) \
(IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY)
#define MAKE_PFN_SSE(P) ((ULONG_PTR)((P) << PAGE_SHIFT))
#define SWAPENTRY_FROM_SSE(E) ((E) >> 1)
#define MAKE_SWAP_SSE(S) (((ULONG_PTR)(S) << 1) | 0x1)
#define DIRTY_SSE(E) ((E) | 2)
#define CLEAN_SSE(E) ((E) & ~2)
#define IS_DIRTY_SSE(E) ((E) & 2)
#define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000)
#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2)
#define MAX_SHARE_COUNT 0x3FF
#define MAKE_SSE(P, C) ((ULONG_PTR)((P) | ((C) << 2)))
VOID
NTAPI
_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment,
const char *file,
int line);
#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__)
VOID
NTAPI
_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment,
const char *file,
int line);
#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__)
VOID
NTAPI
MmGetImageInformation(
@ -1372,6 +1402,27 @@ MmExtendSection(
_In_ PVOID Section,
_Inout_ PLARGE_INTEGER NewSize);
/* sptab.c *******************************************************************/
NTSTATUS
NTAPI
_MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
PLARGE_INTEGER Offset,
ULONG_PTR Entry,
const char *file,
int line);
ULONG_PTR
NTAPI
_MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment,
PLARGE_INTEGER Offset,
const char *file,
int line);
#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__)
#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__)
/* sysldr.c ******************************************************************/
VOID

27
ntoskrnl/mm/balance.c
View file

@ -80,11 +80,7 @@ MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page)
KeBugCheck(MEMORY_MANAGEMENT);
}
if (MmGetReferenceCountPage(Page) == 1)
{
if(Consumer == MC_USER) MmRemoveLRUUserPage(Page);
(void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
}
(void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed);
MmDereferencePage(Page);
@ -142,7 +138,6 @@ NTSTATUS
MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
{
PFN_NUMBER CurrentPage;
PFN_NUMBER NextPage;
NTSTATUS Status;
(*NrFreedPages) = 0;
@ -158,13 +153,14 @@ MmTrimUserMemory(ULONG Target, ULONG Priority, PULONG NrFreedPages)
(*NrFreedPages)++;
}
NextPage = MmGetLRUNextUserPage(CurrentPage);
if (NextPage <= CurrentPage)
{
/* We wrapped around, so we're done */
break;
}
CurrentPage = NextPage;
CurrentPage = MmGetLRUNextUserPage(CurrentPage, TRUE);
}
if (CurrentPage)
{
KIRQL OldIrql = MiAcquirePfnLock();
MmDereferencePage(CurrentPage);
MiReleasePfnLock(OldIrql);
}
return STATUS_SUCCESS;
@ -209,14 +205,13 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
/*
* Allocate always memory for the non paged pool and for the pager thread.
*/
if ((Consumer == MC_SYSTEM) /* || MiIsBalancerThread() */)
if (Consumer == MC_SYSTEM)
{
Page = MmAllocPage(Consumer);
if (Page == 0)
{
KeBugCheck(NO_PAGES_AVAILABLE);
}
if (Consumer == MC_USER) MmInsertLRULastUserPage(Page);
*AllocatedPage = Page;
if (MmAvailablePages < MiMinimumAvailablePages)
MmRebalanceMemoryConsumers();
@ -257,7 +252,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
KeBugCheck(NO_PAGES_AVAILABLE);
}
if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
*AllocatedPage = Page;
if (MmAvailablePages < MiMinimumAvailablePages)
@ -276,7 +270,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait,
{
KeBugCheck(NO_PAGES_AVAILABLE);
}
if(Consumer == MC_USER) MmInsertLRULastUserPage(Page);
*AllocatedPage = Page;
if (MmAvailablePages < MiMinimumAvailablePages)

170
ntoskrnl/mm/freelist.c
View file

@ -35,95 +35,127 @@ SIZE_T MmPagedPoolCommit;
SIZE_T MmPeakCommitment;
SIZE_T MmtotalCommitLimitMaximum;
static RTL_BITMAP MiUserPfnBitMap;
PMMPFN FirstUserLRUPfn;
PMMPFN LastUserLRUPfn;
/* FUNCTIONS *************************************************************/
VOID
NTAPI
MiInitializeUserPfnBitmap(VOID)
{
PVOID Bitmap;
/* Allocate enough buffer for the PFN bitmap and align it on 32-bits */
Bitmap = ExAllocatePoolWithTag(NonPagedPool,
(((MmHighestPhysicalPage + 1) + 31) / 32) * 4,
TAG_MM);
ASSERT(Bitmap);
/* Initialize it and clear all the bits to begin with */
RtlInitializeBitMap(&MiUserPfnBitMap,
Bitmap,
(ULONG)MmHighestPhysicalPage + 1);
RtlClearAllBits(&MiUserPfnBitMap);
}
PFN_NUMBER
NTAPI
MmGetLRUFirstUserPage(VOID)
{
ULONG Position;
PFN_NUMBER Page;
KIRQL OldIrql;
/* Find the first user page */
OldIrql = MiAcquirePfnLock();
Position = RtlFindSetBits(&MiUserPfnBitMap, 1, 0);
MiReleasePfnLock(OldIrql);
if (Position == 0xFFFFFFFF) return 0;
/* Return it */
ASSERT(Position != 0);
ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position));
return Position;
if (FirstUserLRUPfn == NULL)
{
MiReleasePfnLock(OldIrql);
return 0;
}
Page = MiGetPfnEntryIndex(FirstUserLRUPfn);
MmReferencePage(Page);
MiReleasePfnLock(OldIrql);
return Page;
}
static
VOID
NTAPI
MmInsertLRULastUserPage(PFN_NUMBER Pfn)
MmInsertLRULastUserPage(PFN_NUMBER Page)
{
KIRQL OldIrql;
MI_ASSERT_PFN_LOCK_HELD();
/* Set the page as a user page */
ASSERT(Pfn != 0);
ASSERT_IS_ROS_PFN(MiGetPfnEntry(Pfn));
ASSERT(!RtlCheckBit(&MiUserPfnBitMap, (ULONG)Pfn));
OldIrql = MiAcquirePfnLock();
RtlSetBit(&MiUserPfnBitMap, (ULONG)Pfn);
MiReleasePfnLock(OldIrql);
PMMPFN Pfn = MiGetPfnEntry(Page);
if (FirstUserLRUPfn == NULL)
FirstUserLRUPfn = Pfn;
Pfn->PreviousLRU = LastUserLRUPfn;
if (LastUserLRUPfn != NULL)
LastUserLRUPfn->NextLRU = Pfn;
LastUserLRUPfn = Pfn;
}
static
VOID
MmRemoveLRUUserPage(PFN_NUMBER Page)
{
MI_ASSERT_PFN_LOCK_HELD();
/* Unset the page as a user page */
ASSERT(Page != 0);
PMMPFN Pfn = MiGetPfnEntry(Page);
ASSERT_IS_ROS_PFN(Pfn);
if (Pfn->PreviousLRU)
{
ASSERT(Pfn->PreviousLRU->NextLRU == Pfn);
Pfn->PreviousLRU->NextLRU = Pfn->NextLRU;
}
else
{
ASSERT(FirstUserLRUPfn == Pfn);
FirstUserLRUPfn = Pfn->NextLRU;
}
if (Pfn->NextLRU)
{
ASSERT(Pfn->NextLRU->PreviousLRU == Pfn);
Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU;
}
else
{
ASSERT(Pfn == LastUserLRUPfn);
LastUserLRUPfn = Pfn->PreviousLRU;
}
Pfn->PreviousLRU = Pfn->NextLRU = NULL;
}
PFN_NUMBER
NTAPI
MmGetLRUNextUserPage(PFN_NUMBER PreviousPfn)
MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast)
{
ULONG Position;
PFN_NUMBER Page = 0;
KIRQL OldIrql;
/* Find the next user page */
OldIrql = MiAcquirePfnLock();
Position = RtlFindSetBits(&MiUserPfnBitMap, 1, (ULONG)PreviousPfn + 1);
PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage);
PMMPFN NextPfn = PreviousPfn->NextLRU;
/*
* Move this one at the end of the list.
* It may be freed by MmDereferencePage below.
* If it's not, then it means it is still hanging in some process address space.
* This avoids paging-out e.g. ntdll early just because it's mapped first time.
*/
if (MoveToLast)
{
MmRemoveLRUUserPage(PreviousPage);
MmInsertLRULastUserPage(PreviousPage);
}
if (NextPfn)
{
Page = MiGetPfnEntryIndex(NextPfn);
MmReferencePage(Page);
}
MmDereferencePage(PreviousPage);
MiReleasePfnLock(OldIrql);
if (Position == 0xFFFFFFFF) return 0;
/* Return it */
ASSERT(Position != 0);
ASSERT_IS_ROS_PFN(MiGetPfnEntry(Position));
return Position;
}
VOID
NTAPI
MmRemoveLRUUserPage(PFN_NUMBER Page)
{
KIRQL OldIrql;
/* Unset the page as a user page */
ASSERT(Page != 0);
ASSERT_IS_ROS_PFN(MiGetPfnEntry(Page));
ASSERT(RtlCheckBit(&MiUserPfnBitMap, (ULONG)Page));
OldIrql = MiAcquirePfnLock();
RtlClearBit(&MiUserPfnBitMap, (ULONG)Page);
MiReleasePfnLock(OldIrql);
return Page;
}
BOOLEAN
@ -548,6 +580,13 @@ MmDereferencePage(PFN_NUMBER Pfn)
Pfn1->u3.e2.ReferenceCount--;
if (Pfn1->u3.e2.ReferenceCount == 0)
{
/* Apply LRU hack */
if (Pfn1->u4.MustBeCached)
{
MmRemoveLRUUserPage(Pfn);
Pfn1->u4.MustBeCached = 0;
}
/* Mark the page temporarily as valid, we're going to make it free soon */
Pfn1->u3.e1.PageLocation = ActiveAndValid;
@ -590,6 +629,15 @@ MmAllocPage(ULONG Type)
Pfn1->u1.SwapEntry = 0;
Pfn1->RmapListHead = NULL;
Pfn1->NextLRU = NULL;
Pfn1->PreviousLRU = NULL;
if (Type == MC_USER)
{
Pfn1->u4.MustBeCached = 1; /* HACK again */
MmInsertLRULastUserPage(PfnOffset);
}
MiReleasePfnLock(OldIrql);
return PfnOffset;
}

3
ntoskrnl/mm/mminit.c
View file

@ -17,8 +17,6 @@
/* GLOBALS *******************************************************************/
VOID NTAPI MiInitializeUserPfnBitmap(VOID);
BOOLEAN Mm64BitPhysicalAddress = FALSE;
ULONG MmReadClusterSize;
//
@ -235,7 +233,6 @@ MmInitSystem(IN ULONG Phase,
MiDbgDumpAddressSpace();
MmInitGlobalKernelPageDirectory();
MiInitializeUserPfnBitmap();
MmInitializeMemoryConsumer(MC_USER, MmTrimUserMemory);
MmInitializeRmapList();
MmInitSectionImplementation();

38
ntoskrnl/mm/shutdown.c
View file

@ -21,6 +21,44 @@ VOID
MiShutdownSystem(VOID)
{
ULONG i;
PFN_NUMBER Page;
BOOLEAN Dirty;
/* Loop through all the pages owned by the legacy Mm and page them out, if needed. */
/* We do it twice, since flushing can cause the FS to dirtify new pages */
do
{
Dirty = FALSE;
Page = MmGetLRUFirstUserPage();
while (Page)
{
LARGE_INTEGER SegmentOffset;
PMM_SECTION_SEGMENT Segment = MmGetSectionAssociation(Page, &SegmentOffset);
if (Segment)
{
if ((*Segment->Flags) & MM_DATAFILE_SEGMENT)
{
MmLockSectionSegment(Segment);
ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset);
if (!IS_SWAP_FROM_SSE(Entry) && IS_DIRTY_SSE(Entry))
{
Dirty = TRUE;
MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE);
}
MmUnlockSectionSegment(Segment);
}
MmDereferenceSegment(Segment);
}
Page = MmGetLRUNextUserPage(Page, FALSE);
}
} while (Dirty);
/* Loop through all the paging files */
for (i = 0; i < MmNumberOfPagingFiles; i++)