From f421bccbcc02eb15e43c9eb102266a70b5af7fbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Mon, 12 Apr 2021 11:36:19 +0200 Subject: [PATCH] [NTOS:MM] First shot for Working Set list support - Initialize - Add private page (no shared page support yet) - Remove pages - Trim Yes, this is C++ in the kernel. --- ntoskrnl/include/internal/mm.h | 6 + ntoskrnl/mm/ARM3/mminit.c | 3 + ntoskrnl/mm/ARM3/procsup.c | 74 +++-- ntoskrnl/mm/ARM3/session.c | 1 - ntoskrnl/mm/ARM3/wslist.cpp | 484 +++++++++++++++++++++++++++++++++ ntoskrnl/ntos.cmake | 1 + sdk/include/ndk/mmtypes.h | 16 ++ 7 files changed, 541 insertions(+), 44 deletions(-) create mode 100644 ntoskrnl/mm/ARM3/wslist.cpp diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index 5eab082b3f2..2f101c97956 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -1645,6 +1645,12 @@ MmCopyVirtualMemory(IN PEPROCESS SourceProcess, IN KPROCESSOR_MODE PreviousMode, OUT PSIZE_T ReturnSize); +/* wslist.cpp ****************************************************************/ +_Requires_exclusive_lock_held_(WorkingSet->WorkingSetMutex) +VOID +NTAPI +MiInitializeWorkingSetList(_Inout_ PMMSUPPORT WorkingSet); + #ifdef __cplusplus } // extern "C" #endif diff --git a/ntoskrnl/mm/ARM3/mminit.c b/ntoskrnl/mm/ARM3/mminit.c index 9b18537f3ac..400f8965dd5 100644 --- a/ntoskrnl/mm/ARM3/mminit.c +++ b/ntoskrnl/mm/ARM3/mminit.c @@ -2139,6 +2139,9 @@ MmArmInitSystem(IN ULONG Phase, /* Initialize the user mode image list */ InitializeListHead(&MmLoadedUserImageList); + /* Initalize the Working set list */ + InitializeListHead(&MmWorkingSetExpansionHead); + /* Initialize critical section timeout value (relative time is negative) */ MmCriticalSectionTimeout.QuadPart = MmCritsectTimeoutSeconds * (-10000000LL); diff --git a/ntoskrnl/mm/ARM3/procsup.c b/ntoskrnl/mm/ARM3/procsup.c index 23981bcadf1..aa9133b1929 100644 --- a/ntoskrnl/mm/ARM3/procsup.c +++ b/ntoskrnl/mm/ARM3/procsup.c @@ -896,40 +896,6 @@ MiInsertSharedUserPageVad(VOID) } #endif -VOID -NTAPI -MiInitializeWorkingSetList(IN PEPROCESS CurrentProcess) -{ - PMMPFN Pfn1; - PMMPTE sysPte; - MMPTE tempPte; - - /* Setup some bogus list data */ - MmWorkingSetList->LastEntry = CurrentProcess->Vm.MinimumWorkingSetSize; - MmWorkingSetList->HashTable = NULL; - MmWorkingSetList->HashTableSize = 0; - MmWorkingSetList->NumberOfImageWaiters = 0; - MmWorkingSetList->Wsle = (PVOID)(ULONG_PTR)0xDEADBABEDEADBABEULL; - MmWorkingSetList->VadBitMapHint = 1; - MmWorkingSetList->HashTableStart = (PVOID)(ULONG_PTR)0xBADAB00BBADAB00BULL; - MmWorkingSetList->HighestPermittedHashAddress = (PVOID)(ULONG_PTR)0xCAFEBABECAFEBABEULL; - MmWorkingSetList->FirstFree = 1; - MmWorkingSetList->FirstDynamic = 2; - MmWorkingSetList->NextSlot = 3; - MmWorkingSetList->LastInitializedWsle = 4; - - /* The rule is that the owner process is always in the FLINK of the PDE's PFN entry */ - Pfn1 = MiGetPfnEntry(CurrentProcess->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT); - ASSERT(Pfn1->u4.PteFrame == MiGetPfnEntryIndex(Pfn1)); - Pfn1->u1.Event = (PKEVENT)CurrentProcess; - - /* Map the process working set in kernel space */ - sysPte = MiReserveSystemPtes(1, SystemPteSpace); - MI_MAKE_HARDWARE_PTE_KERNEL(&tempPte, sysPte, MM_READWRITE, CurrentProcess->WorkingSetPage); - MI_WRITE_VALID_PTE(sysPte, tempPte); - CurrentProcess->Vm.VmWorkingSetList = MiPteToAddress(sysPte); -} - NTSTATUS NTAPI MmInitializeProcessAddressSpace(IN PEPROCESS Process, @@ -944,6 +910,7 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, PMMPTE PointerPte; KIRQL OldIrql; PMMPDE PointerPde; + PMMPFN Pfn; PFN_NUMBER PageFrameNumber; UNICODE_STRING FileName; PWCHAR Source; @@ -980,6 +947,8 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, /* On x64 the PFNs for the initial process are already set up */ if (Process != &KiInitialProcess) { #endif + /* Lock our working set */ + MiLockProcessWorkingSet(Process, PsGetCurrentThread()); /* Lock PFN database */ OldIrql = MiAcquirePfnLock(); @@ -997,7 +966,7 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, MiInitializePfn(PageFrameNumber, PointerPte, TRUE); /* Do the same for hyperspace */ - PointerPde = MiAddressToPde((PVOID)HYPER_SPACE); + PointerPde = MiAddressToPde(HYPER_SPACE); PageFrameNumber = PFN_FROM_PTE(PointerPde); MiInitializePfn(PageFrameNumber, (PMMPTE)PointerPde, TRUE); #if (_MI_PAGING_LEVELS == 2) @@ -1019,18 +988,30 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, ASSERT(Process->Pcb.DirectoryTableBase[1] == PageFrameNumber * PAGE_SIZE); #endif - /* Setup the PFN for the PTE for the working set */ - PointerPte = MiAddressToPte(MI_WORKING_SET_LIST); - MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, MM_READWRITE, 0); - ASSERT(PointerPte->u.Long != 0); + /* Do the same for the Working set list */ + PointerPte = MiAddressToPte(MmWorkingSetList); PageFrameNumber = PFN_FROM_PTE(PointerPte); - MI_WRITE_INVALID_PTE(PointerPte, DemandZeroPte); MiInitializePfn(PageFrameNumber, PointerPte, TRUE); - TempPte.u.Hard.PageFrameNumber = PageFrameNumber; - MI_WRITE_VALID_PTE(PointerPte, TempPte); + + /* This should be in hyper space, but not in the mapping range */ + Process->Vm.VmWorkingSetList = MmWorkingSetList; + ASSERT(((ULONG_PTR)MmWorkingSetList >= MI_MAPPING_RANGE_END) && ((ULONG_PTR)MmWorkingSetList <= HYPER_SPACE_END)); /* Now initialize the working set list */ - MiInitializeWorkingSetList(Process); + MiInitializeWorkingSetList(&Process->Vm); + + /* Map the process working set in kernel space */ + /* FIXME: there should be no need */ + PointerPte = MiReserveSystemPtes(1, SystemPteSpace); + MI_MAKE_HARDWARE_PTE_KERNEL(&TempPte, PointerPte, MM_READWRITE, Process->WorkingSetPage); + MI_WRITE_VALID_PTE(PointerPte, TempPte); + Process->Vm.VmWorkingSetList = MiPteToAddress(PointerPte); + + /* The rule is that the owner process is always in the FLINK of the PDE's PFN entry */ + Pfn = MiGetPfnEntry(Process->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT); + ASSERT(Pfn->u4.PteFrame == MiGetPfnEntryIndex(Pfn)); + ASSERT(Pfn->u1.WsIndex == 0); + Pfn->u1.Event = (PKEVENT)Process; /* Sanity check */ ASSERT(Process->PhysicalVadRoot == NULL); @@ -1038,6 +1019,8 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process, /* Release PFN lock */ MiReleasePfnLock(OldIrql); + /* Release the process working set */ + MiUnlockProcessWorkingSet(Process, PsGetCurrentThread()); #ifdef _M_AMD64 } /* On x64 the PFNs for the initial process are already set up */ #endif @@ -1358,6 +1341,11 @@ MmDeleteProcessAddressSpace2(IN PEPROCESS Process) //ASSERT(Process->CommitCharge == 0); + /* Remove us from the list */ + OldIrql = MiAcquireExpansionLock(); + RemoveEntryList(&Process->Vm.WorkingSetExpansionLinks); + MiReleaseExpansionLock(OldIrql); + /* Acquire the PFN lock */ OldIrql = MiAcquirePfnLock(); diff --git a/ntoskrnl/mm/ARM3/session.c b/ntoskrnl/mm/ARM3/session.c index 898dbbbaebc..b897a491bb0 100644 --- a/ntoskrnl/mm/ARM3/session.c +++ b/ntoskrnl/mm/ARM3/session.c @@ -41,7 +41,6 @@ MiInitializeSessionWsSupport(VOID) { /* Initialize the list heads */ InitializeListHead(&MiSessionWsList); - InitializeListHead(&MmWorkingSetExpansionHead); } BOOLEAN diff --git a/ntoskrnl/mm/ARM3/wslist.cpp b/ntoskrnl/mm/ARM3/wslist.cpp new file mode 100644 index 00000000000..430e2c1a564 --- /dev/null +++ b/ntoskrnl/mm/ARM3/wslist.cpp @@ -0,0 +1,484 @@ +/* + * PROJECT: ReactOS Kernel + * LICENSE: BSD-3-Clause (https://spdx.org/licenses/BSD-3-Clause.html) + * FILE: ntoskrnl/mm/ARM3/wslist.cpp + * PURPOSE: Working set list management + * PROGRAMMERS: Jérôme Gardou + */ + +/* INCLUDES *******************************************************************/ +#include + +#define NDEBUG +#include + +#define MODULE_INVOLVED_IN_ARM3 +#include "miarm.h" + +/* GLOBALS ********************************************************************/ +PMMWSL MmWorkingSetList; +KEVENT MmWorkingSetManagerEvent; + +/* LOCAL FUNCTIONS ************************************************************/ +namespace ntoskrnl +{ + +static MMPTE GetPteTemplateForWsList(PMMWSL WsList) +{ + return (WsList == MmSystemCacheWorkingSetList) ? ValidKernelPte : ValidKernelPteLocal; +} + +static ULONG GetNextPageColorForWsList(PMMWSL WsList) +{ + return (WsList == MmSystemCacheWorkingSetList) ? MI_GET_NEXT_COLOR() : MI_GET_NEXT_PROCESS_COLOR(PsGetCurrentProcess()); +} + +static void FreeWsleIndex(PMMWSL WsList, ULONG Index) +{ + PMMWSLE Wsle = WsList->Wsle; + ULONG& LastEntry = WsList->LastEntry; + ULONG& FirstFree = WsList->FirstFree; + ULONG& LastInitializedWsle = WsList->LastInitializedWsle; + + /* Erase it now */ + Wsle[Index].u1.Long = 0; + + if (Index == (LastEntry - 1)) + { + /* We're freeing the last index of our list. */ + while (Wsle[Index].u1.e1.Valid == 0) + Index--; + + /* Should we bother about the Free entries */ + if (FirstFree < Index) + { + /* Try getting the index of the last free entry */ + ASSERT(Wsle[Index + 1].u1.Free.MustBeZero == 0); + ULONG PreviousFree = Wsle[Index + 1].u1.Free.PreviousFree; + ASSERT(PreviousFree < LastEntry); + ULONG LastFree = Index + 1 - PreviousFree; +#ifdef MMWSLE_PREVIOUS_FREE_JUMP + while (Wsle[LastFree].u1.e1.Valid) + { + ASSERT(LastFree > MMWSLE_PREVIOUS_FREE_JUMP); + LastFree -= MMWSLE_PREVIOUS_FREE_JUMP; + } +#endif + /* Update */ + ASSERT(LastFree >= FirstFree); + Wsle[FirstFree].u1.Free.PreviousFree = (Index + 1 - LastFree) & MMWSLE_PREVIOUS_FREE_MASK; + Wsle[LastFree].u1.Free.NextFree = 0; + } + else + { + /* No more free entries in our array */ + FirstFree = ULONG_MAX; + } + /* This is the new size of our array */ + LastEntry = Index + 1; + /* Should we shrink the alloc? */ + while ((LastInitializedWsle - LastEntry) > (PAGE_SIZE / sizeof(MMWSLE))) + { + PMMPTE PointerPte = MiAddressToPte(Wsle + LastInitializedWsle - 1); + /* We must not free ourself! */ + ASSERT(MiPteToAddress(PointerPte) != WsList); + + PFN_NUMBER Page = PFN_FROM_PTE(PointerPte); + PMMPFN Pfn = MiGetPfnEntry(Page); + + MI_SET_PFN_DELETED(Pfn); + MiDecrementShareCount(MiGetPfnEntry(Pfn->u4.PteFrame), Pfn->u4.PteFrame); + MiDecrementShareCount(Pfn, Page); + + PointerPte->u.Long = 0; + + KeInvalidateTlbEntry(Wsle + LastInitializedWsle - 1); + LastInitializedWsle -= PAGE_SIZE / sizeof(MMWSLE); + } + return; + } + + if (FirstFree == ULONG_MAX) + { + /* We're the first one. */ + FirstFree = Index; + Wsle[FirstFree].u1.Free.PreviousFree = (LastEntry - FirstFree) & MMWSLE_PREVIOUS_FREE_MASK; + return; + } + + /* We must find where to place ourself */ + ULONG NextFree = FirstFree; + ULONG PreviousFree = 0; + while (NextFree < Index) + { + ASSERT(Wsle[NextFree].u1.Free.MustBeZero == 0); + if (Wsle[NextFree].u1.Free.NextFree == 0) + break; + PreviousFree = NextFree; + NextFree += Wsle[NextFree].u1.Free.NextFree; + } + + if (NextFree < Index) + { + /* This is actually the last free entry */ + Wsle[NextFree].u1.Free.NextFree = Index - NextFree; + Wsle[Index].u1.Free.PreviousFree = (Index - NextFree) & MMWSLE_PREVIOUS_FREE_MASK; + Wsle[FirstFree].u1.Free.PreviousFree = (LastEntry - Index) & MMWSLE_PREVIOUS_FREE_MASK; + return; + } + + if (PreviousFree == 0) + { + /* This is the first free */ + Wsle[Index].u1.Free.NextFree = FirstFree - Index; + Wsle[Index].u1.Free.PreviousFree = Wsle[FirstFree].u1.Free.PreviousFree; + Wsle[FirstFree].u1.Free.PreviousFree = (FirstFree - Index) & MMWSLE_PREVIOUS_FREE_MASK; + FirstFree = Index; + return; + } + + /* Insert */ + Wsle[PreviousFree].u1.Free.NextFree = (Index - PreviousFree); + Wsle[Index].u1.Free.PreviousFree = (Index - PreviousFree) & MMWSLE_PREVIOUS_FREE_MASK; + Wsle[Index].u1.Free.NextFree = NextFree - Index; + Wsle[NextFree].u1.Free.PreviousFree = (NextFree - Index) & MMWSLE_PREVIOUS_FREE_MASK; +} + +static ULONG GetFreeWsleIndex(PMMWSL WsList) +{ + ULONG Index; + if (WsList->FirstFree != ULONG_MAX) + { + Index = WsList->FirstFree; + ASSERT(Index < WsList->LastInitializedWsle); + MMWSLE_FREE_ENTRY& FreeWsle = WsList->Wsle[Index].u1.Free; + ASSERT(FreeWsle.MustBeZero == 0); + if (FreeWsle.NextFree != 0) + { + WsList->FirstFree += FreeWsle.NextFree; + WsList->Wsle[WsList->FirstFree].u1.Free.PreviousFree = FreeWsle.PreviousFree; + } + else + { + WsList->FirstFree = ULONG_MAX; + } + } + else + { + Index = WsList->LastEntry++; + if (Index >= WsList->LastInitializedWsle) + { + /* Grow our array */ + PMMPTE PointerPte = MiAddressToPte(&WsList->Wsle[WsList->LastInitializedWsle]); + ASSERT(PointerPte->u.Hard.Valid == 0); + MMPTE TempPte = GetPteTemplateForWsList(WsList); + TempPte.u.Hard.PageFrameNumber = MiRemoveAnyPage(GetNextPageColorForWsList(WsList)); + MiInitializePfnAndMakePteValid(TempPte.u.Hard.PageFrameNumber, PointerPte, TempPte); + WsList->LastInitializedWsle += PAGE_SIZE / sizeof(MMWSLE); + } + } + + WsList->Wsle[Index].u1.Long = 0; + return Index; +} + +static +VOID +RemoveFromWsList(PMMWSL WsList, PVOID Address) +{ + /* Make sure that we are holding the right locks. */ + ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread())); + MI_ASSERT_PFN_LOCK_HELD(); + + PMMPTE PointerPte = MiAddressToPte(Address); + + /* Make sure we are removing a paged-in address */ + ASSERT(PointerPte->u.Hard.Valid == 1); + PMMPFN Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); + ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); + + /* Shared pages not supported yet */ + ASSERT(Pfn1->u3.e1.PrototypePte == 0); + + /* Nor are "ROS PFN" */ + ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE); + + /* And we should have a valid index here */ + ASSERT(Pfn1->u1.WsIndex != 0); + + FreeWsleIndex(WsList, Pfn1->u1.WsIndex); +} + +static +ULONG +TrimWsList(PMMWSL WsList) +{ + /* This should be done under WS lock */ + ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread())); + + ULONG Ret = 0; + + /* Walk the array */ + for (ULONG i = WsList->FirstDynamic; i < WsList->LastEntry; i++) + { + MMWSLE& Entry = WsList->Wsle[i]; + if (!Entry.u1.e1.Valid) + continue; + + /* Only direct entries for now */ + ASSERT(Entry.u1.e1.Direct == 1); + + /* Check the PTE */ + PMMPTE PointerPte = MiAddressToPte(Entry.u1.VirtualAddress); + + /* This must be valid */ + ASSERT(PointerPte->u.Hard.Valid); + + /* If the PTE was accessed, simply reset and that's the end of it */ + if (PointerPte->u.Hard.Accessed) + { + Entry.u1.e1.Age = 0; + PointerPte->u.Hard.Accessed = 0; + KeInvalidateTlbEntry(Entry.u1.VirtualAddress); + continue; + } + + /* If the entry is not so old, just age it */ + if (Entry.u1.e1.Age < 3) + { + Entry.u1.e1.Age++; + continue; + } + + if ((Entry.u1.e1.LockedInMemory) || (Entry.u1.e1.LockedInWs)) + { + /* This one is locked. Next time, maybe... */ + continue; + } + + /* FIXME: Invalidating PDEs breaks legacy MMs */ + if (MI_IS_PAGE_TABLE_ADDRESS(Entry.u1.VirtualAddress)) + continue; + + /* Please put yourself aside and make place for the younger ones */ + PFN_NUMBER Page = PFN_FROM_PTE(PointerPte); + KIRQL OldIrql = MiAcquirePfnLock(); + + PMMPFN Pfn = MiGetPfnEntry(Page); + + /* Not supported yet */ + ASSERT(Pfn->u3.e1.PrototypePte == 0); + ASSERT(!MI_IS_ROS_PFN(Pfn)); + + /* FIXME: Remove this hack when possible */ + if (Pfn->Wsle.u1.e1.LockedInMemory || (Pfn->Wsle.u1.e1.LockedInWs)) + { + MiReleasePfnLock(OldIrql); + continue; + } + + /* We can remove it from the list. Save Protection first */ + ULONG Protection = Entry.u1.e1.Protection; + RemoveFromWsList(WsList, Entry.u1.VirtualAddress); + + /* Dirtify the page, if needed */ + if (PointerPte->u.Hard.Dirty) + Pfn->u3.e1.Modified = 1; + + /* Make this a transition PTE */ + MI_MAKE_TRANSITION_PTE(PointerPte, Page, Protection); + KeInvalidateTlbEntry(MiAddressToPte(PointerPte)); + + /* Drop the share count. This will take care of putting it in the standby or modified list. */ + MiDecrementShareCount(Pfn, Page); + + MiReleasePfnLock(OldIrql); + Ret++; + } + return Ret; +} + +/* GLOBAL FUNCTIONS ***********************************************************/ +extern "C" +{ + +VOID +NTAPI +MiInsertInWorkingSetList( + _Inout_ PMMSUPPORT Vm, + _In_ PVOID Address, + _In_ ULONG Protection) +{ + PMMWSL WsList = Vm->VmWorkingSetList; + + /* Make sure that we are holding the right locks. */ + ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread())); + MI_ASSERT_PFN_LOCK_HELD(); + + PMMPTE PointerPte = MiAddressToPte(Address); + + /* Make sure we are adding a paged-in address */ + ASSERT(PointerPte->u.Hard.Valid == 1); + PMMPFN Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte)); + ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); + + /* Shared pages not supported yet */ + ASSERT(Pfn1->u1.WsIndex == 0); + ASSERT(Pfn1->u3.e1.PrototypePte == 0); + + /* Nor are "ROS PFN" */ + ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE); + + Pfn1->u1.WsIndex = GetFreeWsleIndex(WsList); + MMWSLENTRY& NewWsle = WsList->Wsle[Pfn1->u1.WsIndex].u1.e1; + NewWsle.VirtualPageNumber = reinterpret_cast(Address) >> PAGE_SHIFT; + NewWsle.Protection = Protection; + NewWsle.Direct = 1; + NewWsle.Hashed = 0; + NewWsle.LockedInMemory = 0; + NewWsle.LockedInWs = 0; + NewWsle.Age = 0; + NewWsle.Valid = 1; + + Vm->WorkingSetSize += PAGE_SIZE; + if (Vm->WorkingSetSize > Vm->PeakWorkingSetSize) + Vm->PeakWorkingSetSize = Vm->WorkingSetSize; +} + +VOID +NTAPI +MiRemoveFromWorkingSetList( + _Inout_ PMMSUPPORT Vm, + _In_ PVOID Address) +{ + RemoveFromWsList(Vm->VmWorkingSetList, Address); + + Vm->WorkingSetSize -= PAGE_SIZE; +} + +_Requires_exclusive_lock_held_(WorkingSet->WorkingSetMutex) +VOID +NTAPI +MiInitializeWorkingSetList(_Inout_ PMMSUPPORT WorkingSet) +{ + PMMWSL WsList = WorkingSet->VmWorkingSetList; + + /* Initialize some fields */ + WsList->FirstFree = ULONG_MAX; + WsList->Wsle = reinterpret_cast(WsList + 1); + WsList->LastEntry = 0; + /* The first page is already allocated */ + WsList->LastInitializedWsle = (PAGE_SIZE - sizeof(*WsList)) / sizeof(MMWSLE); + + /* Insert the address we already know: our PDE base and the Working Set List */ + if (MI_IS_PROCESS_WORKING_SET(WorkingSet)) + { + ASSERT(WorkingSet->VmWorkingSetList == MmWorkingSetList); +#if _MI_PAGING_LEVELS == 4 + MiInsertInWorkingSetList(WorkingSet, (PVOID)PXE_BASE, 0U); +#elif _MI_PAGING_LEVELS == 3 + MiInsertInWorkingSetList(WorkingSet, (PVOID)PPE_BASE, 0U); +#elif _MI_PAGING_LEVELS == 2 + MiInsertInWorkingSetList(WorkingSet, (PVOID)PDE_BASE, 0U); +#endif + } + +#if _MI_PAGING_LEVELS == 4 + MiInsertInWorkingSetList(WorkingSet, MiAddressToPpe(WorkingSet->VmWorkingSetList), 0UL); +#endif +#if _MI_PAGING_LEVELS >= 3 + MiInsertInWorkingSetList(WorkingSet, MiAddressToPde(WorkingSet->VmWorkingSetList), 0UL); +#endif + MiInsertInWorkingSetList(WorkingSet, (PVOID)MiAddressToPte(WorkingSet->VmWorkingSetList), 0UL); + MiInsertInWorkingSetList(WorkingSet, (PVOID)WorkingSet->VmWorkingSetList, 0UL); + + /* From now on, every added page can be trimmed at any time */ + WsList->FirstDynamic = WsList->LastEntry; + + /* We can add this to our list */ + ExInterlockedInsertTailList(&MmWorkingSetExpansionHead, &WorkingSet->WorkingSetExpansionLinks, &MmExpansionLock); +} + +VOID +NTAPI +MmWorkingSetManager(VOID) +{ + PLIST_ENTRY VmListEntry; + PMMSUPPORT Vm = NULL; + KIRQL OldIrql; + + OldIrql = MiAcquireExpansionLock(); + + for (VmListEntry = MmWorkingSetExpansionHead.Flink; + VmListEntry != &MmWorkingSetExpansionHead; + VmListEntry = VmListEntry->Flink) + { + BOOLEAN TrimHard = MmAvailablePages < MmMinimumFreePages; + PEPROCESS Process = NULL; + + /* Don't do anything if we have plenty of free pages. */ + if ((MmAvailablePages + MmModifiedPageListHead.Total) >= MmPlentyFreePages) + break; + + Vm = CONTAINING_RECORD(VmListEntry, MMSUPPORT, WorkingSetExpansionLinks); + + /* Let the legacy Mm System space alone */ + if (Vm == MmGetKernelAddressSpace()) + continue; + + if (MI_IS_PROCESS_WORKING_SET(Vm)) + { + Process = CONTAINING_RECORD(Vm, EPROCESS, Vm); + + /* Make sure the process is not terminating abd attach to it */ + if (!ExAcquireRundownProtection(&Process->RundownProtect)) + continue; + ASSERT(!KeIsAttachedProcess()); + KeAttachProcess(&Process->Pcb); + } + else + { + /* FIXME: Session & system space unsupported */ + continue; + } + + MiReleaseExpansionLock(OldIrql); + + /* Share-lock for now, we're only reading */ + MiLockWorkingSetShared(PsGetCurrentThread(), Vm); + + if (((Vm->WorkingSetSize > Vm->MaximumWorkingSetSize) || + (TrimHard && (Vm->WorkingSetSize > Vm->MinimumWorkingSetSize))) && + MiConvertSharedWorkingSetLockToExclusive(PsGetCurrentThread(), Vm)) + { + /* We're done */ + Vm->Flags.BeingTrimmed = 1; + + ULONG Trimmed = TrimWsList(Vm->VmWorkingSetList); + + /* We're done */ + Vm->WorkingSetSize -= Trimmed * PAGE_SIZE; + Vm->Flags.BeingTrimmed = 0; + MiUnlockWorkingSet(PsGetCurrentThread(), Vm); + } + else + { + MiUnlockWorkingSetShared(PsGetCurrentThread(), Vm); + } + + /* Lock again */ + OldIrql = MiAcquireExpansionLock(); + + if (Process) + { + KeDetachProcess(); + ExReleaseRundownProtection(&Process->RundownProtect); + } + } + + MiReleaseExpansionLock(OldIrql); +} + +} // extern "C" + +} // namespace ntoskrnl diff --git a/ntoskrnl/ntos.cmake b/ntoskrnl/ntos.cmake index dbb9f911016..6e1edf2d944 100644 --- a/ntoskrnl/ntos.cmake +++ b/ntoskrnl/ntos.cmake @@ -229,6 +229,7 @@ list(APPEND SOURCE ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/syspte.c ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/vadnode.c ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/virtual.c + ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/wslist.cpp ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/zeropage.c ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/balance.c ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/freelist.c diff --git a/sdk/include/ndk/mmtypes.h b/sdk/include/ndk/mmtypes.h index 3e9fabf1c02..28d6cb00006 100644 --- a/sdk/include/ndk/mmtypes.h +++ b/sdk/include/ndk/mmtypes.h @@ -831,6 +831,21 @@ typedef struct _MMWSLENTRY ULONG_PTR VirtualPageNumber: MM_PAGE_FRAME_NUMBER_SIZE; } MMWSLENTRY, *PMMWSLENTRY; +typedef struct _MMWSLE_FREE_ENTRY +{ + ULONG MustBeZero:1; +#ifdef _WIN64 + ULONG PreviousFree: 31; + LONG NextFree; +#define MMWSLE_PREVIOUS_FREE_MASK 0x7FFFFFFF +#else + ULONG PreviousFree: 11; +#define MMWSLE_PREVIOUS_FREE_MASK 0x7FF +#define MMWSLE_PREVIOUS_FREE_JUMP 0x800 + LONG NextFree: 20; +#endif +} MMWSLE_FREE_ENTRY, *PMMWSLE_FREE_ENTRY; + typedef struct _MMWSLE { union @@ -838,6 +853,7 @@ typedef struct _MMWSLE PVOID VirtualAddress; ULONG_PTR Long; MMWSLENTRY e1; + MMWSLE_FREE_ENTRY Free; } u1; } MMWSLE, *PMMWSLE;