[NTOS:MM] First shot for Working Set list support

- Initialize
- Add private page (no shared page support yet)
- Remove pages
- Trim

Yes, this is C++ in the kernel.

ntoskrnl/include/internal/mm.h

@@ -1645,6 +1645,12 @@ MmCopyVirtualMemory(IN PEPROCESS SourceProcess,
                     IN KPROCESSOR_MODE PreviousMode,
                     OUT PSIZE_T ReturnSize);
 
+/* wslist.cpp ****************************************************************/
+_Requires_exclusive_lock_held_(WorkingSet->WorkingSetMutex)
+VOID
+NTAPI
+MiInitializeWorkingSetList(_Inout_ PMMSUPPORT WorkingSet);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

ntoskrnl/mm/ARM3/mminit.c

@@ -2139,6 +2139,9 @@ MmArmInitSystem(IN ULONG Phase,
         /* Initialize the user mode image list */
         InitializeListHead(&MmLoadedUserImageList);
 
+        /* Initalize the Working set list */
+        InitializeListHead(&MmWorkingSetExpansionHead);
+
         /* Initialize critical section timeout value (relative time is negative) */
         MmCriticalSectionTimeout.QuadPart = MmCritsectTimeoutSeconds * (-10000000LL);
 

ntoskrnl/mm/ARM3/procsup.c

@@ -896,40 +896,6 @@ MiInsertSharedUserPageVad(VOID)
 }
 #endif
 
-VOID
-NTAPI
-MiInitializeWorkingSetList(IN PEPROCESS CurrentProcess)
-{
-    PMMPFN Pfn1;
-    PMMPTE sysPte;
-    MMPTE tempPte;
-
-    /* Setup some bogus list data */
-    MmWorkingSetList->LastEntry = CurrentProcess->Vm.MinimumWorkingSetSize;
-    MmWorkingSetList->HashTable = NULL;
-    MmWorkingSetList->HashTableSize = 0;
-    MmWorkingSetList->NumberOfImageWaiters = 0;
-    MmWorkingSetList->Wsle = (PVOID)(ULONG_PTR)0xDEADBABEDEADBABEULL;
-    MmWorkingSetList->VadBitMapHint = 1;
-    MmWorkingSetList->HashTableStart = (PVOID)(ULONG_PTR)0xBADAB00BBADAB00BULL;
-    MmWorkingSetList->HighestPermittedHashAddress = (PVOID)(ULONG_PTR)0xCAFEBABECAFEBABEULL;
-    MmWorkingSetList->FirstFree = 1;
-    MmWorkingSetList->FirstDynamic = 2;
-    MmWorkingSetList->NextSlot = 3;
-    MmWorkingSetList->LastInitializedWsle = 4;
-
-    /* The rule is that the owner process is always in the FLINK of the PDE's PFN entry */
-    Pfn1 = MiGetPfnEntry(CurrentProcess->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT);
-    ASSERT(Pfn1->u4.PteFrame == MiGetPfnEntryIndex(Pfn1));
-    Pfn1->u1.Event = (PKEVENT)CurrentProcess;
-
-    /* Map the process working set in kernel space */
-    sysPte = MiReserveSystemPtes(1, SystemPteSpace);
-    MI_MAKE_HARDWARE_PTE_KERNEL(&tempPte, sysPte, MM_READWRITE, CurrentProcess->WorkingSetPage);
-    MI_WRITE_VALID_PTE(sysPte, tempPte);
-    CurrentProcess->Vm.VmWorkingSetList = MiPteToAddress(sysPte);
-}
-
 NTSTATUS
 NTAPI
 MmInitializeProcessAddressSpace(IN PEPROCESS Process,
@@ -944,6 +910,7 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
     PMMPTE PointerPte;
     KIRQL OldIrql;
     PMMPDE PointerPde;
+    PMMPFN Pfn;
     PFN_NUMBER PageFrameNumber;
     UNICODE_STRING FileName;
     PWCHAR Source;
@@ -980,6 +947,8 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
     /* On x64 the PFNs for the initial process are already set up */
     if (Process != &KiInitialProcess) {
 #endif
+    /* Lock our working set */
+    MiLockProcessWorkingSet(Process, PsGetCurrentThread());
 
     /* Lock PFN database */
     OldIrql = MiAcquirePfnLock();
@@ -997,7 +966,7 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
 
     /* Do the same for hyperspace */
-    PointerPde = MiAddressToPde((PVOID)HYPER_SPACE);
+    PointerPde = MiAddressToPde(HYPER_SPACE);
     PageFrameNumber = PFN_FROM_PTE(PointerPde);
     MiInitializePfn(PageFrameNumber, (PMMPTE)PointerPde, TRUE);
 #if (_MI_PAGING_LEVELS == 2)
@@ -1019,18 +988,30 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
     ASSERT(Process->Pcb.DirectoryTableBase[1] == PageFrameNumber * PAGE_SIZE);
 #endif
 
-    /* Setup the PFN for the PTE for the working set */
-    PointerPte = MiAddressToPte(MI_WORKING_SET_LIST);
-    MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, MM_READWRITE, 0);
-    ASSERT(PointerPte->u.Long != 0);
+    /* Do the same for the Working set list */
+    PointerPte = MiAddressToPte(MmWorkingSetList);
     PageFrameNumber = PFN_FROM_PTE(PointerPte);
-    MI_WRITE_INVALID_PTE(PointerPte, DemandZeroPte);
     MiInitializePfn(PageFrameNumber, PointerPte, TRUE);
-    TempPte.u.Hard.PageFrameNumber = PageFrameNumber;
-    MI_WRITE_VALID_PTE(PointerPte, TempPte);
+
+    /* This should be in hyper space, but not in the mapping range */
+    Process->Vm.VmWorkingSetList = MmWorkingSetList;
+    ASSERT(((ULONG_PTR)MmWorkingSetList >= MI_MAPPING_RANGE_END) && ((ULONG_PTR)MmWorkingSetList <= HYPER_SPACE_END));
 
     /* Now initialize the working set list */
-    MiInitializeWorkingSetList(Process);
+    MiInitializeWorkingSetList(&Process->Vm);
+
+    /* Map the process working set in kernel space */
+    /* FIXME: there should be no need */
+    PointerPte = MiReserveSystemPtes(1, SystemPteSpace);
+    MI_MAKE_HARDWARE_PTE_KERNEL(&TempPte, PointerPte, MM_READWRITE, Process->WorkingSetPage);
+    MI_WRITE_VALID_PTE(PointerPte, TempPte);
+    Process->Vm.VmWorkingSetList = MiPteToAddress(PointerPte);
+
+    /* The rule is that the owner process is always in the FLINK of the PDE's PFN entry */
+    Pfn = MiGetPfnEntry(Process->Pcb.DirectoryTableBase[0] >> PAGE_SHIFT);
+    ASSERT(Pfn->u4.PteFrame == MiGetPfnEntryIndex(Pfn));
+    ASSERT(Pfn->u1.WsIndex == 0);
+    Pfn->u1.Event = (PKEVENT)Process;
 
     /* Sanity check */
     ASSERT(Process->PhysicalVadRoot == NULL);
@@ -1038,6 +1019,8 @@ MmInitializeProcessAddressSpace(IN PEPROCESS Process,
     /* Release PFN lock */
     MiReleasePfnLock(OldIrql);
 
+    /* Release the process working set */
+    MiUnlockProcessWorkingSet(Process, PsGetCurrentThread());
 #ifdef _M_AMD64
    } /* On x64 the PFNs for the initial process are already set up */
 #endif
@@ -1358,6 +1341,11 @@ MmDeleteProcessAddressSpace2(IN PEPROCESS Process)
 
     //ASSERT(Process->CommitCharge == 0);
 
+    /* Remove us from the list */
+    OldIrql = MiAcquireExpansionLock();
+    RemoveEntryList(&Process->Vm.WorkingSetExpansionLinks);
+    MiReleaseExpansionLock(OldIrql);
+
     /* Acquire the PFN lock */
     OldIrql = MiAcquirePfnLock();
 

ntoskrnl/mm/ARM3/session.c

@@ -41,7 +41,6 @@ MiInitializeSessionWsSupport(VOID)
 {
     /* Initialize the list heads */
     InitializeListHead(&MiSessionWsList);
-    InitializeListHead(&MmWorkingSetExpansionHead);
 }
 
 BOOLEAN

ntoskrnl/mm/ARM3/wslist.cpp

@@ -0,0 +1,484 @@
+/*
+ * PROJECT:         ReactOS Kernel
+ * LICENSE:         BSD-3-Clause (https://spdx.org/licenses/BSD-3-Clause.html)
+ * FILE:            ntoskrnl/mm/ARM3/wslist.cpp
+ * PURPOSE:         Working set list management
+ * PROGRAMMERS:     Jérôme Gardou
+ */
+
+/* INCLUDES *******************************************************************/
+#include <ntoskrnl.h>
+
+#define NDEBUG
+#include <debug.h>
+
+#define MODULE_INVOLVED_IN_ARM3
+#include "miarm.h"
+
+/* GLOBALS ********************************************************************/
+PMMWSL MmWorkingSetList;
+KEVENT MmWorkingSetManagerEvent;
+
+/* LOCAL FUNCTIONS ************************************************************/
+namespace ntoskrnl
+{
+
+static MMPTE GetPteTemplateForWsList(PMMWSL WsList)
+{
+    return (WsList == MmSystemCacheWorkingSetList) ? ValidKernelPte : ValidKernelPteLocal;
+}
+
+static ULONG GetNextPageColorForWsList(PMMWSL WsList)
+{
+    return (WsList == MmSystemCacheWorkingSetList) ? MI_GET_NEXT_COLOR() : MI_GET_NEXT_PROCESS_COLOR(PsGetCurrentProcess());
+}
+
+static void FreeWsleIndex(PMMWSL WsList, ULONG Index)
+{
+    PMMWSLE Wsle = WsList->Wsle;
+    ULONG& LastEntry = WsList->LastEntry;
+    ULONG& FirstFree = WsList->FirstFree;
+    ULONG& LastInitializedWsle = WsList->LastInitializedWsle;
+
+    /* Erase it now */
+    Wsle[Index].u1.Long = 0;
+
+    if (Index == (LastEntry - 1))
+    {
+        /* We're freeing the last index of our list. */
+        while (Wsle[Index].u1.e1.Valid == 0)
+            Index--;
+
+        /* Should we bother about the Free entries */
+        if (FirstFree < Index)
+        {
+            /* Try getting the index of the last free entry */
+            ASSERT(Wsle[Index + 1].u1.Free.MustBeZero == 0);
+            ULONG PreviousFree = Wsle[Index + 1].u1.Free.PreviousFree;
+            ASSERT(PreviousFree < LastEntry);
+            ULONG LastFree = Index + 1 - PreviousFree;
+#ifdef MMWSLE_PREVIOUS_FREE_JUMP
+            while (Wsle[LastFree].u1.e1.Valid)
+            {
+                ASSERT(LastFree > MMWSLE_PREVIOUS_FREE_JUMP);
+                LastFree -= MMWSLE_PREVIOUS_FREE_JUMP;
+            }
+#endif
+            /* Update */
+            ASSERT(LastFree >= FirstFree);
+            Wsle[FirstFree].u1.Free.PreviousFree = (Index + 1 - LastFree) & MMWSLE_PREVIOUS_FREE_MASK;
+            Wsle[LastFree].u1.Free.NextFree = 0;
+        }
+        else
+        {
+            /* No more free entries in our array */
+            FirstFree = ULONG_MAX;
+        }
+        /* This is the new size of our array */
+        LastEntry = Index + 1;
+        /* Should we shrink the alloc? */
+        while ((LastInitializedWsle - LastEntry) > (PAGE_SIZE / sizeof(MMWSLE)))
+        {
+            PMMPTE PointerPte = MiAddressToPte(Wsle + LastInitializedWsle - 1);
+            /* We must not free ourself! */
+            ASSERT(MiPteToAddress(PointerPte) != WsList);
+
+            PFN_NUMBER Page = PFN_FROM_PTE(PointerPte);
+            PMMPFN Pfn = MiGetPfnEntry(Page);
+
+            MI_SET_PFN_DELETED(Pfn);
+            MiDecrementShareCount(MiGetPfnEntry(Pfn->u4.PteFrame), Pfn->u4.PteFrame);
+            MiDecrementShareCount(Pfn, Page);
+
+            PointerPte->u.Long = 0;
+
+            KeInvalidateTlbEntry(Wsle + LastInitializedWsle - 1);
+            LastInitializedWsle -= PAGE_SIZE / sizeof(MMWSLE);
+        }
+        return;
+    }
+
+    if (FirstFree == ULONG_MAX)
+    {
+        /* We're the first one. */
+        FirstFree = Index;
+        Wsle[FirstFree].u1.Free.PreviousFree = (LastEntry - FirstFree) & MMWSLE_PREVIOUS_FREE_MASK;
+        return;
+    }
+
+    /* We must find where to place ourself */
+    ULONG NextFree = FirstFree;
+    ULONG PreviousFree = 0;
+    while (NextFree < Index)
+    {
+        ASSERT(Wsle[NextFree].u1.Free.MustBeZero == 0);
+        if (Wsle[NextFree].u1.Free.NextFree == 0)
+            break;
+        PreviousFree = NextFree;
+        NextFree += Wsle[NextFree].u1.Free.NextFree;
+    }
+
+    if (NextFree < Index)
+    {
+        /* This is actually the last free entry */
+        Wsle[NextFree].u1.Free.NextFree = Index - NextFree;
+        Wsle[Index].u1.Free.PreviousFree = (Index - NextFree) & MMWSLE_PREVIOUS_FREE_MASK;
+        Wsle[FirstFree].u1.Free.PreviousFree = (LastEntry - Index) & MMWSLE_PREVIOUS_FREE_MASK;
+        return;
+    }
+
+    if (PreviousFree == 0)
+    {
+        /* This is the first free */
+        Wsle[Index].u1.Free.NextFree = FirstFree - Index;
+        Wsle[Index].u1.Free.PreviousFree = Wsle[FirstFree].u1.Free.PreviousFree;
+        Wsle[FirstFree].u1.Free.PreviousFree = (FirstFree - Index) & MMWSLE_PREVIOUS_FREE_MASK;
+        FirstFree = Index;
+        return;
+    }
+
+    /* Insert */
+    Wsle[PreviousFree].u1.Free.NextFree = (Index - PreviousFree);
+    Wsle[Index].u1.Free.PreviousFree = (Index - PreviousFree) & MMWSLE_PREVIOUS_FREE_MASK;
+    Wsle[Index].u1.Free.NextFree = NextFree - Index;
+    Wsle[NextFree].u1.Free.PreviousFree = (NextFree - Index) & MMWSLE_PREVIOUS_FREE_MASK;
+}
+
+static ULONG GetFreeWsleIndex(PMMWSL WsList)
+{
+    ULONG Index;
+    if (WsList->FirstFree != ULONG_MAX)
+    {
+        Index = WsList->FirstFree;
+        ASSERT(Index < WsList->LastInitializedWsle);
+        MMWSLE_FREE_ENTRY& FreeWsle = WsList->Wsle[Index].u1.Free;
+        ASSERT(FreeWsle.MustBeZero == 0);
+        if (FreeWsle.NextFree != 0)
+        {
+            WsList->FirstFree += FreeWsle.NextFree;
+            WsList->Wsle[WsList->FirstFree].u1.Free.PreviousFree = FreeWsle.PreviousFree;
+        }
+        else
+        {
+            WsList->FirstFree = ULONG_MAX;
+        }
+    }
+    else
+    {
+        Index = WsList->LastEntry++;
+        if (Index >= WsList->LastInitializedWsle)
+        {
+            /* Grow our array */
+            PMMPTE PointerPte = MiAddressToPte(&WsList->Wsle[WsList->LastInitializedWsle]);
+            ASSERT(PointerPte->u.Hard.Valid == 0);
+            MMPTE TempPte = GetPteTemplateForWsList(WsList);
+            TempPte.u.Hard.PageFrameNumber = MiRemoveAnyPage(GetNextPageColorForWsList(WsList));
+            MiInitializePfnAndMakePteValid(TempPte.u.Hard.PageFrameNumber, PointerPte, TempPte);
+            WsList->LastInitializedWsle += PAGE_SIZE / sizeof(MMWSLE);
+        }
+    }
+
+    WsList->Wsle[Index].u1.Long = 0;
+    return Index;
+}
+
+static
+VOID
+RemoveFromWsList(PMMWSL WsList, PVOID Address)
+{
+    /* Make sure that we are holding the right locks. */
+    ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
+    MI_ASSERT_PFN_LOCK_HELD();
+
+    PMMPTE PointerPte = MiAddressToPte(Address);
+
+    /* Make sure we are removing a paged-in address */
+    ASSERT(PointerPte->u.Hard.Valid == 1);
+    PMMPFN Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
+    ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
+
+    /* Shared pages not supported yet */
+    ASSERT(Pfn1->u3.e1.PrototypePte == 0);
+
+    /* Nor are "ROS PFN" */
+    ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
+
+    /* And we should have a valid index here */
+    ASSERT(Pfn1->u1.WsIndex != 0);
+
+    FreeWsleIndex(WsList, Pfn1->u1.WsIndex);
+}
+
+static
+ULONG
+TrimWsList(PMMWSL WsList)
+{
+    /* This should be done under WS lock */
+    ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
+
+    ULONG Ret = 0;
+
+    /* Walk the array */
+    for (ULONG i = WsList->FirstDynamic; i < WsList->LastEntry; i++)
+    {
+        MMWSLE& Entry = WsList->Wsle[i];
+        if (!Entry.u1.e1.Valid)
+            continue;
+
+        /* Only direct entries for now */
+        ASSERT(Entry.u1.e1.Direct == 1);
+
+        /* Check the PTE */
+        PMMPTE PointerPte = MiAddressToPte(Entry.u1.VirtualAddress);
+
+        /* This must be valid */
+        ASSERT(PointerPte->u.Hard.Valid);
+
+        /* If the PTE was accessed, simply reset and that's the end of it */
+        if (PointerPte->u.Hard.Accessed)
+        {
+            Entry.u1.e1.Age = 0;
+            PointerPte->u.Hard.Accessed = 0;
+            KeInvalidateTlbEntry(Entry.u1.VirtualAddress);
+            continue;
+        }
+
+        /* If the entry is not so old, just age it */
+        if (Entry.u1.e1.Age < 3)
+        {
+            Entry.u1.e1.Age++;
+            continue;
+        }
+
+        if ((Entry.u1.e1.LockedInMemory) || (Entry.u1.e1.LockedInWs))
+        {
+            /* This one is locked. Next time, maybe... */
+            continue;
+        }
+
+        /* FIXME: Invalidating PDEs breaks legacy MMs */
+        if (MI_IS_PAGE_TABLE_ADDRESS(Entry.u1.VirtualAddress))
+            continue;
+
+        /* Please put yourself aside and make place for the younger ones */
+        PFN_NUMBER Page = PFN_FROM_PTE(PointerPte);
+        KIRQL OldIrql = MiAcquirePfnLock();
+
+        PMMPFN Pfn = MiGetPfnEntry(Page);
+
+        /* Not supported yet */
+        ASSERT(Pfn->u3.e1.PrototypePte == 0);
+        ASSERT(!MI_IS_ROS_PFN(Pfn));
+
+        /* FIXME: Remove this hack when possible */
+        if (Pfn->Wsle.u1.e1.LockedInMemory || (Pfn->Wsle.u1.e1.LockedInWs))
+        {
+            MiReleasePfnLock(OldIrql);
+            continue;
+        }
+
+        /* We can remove it from the list. Save Protection first */
+        ULONG Protection = Entry.u1.e1.Protection;
+        RemoveFromWsList(WsList, Entry.u1.VirtualAddress);
+
+        /* Dirtify the page, if needed */
+        if (PointerPte->u.Hard.Dirty)
+            Pfn->u3.e1.Modified = 1;
+
+        /* Make this a transition PTE */
+        MI_MAKE_TRANSITION_PTE(PointerPte, Page, Protection);
+        KeInvalidateTlbEntry(MiAddressToPte(PointerPte));
+
+        /* Drop the share count. This will take care of putting it in the standby or modified list. */
+        MiDecrementShareCount(Pfn, Page);
+
+        MiReleasePfnLock(OldIrql);
+        Ret++;
+    }
+    return Ret;
+}
+
+/* GLOBAL FUNCTIONS ***********************************************************/
+extern "C"
+{
+
+VOID
+NTAPI
+MiInsertInWorkingSetList(
+    _Inout_ PMMSUPPORT Vm,
+    _In_ PVOID Address,
+    _In_ ULONG Protection)
+{
+    PMMWSL WsList = Vm->VmWorkingSetList;
+
+    /* Make sure that we are holding the right locks. */
+    ASSERT(MM_ANY_WS_LOCK_HELD(PsGetCurrentThread()));
+    MI_ASSERT_PFN_LOCK_HELD();
+
+    PMMPTE PointerPte = MiAddressToPte(Address);
+
+    /* Make sure we are adding a paged-in address */
+    ASSERT(PointerPte->u.Hard.Valid == 1);
+    PMMPFN Pfn1 = MiGetPfnEntry(PFN_FROM_PTE(PointerPte));
+    ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid);
+
+    /* Shared pages not supported yet */
+    ASSERT(Pfn1->u1.WsIndex == 0);
+    ASSERT(Pfn1->u3.e1.PrototypePte == 0);
+
+    /* Nor are "ROS PFN" */
+    ASSERT(MI_IS_ROS_PFN(Pfn1) == FALSE);
+
+    Pfn1->u1.WsIndex = GetFreeWsleIndex(WsList);
+    MMWSLENTRY& NewWsle = WsList->Wsle[Pfn1->u1.WsIndex].u1.e1;
+    NewWsle.VirtualPageNumber = reinterpret_cast<ULONG_PTR>(Address) >> PAGE_SHIFT;
+    NewWsle.Protection = Protection;
+    NewWsle.Direct = 1;
+    NewWsle.Hashed = 0;
+    NewWsle.LockedInMemory = 0;
+    NewWsle.LockedInWs = 0;
+    NewWsle.Age = 0;
+    NewWsle.Valid = 1;
+
+    Vm->WorkingSetSize += PAGE_SIZE;
+    if (Vm->WorkingSetSize > Vm->PeakWorkingSetSize)
+        Vm->PeakWorkingSetSize = Vm->WorkingSetSize;
+}
+
+VOID
+NTAPI
+MiRemoveFromWorkingSetList(
+    _Inout_ PMMSUPPORT Vm,
+    _In_ PVOID Address)
+{
+    RemoveFromWsList(Vm->VmWorkingSetList, Address);
+
+    Vm->WorkingSetSize -= PAGE_SIZE;
+}
+
+_Requires_exclusive_lock_held_(WorkingSet->WorkingSetMutex)
+VOID
+NTAPI
+MiInitializeWorkingSetList(_Inout_ PMMSUPPORT WorkingSet)
+{
+    PMMWSL WsList = WorkingSet->VmWorkingSetList;
+
+    /* Initialize some fields */
+    WsList->FirstFree = ULONG_MAX;
+    WsList->Wsle = reinterpret_cast<PMMWSLE>(WsList + 1);
+    WsList->LastEntry = 0;
+    /* The first page is already allocated */
+    WsList->LastInitializedWsle = (PAGE_SIZE - sizeof(*WsList)) / sizeof(MMWSLE);
+
+    /* Insert the address we already know: our PDE base and the Working Set List */
+    if (MI_IS_PROCESS_WORKING_SET(WorkingSet))
+    {
+        ASSERT(WorkingSet->VmWorkingSetList == MmWorkingSetList);
+#if _MI_PAGING_LEVELS == 4
+        MiInsertInWorkingSetList(WorkingSet, (PVOID)PXE_BASE, 0U);
+#elif _MI_PAGING_LEVELS == 3
+        MiInsertInWorkingSetList(WorkingSet, (PVOID)PPE_BASE, 0U);
+#elif _MI_PAGING_LEVELS == 2
+        MiInsertInWorkingSetList(WorkingSet, (PVOID)PDE_BASE, 0U);
+#endif
+    }
+
+#if _MI_PAGING_LEVELS == 4
+    MiInsertInWorkingSetList(WorkingSet, MiAddressToPpe(WorkingSet->VmWorkingSetList), 0UL);
+#endif
+#if _MI_PAGING_LEVELS >= 3
+    MiInsertInWorkingSetList(WorkingSet, MiAddressToPde(WorkingSet->VmWorkingSetList), 0UL);
+#endif
+    MiInsertInWorkingSetList(WorkingSet, (PVOID)MiAddressToPte(WorkingSet->VmWorkingSetList), 0UL);
+    MiInsertInWorkingSetList(WorkingSet, (PVOID)WorkingSet->VmWorkingSetList, 0UL);
+
+    /* From now on, every added page can be trimmed at any time */
+    WsList->FirstDynamic = WsList->LastEntry;
+
+    /* We can add this to our list */
+    ExInterlockedInsertTailList(&MmWorkingSetExpansionHead, &WorkingSet->WorkingSetExpansionLinks, &MmExpansionLock);
+}
+
+VOID
+NTAPI
+MmWorkingSetManager(VOID)
+{
+    PLIST_ENTRY VmListEntry;
+    PMMSUPPORT Vm = NULL;
+    KIRQL OldIrql;
+
+    OldIrql = MiAcquireExpansionLock();
+
+    for (VmListEntry = MmWorkingSetExpansionHead.Flink;
+         VmListEntry != &MmWorkingSetExpansionHead;
+         VmListEntry = VmListEntry->Flink)
+    {
+        BOOLEAN TrimHard = MmAvailablePages < MmMinimumFreePages;
+        PEPROCESS Process = NULL;
+
+        /* Don't do anything if we have plenty of free pages. */
+        if ((MmAvailablePages + MmModifiedPageListHead.Total) >= MmPlentyFreePages)
+            break;
+
+        Vm = CONTAINING_RECORD(VmListEntry, MMSUPPORT, WorkingSetExpansionLinks);
+
+        /* Let the legacy Mm System space alone */
+        if (Vm == MmGetKernelAddressSpace())
+            continue;
+
+        if (MI_IS_PROCESS_WORKING_SET(Vm))
+        {
+            Process = CONTAINING_RECORD(Vm, EPROCESS, Vm);
+
+            /* Make sure the process is not terminating abd attach to it */
+            if (!ExAcquireRundownProtection(&Process->RundownProtect))
+                continue;
+            ASSERT(!KeIsAttachedProcess());
+            KeAttachProcess(&Process->Pcb);
+        }
+        else
+        {
+            /* FIXME: Session & system space unsupported */
+            continue;
+        }
+
+        MiReleaseExpansionLock(OldIrql);
+
+        /* Share-lock for now, we're only reading */
+        MiLockWorkingSetShared(PsGetCurrentThread(), Vm);
+
+        if (((Vm->WorkingSetSize > Vm->MaximumWorkingSetSize) ||
+            (TrimHard && (Vm->WorkingSetSize > Vm->MinimumWorkingSetSize))) &&
+            MiConvertSharedWorkingSetLockToExclusive(PsGetCurrentThread(), Vm))
+        {
+            /* We're done */
+            Vm->Flags.BeingTrimmed = 1;
+
+            ULONG Trimmed = TrimWsList(Vm->VmWorkingSetList);
+
+            /* We're done */
+            Vm->WorkingSetSize -= Trimmed * PAGE_SIZE;
+            Vm->Flags.BeingTrimmed = 0;
+            MiUnlockWorkingSet(PsGetCurrentThread(), Vm);
+        }
+        else
+        {
+            MiUnlockWorkingSetShared(PsGetCurrentThread(), Vm);
+        }
+
+        /* Lock again */
+        OldIrql = MiAcquireExpansionLock();
+
+        if (Process)
+        {
+            KeDetachProcess();
+            ExReleaseRundownProtection(&Process->RundownProtect);
+        }
+    }
+
+    MiReleaseExpansionLock(OldIrql);
+}
+
+} // extern "C"
+
+} // namespace ntoskrnl

ntoskrnl/ntos.cmake

@@ -229,6 +229,7 @@ list(APPEND SOURCE
     ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/syspte.c
     ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/vadnode.c
     ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/virtual.c
+    ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/wslist.cpp
     ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/ARM3/zeropage.c
     ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/balance.c
     ${REACTOS_SOURCE_DIR}/ntoskrnl/mm/freelist.c

sdk/include/ndk/mmtypes.h

@@ -831,6 +831,21 @@ typedef struct _MMWSLENTRY
     ULONG_PTR VirtualPageNumber: MM_PAGE_FRAME_NUMBER_SIZE;
 } MMWSLENTRY, *PMMWSLENTRY;
 
+typedef struct _MMWSLE_FREE_ENTRY
+{
+    ULONG MustBeZero:1;
+#ifdef _WIN64
+    ULONG PreviousFree: 31;
+    LONG NextFree;
+#define MMWSLE_PREVIOUS_FREE_MASK 0x7FFFFFFF
+#else
+    ULONG PreviousFree: 11;
+#define MMWSLE_PREVIOUS_FREE_MASK 0x7FF
+#define MMWSLE_PREVIOUS_FREE_JUMP 0x800
+    LONG NextFree: 20;
+#endif
+} MMWSLE_FREE_ENTRY, *PMMWSLE_FREE_ENTRY;
+
 typedef struct _MMWSLE
 {
     union
@@ -838,6 +853,7 @@ typedef struct _MMWSLE
         PVOID VirtualAddress;
         ULONG_PTR Long;
         MMWSLENTRY e1;
+        MMWSLE_FREE_ENTRY Free;
     } u1;
 } MMWSLE, *PMMWSLE;