[NTOS:MM] Fix a bit the page-out/page-in logic

- Do not lock the section segment when we are serving a fault for a process private page.
 - Do not keep the process address space lock while writing to pagefile.
 - Do not wait for an event that might never be set.
This commit is contained in:
Jérôme Gardou 2021-02-05 15:04:07 +01:00 committed by Jérôme Gardou
parent c4232ae995
commit 36a92e6ea5
6 changed files with 88 additions and 115 deletions

View file

@ -17,16 +17,6 @@
#define SEC_CACHE (0x20000000) #define SEC_CACHE (0x20000000)
#define MiWaitForPageEvent(Process,Address) do { \
DPRINT("MiWaitForPageEvent %p:%p #\n", Process, Address); \
KeWaitForSingleObject(&MmWaitPageEvent, 0, KernelMode, FALSE, NULL); \
} while(0)
#define MiSetPageEvent(Process,Address) do { \
DPRINT("MiSetPageEvent %p:%p #\n",Process, (PVOID)(Address)); \
KeSetEvent(&MmWaitPageEvent, IO_NO_INCREMENT, FALSE); \
} while(0)
/* We store 8 bits of location with a page association */ /* We store 8 bits of location with a page association */
#define ENTRIES_PER_ELEMENT 256 #define ENTRIES_PER_ELEMENT 256

View file

@ -1376,7 +1376,8 @@ NTAPI
MmAccessFaultSectionView( MmAccessFaultSectionView(
PMMSUPPORT AddressSpace, PMMSUPPORT AddressSpace,
MEMORY_AREA* MemoryArea, MEMORY_AREA* MemoryArea,
PVOID Address PVOID Address,
BOOLEAN Locked
); );
VOID VOID

View file

@ -219,6 +219,8 @@ MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create, PKIRQ
PMMPDE PdeBase; PMMPDE PdeBase;
ULONG PdeOffset = MiGetPdeOffset(Address); ULONG PdeOffset = MiGetPdeOffset(Address);
ASSERT(!Create);
PdeBase = MiMapPageInHyperSpace(PsGetCurrentProcess(), PdeBase = MiMapPageInHyperSpace(PsGetCurrentProcess(),
PTE_TO_PFN(Process->Pcb.DirectoryTableBase[0]), PTE_TO_PFN(Process->Pcb.DirectoryTableBase[0]),
OldIrql); OldIrql);
@ -229,29 +231,8 @@ MmGetPageTableForProcess(PEPROCESS Process, PVOID Address, BOOLEAN Create, PKIRQ
PointerPde = PdeBase + PdeOffset; PointerPde = PdeBase + PdeOffset;
if (PointerPde->u.Hard.Valid == 0) if (PointerPde->u.Hard.Valid == 0)
{ {
KAPC_STATE ApcState; MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PdeBase, *OldIrql);
NTSTATUS Status; return NULL;
if (!Create)
{
MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PdeBase, *OldIrql);
return NULL;
}
KeStackAttachProcess(&Process->Pcb, &ApcState);
Status = MiDispatchFault(0x1,
MiAddressToPte(Address),
MiAddressToPde(Address),
NULL,
FALSE,
Process,
NULL,
NULL);
KeUnstackDetachProcess(&ApcState);
if (!NT_SUCCESS(Status))
return NULL;
} }
Pfn = PointerPde->u.Hard.PageFrameNumber; Pfn = PointerPde->u.Hard.PageFrameNumber;

View file

@ -77,7 +77,8 @@ MmpAccessFault(KPROCESSOR_MODE Mode,
case MEMORY_AREA_SECTION_VIEW: case MEMORY_AREA_SECTION_VIEW:
Status = MmAccessFaultSectionView(AddressSpace, Status = MmAccessFaultSectionView(AddressSpace,
MemoryArea, MemoryArea,
(PVOID)Address); (PVOID)Address,
!FromMdl);
break; break;
#ifdef NEWCC #ifdef NEWCC
case MEMORY_AREA_CACHE: case MEMORY_AREA_CACHE:
@ -169,7 +170,7 @@ MmNotPresentFault(KPROCESSOR_MODE Mode,
Status = MmNotPresentFaultSectionView(AddressSpace, Status = MmNotPresentFaultSectionView(AddressSpace,
MemoryArea, MemoryArea,
(PVOID)Address, (PVOID)Address,
FromMdl); !FromMdl);
break; break;
#ifdef NEWCC #ifdef NEWCC
case MEMORY_AREA_CACHE: case MEMORY_AREA_CACHE:

View file

@ -141,15 +141,13 @@ GetEntry:
/* The segment is being read or something. Give up */ /* The segment is being read or something. Give up */
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace); MmUnlockAddressSpace(AddressSpace);
if (Address < MmSystemRangeStart) ExReleaseRundownProtection(&Process->RundownProtect);
{ ObDereferenceObject(Process);
ExReleaseRundownProtection(&Process->RundownProtect);
ObDereferenceObject(Process);
}
return(STATUS_UNSUCCESSFUL); return(STATUS_UNSUCCESSFUL);
} }
/* Delete this virtual mapping in the process */ /* Delete this virtual mapping in the process */
MmDeleteRmap(Page, Process, Address);
MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage); MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage);
/* We checked this earlier */ /* We checked this earlier */
@ -162,6 +160,11 @@ GetEntry:
/* This page is private to the process */ /* This page is private to the process */
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
/* Attach to it, if needed */
ASSERT(PsGetCurrentProcess() == PsInitialSystemProcess);
if (Process != PsInitialSystemProcess)
KeAttachProcess(&Process->Pcb);
/* Check if we should write it back to the page file */ /* Check if we should write it back to the page file */
SwapEntry = MmGetSavedSwapEntryPage(Page); SwapEntry = MmGetSavedSwapEntryPage(Page);
@ -177,14 +180,14 @@ GetEntry:
/* We can't, so let this page in the Process VM */ /* We can't, so let this page in the Process VM */
MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1);
MmInsertRmap(Page, Process, Address);
MmSetDirtyPage(Process, Address); MmSetDirtyPage(Process, Address);
MmUnlockAddressSpace(AddressSpace); MmUnlockAddressSpace(AddressSpace);
if (Address < MmSystemRangeStart) if (Process != PsInitialSystemProcess)
{ KeDetachProcess();
ExReleaseRundownProtection(&Process->RundownProtect); ExReleaseRundownProtection(&Process->RundownProtect);
ObDereferenceObject(Process); ObDereferenceObject(Process);
}
return STATUS_UNSUCCESSFUL; return STATUS_UNSUCCESSFUL;
} }
@ -192,7 +195,18 @@ GetEntry:
if (Dirty) if (Dirty)
{ {
SWAPENTRY Dummy;
/* Put a wait entry into the process and unlock */
MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY);
MmUnlockAddressSpace(AddressSpace);
Status = MmWriteToSwapPage(SwapEntry, Page); Status = MmWriteToSwapPage(SwapEntry, Page);
MmLockAddressSpace(AddressSpace);
MmDeletePageFileMapping(Process, Address, &Dummy);
ASSERT(Dummy == MM_WAIT_ENTRY);
if (!NT_SUCCESS(Status)) if (!NT_SUCCESS(Status))
{ {
/* We failed at saving the content of this page. Keep it in */ /* We failed at saving the content of this page. Keep it in */
@ -206,9 +220,12 @@ GetEntry:
/* We can't, so let this page in the Process VM */ /* We can't, so let this page in the Process VM */
MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1);
MmInsertRmap(Page, Process, Address);
MmSetDirtyPage(Process, Address); MmSetDirtyPage(Process, Address);
MmUnlockAddressSpace(AddressSpace); MmUnlockAddressSpace(AddressSpace);
if (Process != PsInitialSystemProcess)
KeDetachProcess();
ExReleaseRundownProtection(&Process->RundownProtect); ExReleaseRundownProtection(&Process->RundownProtect);
ObDereferenceObject(Process); ObDereferenceObject(Process);
@ -223,10 +240,11 @@ GetEntry:
MmSetSavedSwapEntryPage(Page, 0); MmSetSavedSwapEntryPage(Page, 0);
} }
MmUnlockAddressSpace(AddressSpace);
/* We can finally let this page go */ /* We can finally let this page go */
MmDeleteRmap(Page, Process, Address);
MmUnlockAddressSpace(AddressSpace);
if (Process != PsInitialSystemProcess)
KeDetachProcess();
#if DBG #if DBG
OldIrql = MiAcquirePfnLock(); OldIrql = MiAcquirePfnLock();
ASSERT(MmGetRmapListHeadPage(Page) == NULL); ASSERT(MmGetRmapListHeadPage(Page) == NULL);
@ -240,9 +258,6 @@ GetEntry:
return STATUS_SUCCESS; return STATUS_SUCCESS;
} }
/* Delete this RMAP */
MmDeleteRmap(Page, Process, Address);
/* One less mapping referencing this segment */ /* One less mapping referencing this segment */
Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL); Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, TRUE, NULL);

View file

@ -1461,7 +1461,11 @@ MmAlterViewAttributes(PMMSUPPORT AddressSpace,
MmGetPageFileMapping(Process, Address, &SwapEntry); MmGetPageFileMapping(Process, Address, &SwapEntry);
if (SwapEntry != MM_WAIT_ENTRY) if (SwapEntry != MM_WAIT_ENTRY)
break; break;
MiWaitForPageEvent(Process, Address); MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace);
YieldProcessor();
MmLockAddressSpace(AddressSpace);
MmLockSectionSegment(Segment);
} }
while (TRUE); while (TRUE);
@ -1522,6 +1526,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace);
SWAPENTRY SwapEntry; SWAPENTRY SwapEntry;
ASSERT(Locked);
/* /*
* There is a window between taking the page fault and locking the * There is a window between taking the page fault and locking the
* address space when another thread could load the page so we check * address space when another thread could load the page so we check
@ -1577,39 +1583,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
return STATUS_GUARD_PAGE_VIOLATION; return STATUS_GUARD_PAGE_VIOLATION;
} }
/*
* Lock the segment
*/
MmLockSectionSegment(Segment);
Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
/*
* Check if this page needs to be mapped COW
*/
if ((Segment->WriteCopy) &&
(Region->Protect == PAGE_READWRITE ||
Region->Protect == PAGE_EXECUTE_READWRITE))
{
Attributes = Region->Protect == PAGE_READWRITE ? PAGE_READONLY : PAGE_EXECUTE_READ;
}
else
{
Attributes = Region->Protect;
}
/*
* Check if someone else is already handling this fault, if so wait
* for them
*/
if (Entry && MM_IS_WAIT_PTE(Entry))
{
MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace);
MiWaitForPageEvent(NULL, NULL);
MmLockAddressSpace(AddressSpace);
DPRINT("Address 0x%p\n", Address);
return STATUS_MM_RESTART_OPERATION;
}
HasSwapEntry = MmIsPageSwapEntry(Process, Address); HasSwapEntry = MmIsPageSwapEntry(Process, Address);
/* See if we should use a private page */ /* See if we should use a private page */
@ -1620,22 +1593,21 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
MmGetPageFileMapping(Process, Address, &SwapEntry); MmGetPageFileMapping(Process, Address, &SwapEntry);
if (SwapEntry == MM_WAIT_ENTRY) if (SwapEntry == MM_WAIT_ENTRY)
{ {
MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace); MmUnlockAddressSpace(AddressSpace);
MiWaitForPageEvent(NULL, NULL); YieldProcessor();
MmLockAddressSpace(AddressSpace); MmLockAddressSpace(AddressSpace);
return STATUS_MM_RESTART_OPERATION; return STATUS_MM_RESTART_OPERATION;
} }
/* /*
* Must be private page we have swapped out. * Must be private page we have swapped out.
*/ */
/* /*
* Sanity check * Sanity check
*/ */
MmDeletePageFileMapping(Process, Address, &SwapEntry); MmDeletePageFileMapping(Process, Address, &DummyEntry);
MmUnlockSectionSegment(Segment); ASSERT(DummyEntry == SwapEntry);
/* Tell everyone else we are serving the fault. */ /* Tell everyone else we are serving the fault. */
MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY); MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY);
@ -1650,18 +1622,17 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
KeBugCheck(MEMORY_MANAGEMENT); KeBugCheck(MEMORY_MANAGEMENT);
} }
if (HasSwapEntry) Status = MmReadFromSwapPage(SwapEntry, Page);
if (!NT_SUCCESS(Status))
{ {
Status = MmReadFromSwapPage(SwapEntry, Page); DPRINT1("MmReadFromSwapPage failed, status = %x\n", Status);
if (!NT_SUCCESS(Status)) KeBugCheck(MEMORY_MANAGEMENT);
{
DPRINT1("MmReadFromSwapPage failed, status = %x\n", Status);
KeBugCheck(MEMORY_MANAGEMENT);
}
} }
MmLockAddressSpace(AddressSpace); MmLockAddressSpace(AddressSpace);
MmDeletePageFileMapping(Process, PAddress, &DummyEntry); MmDeletePageFileMapping(Process, PAddress, &DummyEntry);
ASSERT(DummyEntry == MM_WAIT_ENTRY);
Status = MmCreateVirtualMapping(Process, Status = MmCreateVirtualMapping(Process,
PAddress, PAddress,
Region->Protect, Region->Protect,
@ -1677,8 +1648,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
/* /*
* Store the swap entry for later use. * Store the swap entry for later use.
*/ */
if (HasSwapEntry) MmSetSavedSwapEntryPage(Page, SwapEntry);
MmSetSavedSwapEntryPage(Page, SwapEntry);
/* /*
* Add the page to the process's working set * Add the page to the process's working set
@ -1687,11 +1657,15 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
/* /*
* Finish the operation * Finish the operation
*/ */
MiSetPageEvent(Process, Address);
DPRINT("Address 0x%p\n", Address); DPRINT("Address 0x%p\n", Address);
return STATUS_SUCCESS; return STATUS_SUCCESS;
} }
/*
* Lock the segment
*/
MmLockSectionSegment(Segment);
/* /*
* Satisfying a page fault on a map of /Device/PhysicalMemory is easy * Satisfying a page fault on a map of /Device/PhysicalMemory is easy
*/ */
@ -1717,16 +1691,29 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
/* /*
* Cleanup and release locks * Cleanup and release locks
*/ */
MiSetPageEvent(Process, Address);
DPRINT("Address 0x%p\n", Address); DPRINT("Address 0x%p\n", Address);
return STATUS_SUCCESS; return STATUS_SUCCESS;
} }
/*
* Check if this page needs to be mapped COW
*/
if ((Segment->WriteCopy) &&
(Region->Protect == PAGE_READWRITE ||
Region->Protect == PAGE_EXECUTE_READWRITE))
{
Attributes = Region->Protect == PAGE_READWRITE ? PAGE_READONLY : PAGE_EXECUTE_READ;
}
else
{
Attributes = Region->Protect;
}
/* /*
* Get the entry corresponding to the offset within the section * Get the entry corresponding to the offset within the section
*/ */
Entry = MmGetPageEntrySectionSegment(Segment, &Offset); Entry = MmGetPageEntrySectionSegment(Segment, &Offset);
if (Entry == 0) if (Entry == 0)
{ {
/* /*
@ -1752,7 +1739,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
if (Process) if (Process)
MmInsertRmap(Page, Process, Address); MmInsertRmap(Page, Process, Address);
MiSetPageEvent(Process, Address);
DPRINT("Address 0x%p\n", Address); DPRINT("Address 0x%p\n", Address);
return STATUS_SUCCESS; return STATUS_SUCCESS;
} }
@ -1792,7 +1778,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
{ {
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace); MmUnlockAddressSpace(AddressSpace);
MiWaitForPageEvent(NULL, NULL); YieldProcessor();
MmLockAddressSpace(AddressSpace); MmLockAddressSpace(AddressSpace);
return STATUS_MM_RESTART_OPERATION; return STATUS_MM_RESTART_OPERATION;
} }
@ -1800,6 +1786,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
/* /*
* Release all our locks and read in the page from disk * Release all our locks and read in the page from disk
*/ */
MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace); MmUnlockAddressSpace(AddressSpace);
@ -1826,7 +1813,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
* that has a pending page-in. * that has a pending page-in.
*/ */
Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset); Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset);
if (Entry != Entry1) if (Entry1 != MAKE_SWAP_SSE(MM_WAIT_ENTRY))
{ {
DPRINT1("Someone changed ppte entry while we slept (%x vs %x)\n", Entry, Entry1); DPRINT1("Someone changed ppte entry while we slept (%x vs %x)\n", Entry, Entry1);
KeBugCheck(MEMORY_MANAGEMENT); KeBugCheck(MEMORY_MANAGEMENT);
@ -1859,7 +1846,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
MmSetPageEntrySectionSegment(Segment, &Offset, Entry); MmSetPageEntrySectionSegment(Segment, &Offset, Entry);
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
MiSetPageEvent(Process, Address);
DPRINT("Address 0x%p\n", Address); DPRINT("Address 0x%p\n", Address);
return STATUS_SUCCESS; return STATUS_SUCCESS;
} }
@ -1886,7 +1872,6 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
MmSharePageEntrySectionSegment(Segment, &Offset); MmSharePageEntrySectionSegment(Segment, &Offset);
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
MiSetPageEvent(Process, Address);
DPRINT("Address 0x%p\n", Address); DPRINT("Address 0x%p\n", Address);
return STATUS_SUCCESS; return STATUS_SUCCESS;
} }
@ -1896,7 +1881,8 @@ NTSTATUS
NTAPI NTAPI
MmAccessFaultSectionView(PMMSUPPORT AddressSpace, MmAccessFaultSectionView(PMMSUPPORT AddressSpace,
MEMORY_AREA* MemoryArea, MEMORY_AREA* MemoryArea,
PVOID Address) PVOID Address,
BOOLEAN Locked)
{ {
PMM_SECTION_SEGMENT Segment; PMM_SECTION_SEGMENT Segment;
PFN_NUMBER OldPage; PFN_NUMBER OldPage;
@ -1911,7 +1897,7 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace,
DPRINT("MmAccessFaultSectionView(%p, %p, %p)\n", AddressSpace, MemoryArea, Address); DPRINT("MmAccessFaultSectionView(%p, %p, %p)\n", AddressSpace, MemoryArea, Address);
/* Make sure we have a page mapping for this address. */ /* Make sure we have a page mapping for this address. */
Status = MmNotPresentFaultSectionView(AddressSpace, MemoryArea, Address, TRUE); Status = MmNotPresentFaultSectionView(AddressSpace, MemoryArea, Address, Locked);
if (!NT_SUCCESS(Status)) if (!NT_SUCCESS(Status))
{ {
/* This is invalid access ! */ /* This is invalid access ! */
@ -2011,7 +1997,6 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace,
if (Process) if (Process)
MmInsertRmap(NewPage, Process, PAddress); MmInsertRmap(NewPage, Process, PAddress);
MiSetPageEvent(Process, Address);
DPRINT("Address 0x%p\n", Address); DPRINT("Address 0x%p\n", Address);
return STATUS_SUCCESS; return STATUS_SUCCESS;
} }
@ -3383,7 +3368,7 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address,
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace); MmUnlockAddressSpace(AddressSpace);
MiWaitForPageEvent(NULL, NULL); YieldProcessor();
MmLockAddressSpace(AddressSpace); MmLockAddressSpace(AddressSpace);
MmLockSectionSegment(Segment); MmLockSectionSegment(Segment);
@ -4600,7 +4585,7 @@ MmRosFlushVirtualMemory(
while (MM_IS_WAIT_PTE(Entry)) while (MM_IS_WAIT_PTE(Entry))
{ {
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
MiWaitForPageEvent(NULL, NULL); YieldProcessor();
MmLockSectionSegment(Segment); MmLockSectionSegment(Segment);
Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset);
} }
@ -5010,7 +4995,7 @@ MmMakePagesDirty(
{ {
MmUnlockSectionSegment(Segment); MmUnlockSectionSegment(Segment);
MmUnlockAddressSpace(AddressSpace); MmUnlockAddressSpace(AddressSpace);
MiWaitForPageEvent(NULL, NULL); YieldProcessor();
MmLockAddressSpace(AddressSpace); MmLockAddressSpace(AddressSpace);
MmLockSectionSegment(Segment); MmLockSectionSegment(Segment);
Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset);