[NTOS:MM][NTOS:CC] Performance improvement again

Read files by 64kb chunks instead of page-sized chunks.

ntoskrnl/cc/copy.c

@@ -504,24 +504,6 @@ CcCopyRead (
     CurrentOffset = FileOffset->QuadPart;
     while(CurrentOffset < ReadEnd)
     {
-        if (CurrentOffset >= SharedCacheMap->ValidDataLength.QuadPart)
-        {
-            DPRINT1("Zeroing buffer because we are beyond the VDL.\n");
-            /* We are beyond what is valid. Just zero this out */
-            _SEH2_TRY
-            {
-                RtlZeroMemory(Buffer, Length);
-            }
-            _SEH2_EXCEPT(CcpCheckInvalidUserBuffer(_SEH2_GetExceptionInformation(), Buffer, Length))
-            {
-                ExRaiseStatus(STATUS_INVALID_USER_BUFFER);
-            }
-            _SEH2_END;
-
-            ReadLength += Length;
-            break;
-        }
-
         Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb);
         if (!NT_SUCCESS(Status))
         {
@@ -538,11 +520,6 @@ CcCopyRead (
             if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength))
                 return FALSE;
 
-            /* Do not copy past the section */
-            if (CurrentOffset + VacbLength > SharedCacheMap->SectionSize.QuadPart)
-                CopyLength = SharedCacheMap->SectionSize.QuadPart - CurrentOffset;
-            if (CopyLength != 0)
-            {
             _SEH2_TRY
             {
                 RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength);
@@ -552,11 +529,6 @@ CcCopyRead (
                 ExRaiseStatus(STATUS_INVALID_USER_BUFFER);
             }
             _SEH2_END;
-            }
-
-            /* Zero-out the buffer tail if needed */
-            if (CopyLength < VacbLength)
-                RtlZeroMemory((PUCHAR)Buffer + CopyLength, VacbLength - CopyLength);
 
             ReadLength += VacbLength;
 
@@ -684,10 +656,6 @@ CcCopyWrite (
     if (FileObject->Flags & FO_WRITE_THROUGH)
         CcFlushCache(FileObject->SectionObjectPointer, FileOffset, Length, NULL);
 
-    /* Update VDL */
-    if (WriteEnd > SharedCacheMap->ValidDataLength.QuadPart)
-        SharedCacheMap->ValidDataLength.QuadPart = WriteEnd;
-
     return TRUE;
 }
 
@@ -898,7 +866,7 @@ CcZeroData (
     }
 
     /* See if we should simply truncate the valid data length */
-    if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart) && (EndOffset->QuadPart > SharedCacheMap->ValidDataLength.QuadPart))
+    if ((StartOffset->QuadPart < SharedCacheMap->ValidDataLength.QuadPart) && (EndOffset->QuadPart >= SharedCacheMap->ValidDataLength.QuadPart))
     {
         DPRINT1("Truncating VDL.\n");
         SharedCacheMap->ValidDataLength = *StartOffset;

ntoskrnl/cc/pin.c

@@ -543,7 +543,6 @@ CcSetDirtyPinnedData (
     IN PLARGE_INTEGER Lsn)
 {
     PINTERNAL_BCB iBcb = CONTAINING_RECORD(Bcb, INTERNAL_BCB, PFCB);
-    PROS_SHARED_CACHE_MAP SharedCacheMap = iBcb->Vacb->SharedCacheMap;
 
     CCTRACE(CC_API_DEBUG, "Bcb=%p Lsn=%p\n", Bcb, Lsn);
 
@@ -556,10 +555,6 @@ CcSetDirtyPinnedData (
     {
         CcRosMarkDirtyVacb(iBcb->Vacb);
     }
-
-    /* Update VDL */
-    if (SharedCacheMap->ValidDataLength.QuadPart < (iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength))
-        SharedCacheMap->ValidDataLength.QuadPart = iBcb->PFCB.MappedFileOffset.QuadPart + iBcb->PFCB.MappedLength;
 }
 
 

ntoskrnl/cc/view.c

@@ -166,12 +166,12 @@ MmFlushVirtualMemory(IN PEPROCESS Process,
 NTSTATUS
 NTAPI
 CcRosFlushVacb (
-    PROS_VACB Vacb)
+    _In_ PROS_VACB Vacb,
+    _In_ PIO_STATUS_BLOCK Iosb)
 {
-    IO_STATUS_BLOCK Iosb;
-    SIZE_T FlushSize = VACB_MAPPING_GRANULARITY;
     NTSTATUS Status;
     BOOLEAN HaveLock = FALSE;
+    PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap;
 
     CcRosUnmarkDirtyVacb(Vacb, TRUE);
 
@@ -184,7 +184,10 @@ CcRosFlushVacb (
         HaveLock = TRUE;
     }
 
-    Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb);
+    Status = MmFlushSegment(SharedCacheMap->FileObject->SectionObjectPointer,
+                            &Vacb->FileOffset,
+                            VACB_MAPPING_GRANULARITY,
+                            Iosb);
 
     if (HaveLock)
     {
@@ -194,6 +197,14 @@ CcRosFlushVacb (
 quit:
     if (!NT_SUCCESS(Status))
         CcRosMarkDirtyVacb(Vacb);
+    else
+    {
+        /* Update VDL */
+        if (SharedCacheMap->ValidDataLength.QuadPart < (Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY))
+        {
+            SharedCacheMap->ValidDataLength.QuadPart = Vacb->FileOffset.QuadPart + VACB_MAPPING_GRANULARITY;
+        }
+    }
 
     return Status;
 }
@@ -285,7 +296,8 @@ CcRosFlushDirtyPages (
             continue;
         }
 
-        Status = CcRosFlushVacb(current);
+        IO_STATUS_BLOCK Iosb;
+        Status = CcRosFlushVacb(current, &Iosb);
 
         SharedCacheMap->Callbacks->ReleaseFromLazyWrite(SharedCacheMap->LazyWriteContext);
 
@@ -308,7 +320,7 @@ CcRosFlushDirtyPages (
             ULONG PagesFreed;
 
             /* How many pages did we free? */
-            PagesFreed = VACB_MAPPING_GRANULARITY / PAGE_SIZE;
+            PagesFreed = Iosb.Information / PAGE_SIZE;
             (*Count) += PagesFreed;
 
             if (!Wait)
@@ -756,7 +768,11 @@ CcRosEnsureVacbResident(
 
         if (!NoRead)
         {
-            NTSTATUS Status = MmMakePagesResident(NULL, BaseAddress, Length);
+            PROS_SHARED_CACHE_MAP SharedCacheMap = Vacb->SharedCacheMap;
+            NTSTATUS Status = MmMakeDataSectionResident(SharedCacheMap->FileObject->SectionObjectPointer,
+                                                        Vacb->FileOffset.QuadPart + Offset,
+                                                        Length,
+                                                        &SharedCacheMap->ValidDataLength);
             if (!NT_SUCCESS(Status))
                 ExRaiseStatus(Status);
         }
@@ -935,7 +951,6 @@ CcFlushCache (
     }
 
     Status = STATUS_SUCCESS;
-
     if (IoStatus)
     {
         IoStatus->Information = 0;
@@ -953,9 +968,10 @@ CcFlushCache (
 
         if (vacb != NULL)
         {
+            IO_STATUS_BLOCK VacbIosb;
             if (vacb->Dirty)
             {
-                Status = CcRosFlushVacb(vacb);
+                Status = CcRosFlushVacb(vacb, &VacbIosb);
                 if (!NT_SUCCESS(Status))
                 {
                     goto quit;
@@ -966,7 +982,7 @@ CcFlushCache (
             CcRosReleaseVacb(SharedCacheMap, vacb, FALSE, FALSE);
 
             if (IoStatus)
-                IoStatus->Information += VACB_MAPPING_GRANULARITY;
+                IoStatus->Information += VacbIosb.Information;
         }
 
         if (!DirtyVacb)
@@ -992,6 +1008,10 @@ CcFlushCache (
 
             if (IoStatus)
                 IoStatus->Information += MmIosb.Information;
+
+            /* Update VDL */
+            if (SharedCacheMap->ValidDataLength.QuadPart < FlushEnd)
+                SharedCacheMap->ValidDataLength.QuadPart = FlushEnd;
         }
 
         if (!NT_SUCCESS(RtlLongLongAdd(FlushStart, VACB_MAPPING_GRANULARITY, &FlushStart)))

ntoskrnl/include/internal/cc.h

@@ -310,7 +310,7 @@ CcMdlWriteComplete2(
 
 NTSTATUS
 NTAPI
-CcRosFlushVacb(PROS_VACB Vacb);
+CcRosFlushVacb(PROS_VACB Vacb, PIO_STATUS_BLOCK Iosb);
 
 NTSTATUS
 NTAPI

ntoskrnl/include/internal/mm.h

@@ -1369,13 +1369,6 @@ MmArePagesResident(
     _In_ PVOID BaseAddress,
     _In_ ULONG Length);
 
-NTSTATUS
-NTAPI
-MmMakePagesResident(
-    _In_ PEPROCESS Process,
-    _In_ PVOID Address,
-    _In_ ULONG Length);
-
 NTSTATUS
 NTAPI
 MmMakePagesDirty(
@@ -1399,6 +1392,14 @@ MmFlushSegment(
     _In_ ULONG Length,
     _In_opt_ PIO_STATUS_BLOCK Iosb);
 
+NTSTATUS
+NTAPI
+MmMakeDataSectionResident(
+    _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer,
+    _In_ LONGLONG Offset,
+    _In_ ULONG Length,
+    _In_ PLARGE_INTEGER ValidDataLength);
+
 BOOLEAN
 NTAPI
 MmPurgeSegment(

ntoskrnl/mm/section.c

@@ -1181,61 +1181,172 @@ MiCopyFromUserPage(PFN_NUMBER DestPage, const VOID *SrcAddress)
     return(STATUS_SUCCESS);
 }
 
-#ifndef NEWCC
 static
 NTSTATUS
 NTAPI
-MiReadPage(PMEMORY_AREA MemoryArea,
+MmMakeSegmentResident(
-           LONGLONG SegOffset,
+    _In_ PMM_SECTION_SEGMENT Segment,
-           PPFN_NUMBER Page,
+    _In_ LONGLONG Offset,
-           BOOLEAN IgnoreSize)
+    _In_ ULONG Length,
-/*
+    _In_opt_ PLARGE_INTEGER ValidDataLength)
- * FUNCTION: Read a page for a section backed memory area.
- * PARAMETERS:
- *       MemoryArea - Memory area to read the page for.
- *       Offset - Offset of the page to read.
- *       Page - Variable that receives a page contains the read data.
- */
 {
+    /* Let's use a 64K granularity. */
+    LONGLONG RangeStart, RangeEnd;
     NTSTATUS Status;
-    IO_STATUS_BLOCK IoStatus;
+    PFILE_OBJECT FileObject = Segment->FileObject;
-    KEVENT Event;
-    UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)];
-    PMDL Mdl = (PMDL)MdlBase;
-    PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject;
-    LARGE_INTEGER FileOffset;
-    KIRQL OldIrql;
-    PFSRTL_ADVANCED_FCB_HEADER FcbHeader = FileObject->FsContext;
 
-    FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset;
+    /* Calculate our range, aligned on 64K if possible. */
-
+    Status = RtlLongLongAdd(Offset, Length, &RangeEnd);
-    DPRINT("Reading file at offset %08x:%08x\n", FileOffset.HighPart, FileOffset.LowPart);
+    ASSERT(NT_SUCCESS(Status));
-
-    Status = MmRequestPageMemoryConsumer(MC_USER, FALSE, Page);
     if (!NT_SUCCESS(Status))
         return Status;
 
-    if ((FileOffset.QuadPart > FcbHeader->ValidDataLength.QuadPart) && !IgnoreSize)
+    RangeStart = Offset - (Offset % _64K);
+    if (RangeEnd % _64K)
+        RangeEnd += _64K - (RangeEnd % _64K);
+
+    /* Clamp if needed */
+    if (!FlagOn(*Segment->Flags, MM_DATAFILE_SEGMENT))
     {
-        /* Quick path : data is not valid; return a zero-page */
+        if (RangeEnd > Segment->RawLength.QuadPart)
-        return STATUS_SUCCESS;
+            RangeEnd = Segment->RawLength.QuadPart;
     }
 
-    RtlZeroMemory(MdlBase, sizeof(MdlBase));
+    /* Let's gooooooooo */
-    MmInitializeMdl(Mdl, NULL, PAGE_SIZE);
+    for ( ; RangeStart < RangeEnd; RangeStart += _64K)
-    MmBuildMdlFromPages(Mdl, Page);
+    {
+        /* First take a look at where we miss pages */
+        ULONG ToReadPageBits = 0;
+        LONGLONG ChunkEnd = RangeStart + _64K;
+
+        if (ChunkEnd > RangeEnd)
+            ChunkEnd = RangeEnd;
+
+        MmLockSectionSegment(Segment);
+        for (LONGLONG ChunkOffset = RangeStart; ChunkOffset < ChunkEnd; ChunkOffset += PAGE_SIZE)
+        {
+            LARGE_INTEGER CurrentOffset;
+
+            CurrentOffset.QuadPart = ChunkOffset;
+            ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset);
+
+            /* Let any pending read proceed */
+            while (MM_IS_WAIT_PTE(Entry))
+            {
+                MmUnlockSectionSegment(Segment);
+
+                KeDelayExecutionThread(KernelMode, FALSE, &TinyTime);
+
+                MmLockSectionSegment(Segment);
+                Entry = MmGetPageEntrySectionSegment(Segment, &CurrentOffset);
+            }
+
+            if (Entry != 0)
+            {
+                /* There is a page here. Or a swap entry. Or whatever... */
+                continue;
+            }
+
+            ToReadPageBits |= 1UL << ((ChunkOffset - RangeStart) >> PAGE_SHIFT);
+
+            /* Put a wait entry here */
+            MmSetPageEntrySectionSegment(Segment, &CurrentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
+        }
+        MmUnlockSectionSegment(Segment);
+
+        if (ToReadPageBits == 0)
+        {
+            /* Nothing to do for this chunk */
+            continue;
+        }
+
+        /* Now perform the actual read */
+        LONGLONG ChunkOffset = RangeStart;
+        while (ChunkOffset < ChunkEnd)
+        {
+            /* Move forward if there is a hole */
+            ULONG BitSet;
+            if (!_BitScanForward(&BitSet, ToReadPageBits))
+            {
+                /* Nothing more to read */
+                break;
+            }
+            ToReadPageBits >>= BitSet;
+            ChunkOffset += BitSet * PAGE_SIZE;
+            ASSERT(ChunkOffset < ChunkEnd);
+
+            /* Get the range we have to read */
+            _BitScanForward(&BitSet, ~ToReadPageBits);
+            ULONG ReadLength = BitSet * PAGE_SIZE;
+
+            ASSERT(ReadLength <= _64K);
+
+            /* Clamp (This is for image mappings */
+            if ((ChunkOffset + ReadLength) > ChunkEnd)
+                ReadLength = ChunkEnd - ChunkOffset;
+
+            ASSERT(ReadLength != 0);
+
+            /* Allocate a MDL */
+            PMDL Mdl = IoAllocateMdl(NULL, ReadLength, FALSE, FALSE, NULL);
+            if (!Mdl)
+            {
+                /* Damn. Roll-back. */
+                MmLockSectionSegment(Segment);
+                while (ChunkOffset < ChunkEnd)
+                {
+                    if (ToReadPageBits & 1)
+                    {
+                        LARGE_INTEGER CurrentOffset;
+                        CurrentOffset.QuadPart = ChunkOffset;
+                        ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset)));
+                        MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0);
+                    }
+                    ToReadPageBits >>= 1;
+                    ChunkOffset += PAGE_SIZE;
+                }
+                MmUnlockSectionSegment(Segment);
+                return STATUS_INSUFFICIENT_RESOURCES;
+            }
+
+            /* Get our pages */
+            PPFN_NUMBER Pages = MmGetMdlPfnArray(Mdl);
+            RtlZeroMemory(Pages, BYTES_TO_PAGES(ReadLength) * sizeof(PFN_NUMBER));
+            for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++)
+            {
+                /* MmRequestPageMemoryConsumer succeeds or bugchecks */
+                (void)MmRequestPageMemoryConsumer(MC_USER, FALSE, &Pages[i]);
+            }
             Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ;
 
+            LARGE_INTEGER FileOffset;
+            FileOffset.QuadPart = Segment->Image.FileOffset + ChunkOffset;
+
+            /* Clamp to VDL */
+            if (ValidDataLength && ((FileOffset.QuadPart + ReadLength) > ValidDataLength->QuadPart))
+            {
+                if (FileOffset.QuadPart > ValidDataLength->QuadPart)
+                {
+                    /* Great, nothing to read. */
+                    goto AssignPagesToSegment;
+                }
+
+                Mdl->Size = (FileOffset.QuadPart + ReadLength) - ValidDataLength->QuadPart;
+            }
+
+            KEVENT Event;
             KeInitializeEvent(&Event, NotificationEvent, FALSE);
 
             /* Disable APCs */
+            KIRQL OldIrql;
             KeRaiseIrql(APC_LEVEL, &OldIrql);
 
-    Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &IoStatus);
+            IO_STATUS_BLOCK Iosb;
+            Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &Iosb);
             if (Status == STATUS_PENDING)
             {
                 KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL);
-        Status = IoStatus.Status;
+                Status = Iosb.Status;
             }
 
             if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA)
@@ -1247,60 +1358,57 @@ MiReadPage(PMEMORY_AREA MemoryArea,
 
             if (Status == STATUS_END_OF_FILE)
             {
-        DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName);
+                DPRINT1("Got STATUS_END_OF_FILE at offset %I64d for file %wZ.\n", FileOffset.QuadPart, &FileObject->FileName);
                 Status = STATUS_SUCCESS;
             }
 
-    if ((MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)
+            if (!NT_SUCCESS(Status))
-        && ((SegOffset + PAGE_SIZE) > MemoryArea->SectionData.Segment->RawLength.QuadPart))
             {
-        KIRQL OldIrql;
+                /* Damn. Roll back. */
-        PUCHAR PageMap;
+                for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++)
+                    MmReleasePageMemoryConsumer(MC_USER, Pages[i]);
 
-        DPRINT("Zeroing at offset %I64d for file %wZ.\n", SegOffset, &FileObject->FileName);
+                MmLockSectionSegment(Segment);
-
+                while (ChunkOffset < ChunkEnd)
-        /* Zero out the end of it */
+                {
-        PageMap = MiMapPageInHyperSpace(PsGetCurrentProcess(), *Page, &OldIrql);
+                    if (ToReadPageBits & 1)
-        RtlZeroMemory(PageMap + MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset,
+                    {
-                      PAGE_SIZE - (MemoryArea->SectionData.Segment->RawLength.QuadPart - SegOffset));
+                        LARGE_INTEGER CurrentOffset;
-        MiUnmapPageInHyperSpace(PsGetCurrentProcess(), PageMap, OldIrql);
+                        CurrentOffset.QuadPart = ChunkOffset;
+                        ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset)));
+                        MmSetPageEntrySectionSegment(Segment, &CurrentOffset, 0);
                     }
-
+                    ToReadPageBits >>= 1;
+                    ChunkOffset += PAGE_SIZE;
+                }
+                MmUnlockSectionSegment(Segment);
+                IoFreeMdl(Mdl);;
                 return Status;
             }
 
-#else
+AssignPagesToSegment:
-NTSTATUS
+            MmLockSectionSegment(Segment);
-NTAPI
+
-MiReadPage(PMEMORY_AREA MemoryArea,
+            for (UINT i = 0; i < BYTES_TO_PAGES(ReadLength); i++)
-           LONGLONG SegOffset,
-           PPFN_NUMBER Page)
-/*
- * FUNCTION: Read a page for a section backed memory area.
- * PARAMETERS:
- *       MemoryArea - Memory area to read the page for.
- *       Offset - Offset of the page to read.
- *       Page - Variable that receives a page contains the read data.
- */
             {
-    MM_REQUIRED_RESOURCES Resources;
+                LARGE_INTEGER CurrentOffset;
-    NTSTATUS Status;
+                CurrentOffset.QuadPart = ChunkOffset + (i * PAGE_SIZE);
 
-    RtlZeroMemory(&Resources, sizeof(MM_REQUIRED_RESOURCES));
+                ASSERT(MM_IS_WAIT_PTE(MmGetPageEntrySectionSegment(Segment, &CurrentOffset)));
 
-    Resources.Context = MemoryArea->SectionData.Section->FileObject;
+                MmSetPageEntrySectionSegment(Segment, &CurrentOffset, MAKE_SSE(Pages[i] << PAGE_SHIFT, 0));
-    Resources.FileOffset.QuadPart = SegOffset +
+            }
-                                    MemoryArea->SectionData.Segment->Image.FileOffset;
+
-    Resources.Consumer = MC_USER;
+            MmUnlockSectionSegment(Segment);
-    Resources.Amount = PAGE_SIZE;
+
-
+            IoFreeMdl(Mdl);
-    DPRINT("%S, offset 0x%x, len 0x%x, page 0x%x\n", ((PFILE_OBJECT)Resources.Context)->FileName.Buffer, Resources.FileOffset.LowPart, Resources.Amount, Resources.Page[0]);
+            ToReadPageBits >>= BitSet;
-
+            ChunkOffset += BitSet * PAGE_SIZE;
-    Status = MiReadFilePage(MmGetKernelAddressSpace(), MemoryArea, &Resources);
+        }
-    *Page = Resources.Page[0];
+    }
-    return Status;
+
+    return STATUS_SUCCESS;
 }
-#endif
 
 static VOID
 MmAlterViewAttributes(PMMSUPPORT AddressSpace,
@@ -1610,68 +1718,20 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
 
     if (Entry == 0)
     {
-        SWAPENTRY FakeSwapEntry;
-
         /*
-         * If the entry is zero (and it can't change because we have
+         * If the entry is zero, then we need to load the page.
-         * locked the segment) then we need to load the page.
          */
-
-        /*
-         * Release all our locks and read in the page from disk
-         */
-        MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
-        MmUnlockSectionSegment(Segment);
-        MmCreatePageFileMapping(Process, PAddress, MM_WAIT_ENTRY);
-        MmUnlockAddressSpace(AddressSpace);
-
         if ((Offset.QuadPart >= (LONGLONG)PAGE_ROUND_UP(Segment->RawLength.QuadPart)) && (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap))
         {
+            /* We are beyond the data which is on file. Just get a new page. */
             MI_SET_USAGE(MI_USAGE_SECTION);
             if (Process) MI_SET_PROCESS2(Process->ImageFileName);
             if (!Process) MI_SET_PROCESS2("Kernel Section");
-            Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, &Page);
+            MmRequestPageMemoryConsumer(MC_USER, FALSE, &Page);
-            if (!NT_SUCCESS(Status))
+            MmSetPageEntrySectionSegment(Segment, &Offset, MAKE_SSE(Page << PAGE_SHIFT, 1));
-            {
+            MmUnlockSectionSegment(Segment);
-                DPRINT1("MmRequestPageMemoryConsumer failed (Status %x)\n", Status);
-            }
 
-        }
+            Status = MmCreateVirtualMapping(Process, PAddress, Attributes, &Page, 1);
-        else
-        {
-            DPRINT("Getting fresh page for file %wZ at offset %I64d.\n", &Segment->FileObject->FileName, Offset.QuadPart);
-            Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page, FALSE);
-            if (!NT_SUCCESS(Status))
-            {
-                DPRINT1("MiReadPage failed (Status %x)\n", Status);
-            }
-        }
-        if (!NT_SUCCESS(Status))
-        {
-            /*
-             * FIXME: What do we know in this case?
-             */
-            /*
-             * Cleanup and release locks
-             */
-            MmLockAddressSpace(AddressSpace);
-            MiSetPageEvent(Process, Address);
-            DPRINT("Address 0x%p\n", Address);
-            return(Status);
-        }
-
-        /* Lock both segment and process address space while we proceed. */
-        MmLockAddressSpace(AddressSpace);
-        MmLockSectionSegment(Segment);
-
-        MmDeletePageFileMapping(Process, PAddress, &FakeSwapEntry);
-        DPRINT("CreateVirtualMapping Page %x Process %p PAddress %p Attributes %x\n",
-               Page, Process, PAddress, Attributes);
-        Status = MmCreateVirtualMapping(Process,
-                                        PAddress,
-                                        Attributes,
-                                        &Page,
-                                        1);
             if (!NT_SUCCESS(Status))
             {
                 DPRINT1("Unable to create virtual mapping\n");
@@ -1681,15 +1741,35 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace,
             if (Process)
                 MmInsertRmap(Page, Process, Address);
 
-        /* Set this section offset has being backed by our new page. */
-        Entry = MAKE_SSE(Page << PAGE_SHIFT, 1);
-        MmSetPageEntrySectionSegment(Segment, &Offset, Entry);
-        MmUnlockSectionSegment(Segment);
-
             MiSetPageEvent(Process, Address);
             DPRINT("Address 0x%p\n", Address);
             return(STATUS_SUCCESS);
         }
+
+        MmUnlockSectionSegment(Segment);
+        MmUnlockAddressSpace(AddressSpace);
+
+        /* The data must be paged in. Lock the file, so that the VDL doesn't get updated behind us. */
+        FsRtlAcquireFileExclusive(Segment->FileObject);
+
+        PFSRTL_COMMON_FCB_HEADER FcbHeader = Segment->FileObject->FsContext;
+
+        Status = MmMakeSegmentResident(Segment, Offset.QuadPart, PAGE_SIZE, &FcbHeader->ValidDataLength);
+
+        FsRtlReleaseFile(Segment->FileObject);
+
+        /* Lock address space again */
+        MmLockAddressSpace(AddressSpace);
+        if (!NT_SUCCESS(Status))
+        {
+            /* Damn */
+            DPRINT1("Failed to page data in!\n");
+            return STATUS_IN_PAGE_ERROR;
+        }
+
+        /* Everything went fine. Restart the operation */
+        return STATUS_MM_RESTART_OPERATION;
+    }
     else if (IS_SWAP_FROM_SSE(Entry))
     {
         SWAPENTRY SwapEntry;
@@ -4458,103 +4538,6 @@ MmArePagesResident(
     return Ret;
 }
 
-NTSTATUS
-NTAPI
-MmMakePagesResident(
-    _In_ PEPROCESS Process,
-    _In_ PVOID Address,
-    _In_ ULONG Length)
-{
-    PMEMORY_AREA MemoryArea;
-    PMM_SECTION_SEGMENT Segment;
-    LARGE_INTEGER SegmentOffset, RangeEnd;
-    PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace();
-
-    MmLockAddressSpace(AddressSpace);
-
-    MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address);
-    if (MemoryArea == NULL)
-    {
-        DPRINT1("Unable to find memory area at address %p.\n", Address);
-        MmUnlockAddressSpace(AddressSpace);
-        return STATUS_NOT_MAPPED_VIEW;
-    }
-
-    /* Only supported in old Mm for now */
-    ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW);
-    /* For file mappings */
-    ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap);
-
-    Segment = MemoryArea->SectionData.Segment;
-    MmLockSectionSegment(Segment);
-
-    SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea)
-            + MemoryArea->SectionData.ViewOffset.QuadPart;
-    RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea)
-            + MemoryArea->SectionData.ViewOffset.QuadPart;
-
-    DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart);
-
-    while (SegmentOffset.QuadPart < RangeEnd.QuadPart)
-    {
-        ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset);
-
-        /* Let any pending read proceed */
-        while (MM_IS_WAIT_PTE(Entry))
-        {
-            MmUnlockSectionSegment(Segment);
-            MmUnlockAddressSpace(AddressSpace);
-            MiWaitForPageEvent(NULL, NULL);
-            MmLockAddressSpace(AddressSpace);
-            MmLockSectionSegment(Segment);
-            Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset);
-        }
-
-        /* We are called from Cc, this can't be backed by the page files */
-        ASSERT(!IS_SWAP_FROM_SSE(Entry));
-
-        /* At this point, there may be a valid page there */
-        if (Entry == 0)
-        {
-            PFN_NUMBER Page;
-            NTSTATUS Status;
-
-            /*
-             * Release all our locks and read in the page from disk
-             */
-            MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY));
-            MmUnlockSectionSegment(Segment);
-            MmUnlockAddressSpace(AddressSpace);
-
-            /* FIXME: Read the whole range at once instead of one page at a time */
-            /* Ignore file size, as Cc already checked on its side. */
-            Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page, TRUE);
-            if (!NT_SUCCESS(Status))
-            {
-                /* Reset the Segment entry and fail */
-                MmLockSectionSegment(Segment);
-                MmSetPageEntrySectionSegment(Segment, &SegmentOffset, 0);
-                MmUnlockSectionSegment(Segment);
-                MiSetPageEvent(Process, Address);
-                return Status;
-            }
-
-            MmLockAddressSpace(AddressSpace);
-            MmLockSectionSegment(Segment);
-
-            /* We set it with 0 ref count, nobody maps this page yet. */
-            MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 0));
-            MiSetPageEvent(Process, Address);
-        }
-        SegmentOffset.QuadPart += PAGE_SIZE;
-    }
-
-    MmUnlockSectionSegment(Segment);
-
-    MmUnlockAddressSpace(AddressSpace);
-    return STATUS_SUCCESS;
-}
-
 NTSTATUS
 NTAPI
 MmRosFlushVirtualMemory(
@@ -4723,6 +4706,26 @@ MmPurgeSegment(
     return TRUE;
 }
 
+NTSTATUS
+NTAPI
+MmMakeDataSectionResident(
+    _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer,
+    _In_ LONGLONG Offset,
+    _In_ ULONG Length,
+    _In_ PLARGE_INTEGER ValidDataLength)
+{
+    PMM_SECTION_SEGMENT Segment = MiGrabDataSection(SectionObjectPointer);
+
+    /* There must be a segment for this call */
+    ASSERT(Segment);
+
+    NTSTATUS Status = MmMakeSegmentResident(Segment, Offset, Length, ValidDataLength);
+
+    MmDereferenceSegment(Segment);
+
+    return Status;
+}
+
 NTSTATUS
 NTAPI
 MmFlushSegment(