- Reimplement the way zeroing PTEs are used:

- First, switch to using system PTEs as it should've been from the beginning. Our original implementation was broken and prone to race conditions, which Dmitry graciously fixed.
  - We can now remove the MiZeroPageInternal hack that was used as a way to avoid deadlock/contention in the zero paths.
  - Zeroing PTEs is done at DPC level in ReactOS, to avoid ReactOS-specific race issues. In Windows NT, this operation is always done at passive.
  - Zeroing PTEs are similar to hyperspace PTEs, but they can be mapped in chunks for optimization. 
    - ReactOS does not currently make use of this functionality, so zeroing is pretty slow, especially on bootup if you have lots of memory (all RAM is zeroed).
    - The existing ReactOS "compatibility layer" for hyperspace was augmented to seamlessly use the new zeroing PTE API.
  - You must now unmap zeroing PTEs -- MiZeroPage was modified to do this.
  - System PTE binning, NBQUEUES and SLISTS would optimize this further. TBD.
- Once again, tested on the trinity of supported emulators.


svn path=/trunk/; revision=41578

reactos/ntoskrnl/include/internal/mm.h

@@ -1155,7 +1155,13 @@ MiUnmapPageInHyperSpace(IN PEPROCESS Process,
 
 PVOID
 NTAPI
-MiMapPageToZeroInHyperSpace(IN PFN_NUMBER Page);
+MiMapPagesToZeroInHyperSpace(IN PMMPFN *Pages,
+                             IN PFN_NUMBER NumberOfPages);
+
+VOID
+NTAPI
+MiUnmapPagesInZeroSpace(IN PVOID VirtualAddress,
+                        IN PFN_NUMBER NumberOfPages);
 
 //
 // ReactOS Compatibility Layer
@@ -1168,6 +1174,14 @@ MmCreateHyperspaceMapping(IN PFN_NUMBER Page)
     return MiMapPageInHyperSpace(HyperProcess, Page, &HyperIrql);
 }
 
+PVOID
+FORCEINLINE
+MiMapPageToZeroInHyperSpace(IN PFN_NUMBER Page)
+{
+    PMMPFN Pfn1 = MiGetPfnEntry(Page);
+    return MiMapPagesToZeroInHyperSpace(&Pfn1, 1);
+}
+
 #define MmDeleteHyperspaceMapping(x) MiUnmapPageInHyperSpace(HyperProcess, x, HyperIrql);
 
 /* i386/page.c *********************************************************/

reactos/ntoskrnl/mm/ARM3/hypermap.c

@@ -109,42 +109,101 @@ MiUnmapPageInHyperSpace(IN PEPROCESS Process,
 
 PVOID
 NTAPI
-MiMapPageToZeroInHyperSpace(IN PFN_NUMBER Page)
+MiMapPagesToZeroInHyperSpace(IN PMMPFN *Pages,
+                             IN PFN_NUMBER NumberOfPages)
 {
     MMPTE TempPte;
     PMMPTE PointerPte;
-    PVOID Address; 
+    PFN_NUMBER Offset, PageFrameIndex;
+    PMMPFN Page;
 
     //
-    // Never accept page 0
+    // Sanity checks
     //
-    ASSERT(Page != 0);
+    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+    ASSERT(NumberOfPages != 0);
+    ASSERT(NumberOfPages <= (MI_ZERO_PTES - 1));
     
     //
-    // Build the PTE
+    // Pick the first zeroing PTE
     //
-    TempPte = HyperTemplatePte;
+    PointerPte = MiFirstReservedZeroingPte;
-    TempPte.u.Hard.PageFrameNumber = Page;
 
     //
-    // Get the Zero PTE and its address
+    // Now get the first free PTE
     //
-    PointerPte = MiAddressToPte(MI_ZERO_PTE);
+    Offset = PFN_FROM_PTE(PointerPte);
-    Address = (PVOID)((ULONG_PTR)PointerPte << 10);
+    if (NumberOfPages > Offset)
+    {
+        //
+        // Reset the PTEs
+        //
+        Offset = MI_ZERO_PTES - 1;
+        PointerPte->u.Hard.PageFrameNumber = Offset;
+        KeFlushProcessTb();
+    }
     
     //
-    // Invalidate the old address
+    // Prepare the next PTE
     //
-    __invlpg(Address);
+    PointerPte->u.Hard.PageFrameNumber = Offset - NumberOfPages;
    
     //
     // Write the current PTE
     //
-    TempPte.u.Hard.PageFrameNumber = Page;
+    PointerPte += (Offset + 1);
+    TempPte = HyperTemplatePte;
+    TempPte.u.Hard.Global = FALSE; // Hyperspace is local!
+    do
+    {
+        //
+        // Get the first page entry and its PFN
+        //
+        Page = *Pages++;
+        PageFrameIndex = MiGetPfnEntryIndex(Page);
+        
+        //
+        // Write the PFN
+        //
+        TempPte.u.Hard.PageFrameNumber = PageFrameIndex;
+        
+        //
+        // Set the correct PTE to write to, and set its new value
+        //
+        PointerPte--;
+        ASSERT(PointerPte->u.Hard.Valid == 0);
+        ASSERT(TempPte.u.Hard.Valid == 1);
         *PointerPte = TempPte;
+    } while (--NumberOfPages);
     
     //
     // Return the address
     //
-    return Address;
+    return MiPteToAddress(PointerPte);
 }
+
+VOID
+NTAPI
+MiUnmapPagesInZeroSpace(IN PVOID VirtualAddress,
+                        IN PFN_NUMBER NumberOfPages)
+{
+    PMMPTE PointerPte;
+    
+    //
+    // Sanity checks
+    //
+    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
+    ASSERT (NumberOfPages != 0);
+    ASSERT (NumberOfPages <= (MI_ZERO_PTES - 1));
+    
+    //
+    // Get the first PTE for the mapped zero VA
+    //
+    PointerPte = MiAddressToPte(VirtualAddress);
+
+    //
+    // Blow away the mapped zero PTEs
+    //
+    RtlZeroMemory(PointerPte, NumberOfPages * sizeof(MMPTE));
+}
+

reactos/ntoskrnl/mm/freelist.c

@@ -938,21 +938,6 @@ MmAllocPagesSpecifyRange(ULONG Consumer,
    return NumberOfPagesFound;
 }
 
-static
-NTSTATUS
-MiZeroPageInternal(PFN_TYPE Page)
-{
-   PVOID TempAddress;
-
-   TempAddress = MiMapPageToZeroInHyperSpace(Page);
-   if (TempAddress == NULL)
-   {
-      return(STATUS_NO_MEMORY);
-   }
-   memset(TempAddress, 0, PAGE_SIZE);
-   return(STATUS_SUCCESS);
-}
-
 NTSTATUS
 NTAPI
 MmZeroPageThreadMain(PVOID Ignored)
@@ -1000,7 +985,7 @@ MmZeroPageThreadMain(PVOID Ignored)
          PageDescriptor->Flags.Type = MM_PHYSICAL_PAGE_USED;
          KeReleaseQueuedSpinLock(LockQueuePfnLock, oldIrql);
          Pfn = PageDescriptor - MmPfnDatabase;
-         Status = MiZeroPageInternal(Pfn);
+         Status = MiZeroPage(Pfn);
 
          oldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock);
          if (PageDescriptor->MapCount != 0)

reactos/ntoskrnl/mm/kmap.c

@@ -21,18 +21,18 @@ NTSTATUS
 NTAPI
 MiZeroPage(PFN_TYPE Page)
 {
-   PEPROCESS Process;
    KIRQL Irql;
    PVOID TempAddress;
 
-   Process = PsGetCurrentProcess();
+   Irql = KeRaiseIrqlToDpcLevel();
-   TempAddress = MiMapPageInHyperSpace(Process, Page, &Irql);
+   TempAddress = MiMapPageToZeroInHyperSpace(Page);
    if (TempAddress == NULL)
    {
       return(STATUS_NO_MEMORY);
    }
    memset(TempAddress, 0, PAGE_SIZE);
-   MiUnmapPageInHyperSpace(Process, TempAddress, Irql);
+   MiUnmapPagesInZeroSpace(TempAddress, 1);
+   KeLowerIrql(Irql);
    return(STATUS_SUCCESS);
 }