Intravision/reactos
1
0
Fork 0
mirror of https://github.com/reactos/reactos.git synced 2024-11-18 13:01:40 +00:00
Code Issues Projects Releases Packages Wiki Activity
reactos/ntoskrnl/mm/freelist.c
Timo Kreuzer 33949028fb [NTOS:MM] Fix PFN lock usage
2021-06-19 12:17:18 +02:00

655 lines
16 KiB
C
Raw Blame History

/*
* COPYRIGHT: See COPYING in the top level directory
* PROJECT: ReactOS kernel
* FILE: ntoskrnl/mm/freelist.c
* PURPOSE: Handle the list of free physical pages
*
* PROGRAMMERS: David Welch (welch@cwcom.net)
* Robert Bergkvist
*/
/* INCLUDES ****************************************************************/
#include <ntoskrnl.h>
#define NDEBUG
#include <debug.h>
#define MODULE_INVOLVED_IN_ARM3
#include "ARM3/miarm.h"
#define ASSERT_IS_ROS_PFN(x) ASSERT(MI_IS_ROS_PFN(x) == TRUE);
/* GLOBALS ****************************************************************/
PMMPFN MmPfnDatabase;
PFN_NUMBER MmAvailablePages;
PFN_NUMBER MmResidentAvailablePages;
PFN_NUMBER MmResidentAvailableAtInit;
SIZE_T MmTotalCommittedPages;
SIZE_T MmSharedCommit;
SIZE_T MmDriverCommit;
SIZE_T MmProcessCommit;
SIZE_T MmPagedPoolCommit;
SIZE_T MmPeakCommitment;
SIZE_T MmtotalCommitLimitMaximum;
PMMPFN FirstUserLRUPfn;
PMMPFN LastUserLRUPfn;
/* FUNCTIONS *************************************************************/
PFN_NUMBER
NTAPI
MmGetLRUFirstUserPage(VOID)
{
PFN_NUMBER Page;
KIRQL OldIrql;
/* Find the first user page */
OldIrql = MiAcquirePfnLock();
if (FirstUserLRUPfn == NULL)
{
MiReleasePfnLock(OldIrql);
return 0;
}
Page = MiGetPfnEntryIndex(FirstUserLRUPfn);
MmReferencePage(Page);
MiReleasePfnLock(OldIrql);
return Page;
}
static
VOID
MmInsertLRULastUserPage(PFN_NUMBER Page)
{
MI_ASSERT_PFN_LOCK_HELD();
PMMPFN Pfn = MiGetPfnEntry(Page);
if (FirstUserLRUPfn == NULL)
FirstUserLRUPfn = Pfn;
Pfn->PreviousLRU = LastUserLRUPfn;
if (LastUserLRUPfn != NULL)
LastUserLRUPfn->NextLRU = Pfn;
LastUserLRUPfn = Pfn;
}
static
VOID
MmRemoveLRUUserPage(PFN_NUMBER Page)
{
MI_ASSERT_PFN_LOCK_HELD();
/* Unset the page as a user page */
ASSERT(Page != 0);
PMMPFN Pfn = MiGetPfnEntry(Page);
ASSERT_IS_ROS_PFN(Pfn);
if (Pfn->PreviousLRU)
{
ASSERT(Pfn->PreviousLRU->NextLRU == Pfn);
Pfn->PreviousLRU->NextLRU = Pfn->NextLRU;
}
else
{
ASSERT(FirstUserLRUPfn == Pfn);
FirstUserLRUPfn = Pfn->NextLRU;
}
if (Pfn->NextLRU)
{
ASSERT(Pfn->NextLRU->PreviousLRU == Pfn);
Pfn->NextLRU->PreviousLRU = Pfn->PreviousLRU;
}
else
{
ASSERT(Pfn == LastUserLRUPfn);
LastUserLRUPfn = Pfn->PreviousLRU;
}
Pfn->PreviousLRU = Pfn->NextLRU = NULL;
}
PFN_NUMBER
NTAPI
MmGetLRUNextUserPage(PFN_NUMBER PreviousPage, BOOLEAN MoveToLast)
{
PFN_NUMBER Page = 0;
KIRQL OldIrql;
/* Find the next user page */
OldIrql = MiAcquirePfnLock();
PMMPFN PreviousPfn = MiGetPfnEntry(PreviousPage);
PMMPFN NextPfn = PreviousPfn->NextLRU;
/*
* Move this one at the end of the list.
* It may be freed by MmDereferencePage below.
* If it's not, then it means it is still hanging in some process address space.
* This avoids paging-out e.g. ntdll early just because it's mapped first time.
*/
if ((MoveToLast) && (MmGetReferenceCountPage(PreviousPage) > 1))
{
MmRemoveLRUUserPage(PreviousPage);
MmInsertLRULastUserPage(PreviousPage);
}
if (NextPfn)
{
Page = MiGetPfnEntryIndex(NextPfn);
MmReferencePage(Page);
}
MmDereferencePage(PreviousPage);
MiReleasePfnLock(OldIrql);
return Page;
}
BOOLEAN
NTAPI
MiIsPfnFree(IN PMMPFN Pfn1)
{
/* Must be a free or zero page, with no references, linked */
return ((Pfn1->u3.e1.PageLocation <= StandbyPageList) &&
(Pfn1->u1.Flink) &&
(Pfn1->u2.Blink) &&
!(Pfn1->u3.e2.ReferenceCount));
}
BOOLEAN
NTAPI
MiIsPfnInUse(IN PMMPFN Pfn1)
{
/* Standby list or higher, unlinked, and with references */
return !MiIsPfnFree(Pfn1);
}
PMDL
NTAPI
MiAllocatePagesForMdl(IN PHYSICAL_ADDRESS LowAddress,
IN PHYSICAL_ADDRESS HighAddress,
IN PHYSICAL_ADDRESS SkipBytes,
IN SIZE_T TotalBytes,
IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute,
IN ULONG MdlFlags)
{
PMDL Mdl;
PFN_NUMBER PageCount, LowPage, HighPage, SkipPages, PagesFound = 0, Page;
PPFN_NUMBER MdlPage, LastMdlPage;
KIRQL OldIrql;
PMMPFN Pfn1;
INT LookForZeroedPages;
ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
DPRINT1("ARM3-DEBUG: Being called with %I64x %I64x %I64x %lx %d %lu\n", LowAddress, HighAddress, SkipBytes, TotalBytes, CacheAttribute, MdlFlags);
//
// Convert the low address into a PFN
//
LowPage = (PFN_NUMBER)(LowAddress.QuadPart >> PAGE_SHIFT);
//
// Convert, and normalize, the high address into a PFN
//
HighPage = (PFN_NUMBER)(HighAddress.QuadPart >> PAGE_SHIFT);
if (HighPage > MmHighestPhysicalPage) HighPage = MmHighestPhysicalPage;
//
// Validate skipbytes and convert them into pages
//
if (BYTE_OFFSET(SkipBytes.LowPart)) return NULL;
SkipPages = (PFN_NUMBER)(SkipBytes.QuadPart >> PAGE_SHIFT);
/* This isn't supported at all */
if (SkipPages) DPRINT1("WARNING: Caller requesting SkipBytes, MDL might be mismatched\n");
//
// Now compute the number of pages the MDL will cover
//
PageCount = (PFN_NUMBER)ADDRESS_AND_SIZE_TO_SPAN_PAGES(0, TotalBytes);
do
{
//
// Try creating an MDL for these many pages
//
Mdl = MmCreateMdl(NULL, NULL, PageCount << PAGE_SHIFT);
if (Mdl) break;
//
// This function is not required to return the amount of pages requested
// In fact, it can return as little as 1 page, and callers are supposed
// to deal with this scenario. So re-attempt the allocation with less
// pages than before, and see if it worked this time.
//
PageCount -= (PageCount >> 4);
} while (PageCount);
//
// Wow, not even a single page was around!
//
if (!Mdl) return NULL;
//
// This is where the page array starts....
//
MdlPage = (PPFN_NUMBER)(Mdl + 1);
//
// Lock the PFN database
//
OldIrql = MiAcquirePfnLock();
//
// Are we looking for any pages, without discriminating?
//
if ((LowPage == 0) && (HighPage == MmHighestPhysicalPage))
{
//
// Well then, let's go shopping
//
while (PagesFound < PageCount)
{
/* Grab a page */
MI_SET_USAGE(MI_USAGE_MDL);
MI_SET_PROCESS2("Kernel");
/* FIXME: This check should be smarter */
Page = 0;
if (MmAvailablePages != 0)
Page = MiRemoveAnyPage(0);
if (Page == 0)
{
/* This is not good... hopefully we have at least SOME pages */
ASSERT(PagesFound);
break;
}
/* Grab the page entry for it */
Pfn1 = MiGetPfnEntry(Page);
//
// Make sure it's really free
//
ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
/* Now setup the page and mark it */
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
MI_SET_PFN_DELETED(Pfn1);
Pfn1->u4.PteFrame = 0x1FFEDCB;
Pfn1->u3.e1.StartOfAllocation = 1;
Pfn1->u3.e1.EndOfAllocation = 1;
Pfn1->u4.VerifierAllocation = 0;
//
// Save it into the MDL
//
*MdlPage++ = MiGetPfnEntryIndex(Pfn1);
PagesFound++;
}
}
else
{
//
// You want specific range of pages. We'll do this in two runs
//
for (LookForZeroedPages = 1; LookForZeroedPages >= 0; LookForZeroedPages--)
{
//
// Scan the range you specified
//
for (Page = LowPage; Page < HighPage; Page++)
{
//
// Get the PFN entry for this page
//
Pfn1 = MiGetPfnEntry(Page);
ASSERT(Pfn1);
//
// Make sure it's free and if this is our first pass, zeroed
//
if (MiIsPfnInUse(Pfn1)) continue;
if ((Pfn1->u3.e1.PageLocation == ZeroedPageList) != LookForZeroedPages) continue;
/* Remove the page from the free or zero list */
ASSERT(Pfn1->u3.e1.ReadInProgress == 0);
MI_SET_USAGE(MI_USAGE_MDL);
MI_SET_PROCESS2("Kernel");
MiUnlinkFreeOrZeroedPage(Pfn1);
//
// Sanity checks
//
ASSERT(Pfn1->u3.e2.ReferenceCount == 0);
//
// Now setup the page and mark it
//
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u2.ShareCount = 1;
MI_SET_PFN_DELETED(Pfn1);
Pfn1->u4.PteFrame = 0x1FFEDCB;
Pfn1->u3.e1.StartOfAllocation = 1;
Pfn1->u3.e1.EndOfAllocation = 1;
Pfn1->u4.VerifierAllocation = 0;
//
// Save this page into the MDL
//
*MdlPage++ = Page;
if (++PagesFound == PageCount) break;
}
//
// If the first pass was enough, don't keep going, otherwise, go again
//
if (PagesFound == PageCount) break;
}
}
//
// Now release the PFN count
//
MiReleasePfnLock(OldIrql);
//
// We might've found less pages, but not more ;-)
//
if (PagesFound != PageCount) ASSERT(PagesFound < PageCount);
if (!PagesFound)
{
//
// If we didn' tfind any pages at all, fail
//
DPRINT1("NO MDL PAGES!\n");
ExFreePoolWithTag(Mdl, TAG_MDL);
return NULL;
}
//
// Write out how many pages we found
//
Mdl->ByteCount = (ULONG)(PagesFound << PAGE_SHIFT);
//
// Terminate the MDL array if there's certain missing pages
//
if (PagesFound != PageCount) *MdlPage = LIST_HEAD;
//
// Now go back and loop over all the MDL pages
//
MdlPage = (PPFN_NUMBER)(Mdl + 1);
LastMdlPage = MdlPage + PagesFound;
while (MdlPage < LastMdlPage)
{
//
// Check if we've reached the end
//
Page = *MdlPage++;
if (Page == LIST_HEAD) break;
//
// Get the PFN entry for the page and check if we should zero it out
//
Pfn1 = MiGetPfnEntry(Page);
ASSERT(Pfn1);
if (Pfn1->u3.e1.PageLocation != ZeroedPageList) MiZeroPhysicalPage(Page);
Pfn1->u3.e1.PageLocation = ActiveAndValid;
}
//
// We're done, mark the pages as locked
//
Mdl->Process = NULL;
Mdl->MdlFlags |= MDL_PAGES_LOCKED;
return Mdl;
}
VOID
NTAPI
MmSetRmapListHeadPage(PFN_NUMBER Pfn, PMM_RMAP_ENTRY ListHead)
{
PMMPFN Pfn1;
/* PFN database must be locked */
MI_ASSERT_PFN_LOCK_HELD();
Pfn1 = MiGetPfnEntry(Pfn);
ASSERT(Pfn1);
ASSERT_IS_ROS_PFN(Pfn1);
if (ListHead)
{
/* Should not be trying to insert an RMAP for a non-active page */
ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
/* Set the list head address */
Pfn1->RmapListHead = ListHead;
}
else
{
/* ReactOS semantics dictate the page is STILL active right now */
ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
/* In this case, the RMAP is actually being removed, so clear field */
Pfn1->RmapListHead = NULL;
/* ReactOS semantics will now release the page, which will make it free and enter a colored list */
}
}
PMM_RMAP_ENTRY
NTAPI
MmGetRmapListHeadPage(PFN_NUMBER Pfn)
{
PMMPFN Pfn1;
/* PFN database must be locked */
MI_ASSERT_PFN_LOCK_HELD();
/* Get the entry */
Pfn1 = MiGetPfnEntry(Pfn);
ASSERT(Pfn1);
if (!MI_IS_ROS_PFN(Pfn1))
{
return NULL;
}
/* Should not have an RMAP for a non-active page */
ASSERT(MiIsPfnInUse(Pfn1) == TRUE);
/* Get the list head */
return Pfn1->RmapListHead;
}
VOID
NTAPI
MmSetSavedSwapEntryPage(PFN_NUMBER Pfn, SWAPENTRY SwapEntry)
{
KIRQL oldIrql;
PMMPFN Pfn1;
Pfn1 = MiGetPfnEntry(Pfn);
ASSERT(Pfn1);
ASSERT_IS_ROS_PFN(Pfn1);
oldIrql = MiAcquirePfnLock();
Pfn1->u1.SwapEntry = SwapEntry;
MiReleasePfnLock(oldIrql);
}
SWAPENTRY
NTAPI
MmGetSavedSwapEntryPage(PFN_NUMBER Pfn)
{
SWAPENTRY SwapEntry;
KIRQL oldIrql;
PMMPFN Pfn1;
Pfn1 = MiGetPfnEntry(Pfn);
ASSERT(Pfn1);
ASSERT_IS_ROS_PFN(Pfn1);
oldIrql = MiAcquirePfnLock();
SwapEntry = Pfn1->u1.SwapEntry;
MiReleasePfnLock(oldIrql);
return(SwapEntry);
}
VOID
NTAPI
MmReferencePage(PFN_NUMBER Pfn)
{
PMMPFN Pfn1;
DPRINT("MmReferencePage(PysicalAddress %x)\n", Pfn << PAGE_SHIFT);
MI_ASSERT_PFN_LOCK_HELD();
ASSERT(Pfn != 0);
ASSERT(Pfn <= MmHighestPhysicalPage);
Pfn1 = MiGetPfnEntry(Pfn);
ASSERT(Pfn1);
ASSERT_IS_ROS_PFN(Pfn1);
ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
Pfn1->u3.e2.ReferenceCount++;
}
ULONG
NTAPI
MmGetReferenceCountPage(PFN_NUMBER Pfn)
{
KIRQL oldIrql;
ULONG RCount;
PMMPFN Pfn1;
DPRINT("MmGetReferenceCountPage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
oldIrql = MiAcquirePfnLock();
Pfn1 = MiGetPfnEntry(Pfn);
ASSERT(Pfn1);
ASSERT_IS_ROS_PFN(Pfn1);
RCount = Pfn1->u3.e2.ReferenceCount;
MiReleasePfnLock(oldIrql);
return(RCount);
}
BOOLEAN
NTAPI
MmIsPageInUse(PFN_NUMBER Pfn)
{
return MiIsPfnInUse(MiGetPfnEntry(Pfn));
}
VOID
NTAPI
MmDereferencePage(PFN_NUMBER Pfn)
{
PMMPFN Pfn1;
DPRINT("MmDereferencePage(PhysicalAddress %x)\n", Pfn << PAGE_SHIFT);
MI_ASSERT_PFN_LOCK_HELD();
Pfn1 = MiGetPfnEntry(Pfn);
ASSERT(Pfn1);
ASSERT_IS_ROS_PFN(Pfn1);
ASSERT(Pfn1->u3.e2.ReferenceCount != 0);
Pfn1->u3.e2.ReferenceCount--;
if (Pfn1->u3.e2.ReferenceCount == 0)
{
/* Apply LRU hack */
if (Pfn1->u4.MustBeCached)
{
MmRemoveLRUUserPage(Pfn);
Pfn1->u4.MustBeCached = 0;
}
/* Mark the page temporarily as valid, we're going to make it free soon */
Pfn1->u3.e1.PageLocation = ActiveAndValid;
/* It's not a ROS PFN anymore */
Pfn1->u4.AweAllocation = FALSE;
/* Bring it back into the free list */
DPRINT("Legacy free: %lx\n", Pfn);
MiInsertPageInFreeList(Pfn);
}
}
PFN_NUMBER
NTAPI
MmAllocPage(ULONG Type)
{
PFN_NUMBER PfnOffset;
PMMPFN Pfn1;
KIRQL OldIrql;
OldIrql = MiAcquirePfnLock();
#if MI_TRACE_PFNS
switch(Type)
{
case MC_SYSTEM:
MI_SET_USAGE(MI_USAGE_CACHE);
break;
case MC_USER:
MI_SET_USAGE(MI_USAGE_SECTION);
break;
default:
ASSERT(FALSE);
}
#endif
PfnOffset = MiRemoveZeroPage(MI_GET_NEXT_COLOR());
if (!PfnOffset)
{
KeBugCheck(NO_PAGES_AVAILABLE);
}
DPRINT("Legacy allocate: %lx\n", PfnOffset);
Pfn1 = MiGetPfnEntry(PfnOffset);
Pfn1->u3.e2.ReferenceCount = 1;
Pfn1->u3.e1.PageLocation = ActiveAndValid;
/* This marks the PFN as a ReactOS PFN */
Pfn1->u4.AweAllocation = TRUE;
/* Allocate the extra ReactOS Data and zero it out */
Pfn1->u1.SwapEntry = 0;
Pfn1->RmapListHead = NULL;
Pfn1->NextLRU = NULL;
Pfn1->PreviousLRU = NULL;
if (Type == MC_USER)
{
Pfn1->u4.MustBeCached = 1; /* HACK again */
MmInsertLRULastUserPage(PfnOffset);
}
MiReleasePfnLock(OldIrql);
return PfnOffset;
}
/* EOF */
Reference in a new issue View git blame Copy permalink