/* * PROJECT: ReactOS Kernel * LICENSE: BSD - See COPYING.ARM in the top level directory * FILE: ntoskrnl/mm/ARM3/miarm.h * PURPOSE: ARM Memory Manager Header * PROGRAMMERS: ReactOS Portable Systems Group */ #pragma once #define MI_LOWEST_VAD_ADDRESS (PVOID)MM_LOWEST_USER_ADDRESS /* Make the code cleaner with some definitions for size multiples */ #define _1KB (1024u) #define _1MB (1024 * _1KB) #define _1GB (1024 * _1MB) /* Everyone loves 64K */ #define _64K (64 * _1KB) /* Area mapped by a PDE */ #define PDE_MAPPED_VA (PTE_COUNT * PAGE_SIZE) /* Size of a page table */ #define PT_SIZE (PTE_COUNT * sizeof(MMPTE)) /* Size of a page directory */ #define PD_SIZE (PDE_COUNT * sizeof(MMPDE)) /* Stop using these! */ #define PD_COUNT PPE_PER_PAGE #define PDE_COUNT PDE_PER_PAGE #define PTE_COUNT PTE_PER_PAGE /* Size of all page directories for a process */ #define SYSTEM_PD_SIZE (PD_COUNT * PD_SIZE) #ifdef _M_IX86 C_ASSERT(SYSTEM_PD_SIZE == PAGE_SIZE); #endif // // Protection Bits part of the internal memory manager Protection Mask, from: // http://reactos.org/wiki/Techwiki:Memory_management_in_the_Windows_XP_kernel // https://www.reactos.org/wiki/Techwiki:Memory_Protection_constants // and public assertions. // #define MM_ZERO_ACCESS 0 #define MM_READONLY 1 #define MM_EXECUTE 2 #define MM_EXECUTE_READ 3 #define MM_READWRITE 4 #define MM_WRITECOPY 5 #define MM_EXECUTE_READWRITE 6 #define MM_EXECUTE_WRITECOPY 7 #define MM_PROTECT_ACCESS 7 // // These are flags on top of the actual protection mask // #define MM_NOCACHE 0x08 #define MM_GUARDPAGE 0x10 #define MM_WRITECOMBINE 0x18 #define MM_PROTECT_SPECIAL 0x18 // // These are special cases // #define MM_DECOMMIT (MM_ZERO_ACCESS | MM_GUARDPAGE) #define MM_NOACCESS (MM_ZERO_ACCESS | MM_WRITECOMBINE) #define MM_OUTSWAPPED_KSTACK (MM_EXECUTE_WRITECOPY | MM_WRITECOMBINE) #define MM_INVALID_PROTECTION 0xFFFFFFFF // // Specific PTE Definitions that map to the Memory Manager's Protection Mask Bits // The Memory Manager's definition define the attributes that must be preserved // and these PTE definitions describe the attributes in the hardware sense. This // helps deal with hardware differences between the actual boolean expression of // the argument. // // For example, in the logical attributes, we want to express read-only as a flag // but on x86, it is writability that must be set. On the other hand, on x86, just // like in the kernel, it is disabling the caches that requires a special flag, // while on certain architectures such as ARM, it is enabling the cache which // requires a flag. // #if defined(_M_IX86) // // Access Flags // #define PTE_READONLY 0 // Doesn't exist on x86 #define PTE_EXECUTE 0 // Not worrying about NX yet #define PTE_EXECUTE_READ 0 // Not worrying about NX yet #define PTE_READWRITE 0x2 #define PTE_WRITECOPY 0x200 #define PTE_EXECUTE_READWRITE 0x2 // Not worrying about NX yet #define PTE_EXECUTE_WRITECOPY 0x200 #define PTE_PROTOTYPE 0x400 // // State Flags // #define PTE_VALID 0x1 #define PTE_ACCESSED 0x20 #define PTE_DIRTY 0x40 // // Cache flags // #define PTE_ENABLE_CACHE 0 #define PTE_DISABLE_CACHE 0x10 #define PTE_WRITECOMBINED_CACHE 0x10 #define PTE_PROTECT_MASK 0x612 #elif defined(_M_AMD64) // // Access Flags // #define PTE_READONLY 0x8000000000000000ULL #define PTE_EXECUTE 0x0000000000000000ULL #define PTE_EXECUTE_READ PTE_EXECUTE /* EXECUTE implies READ on x64 */ #define PTE_READWRITE 0x8000000000000002ULL #define PTE_WRITECOPY 0x8000000000000200ULL #define PTE_EXECUTE_READWRITE 0x0000000000000002ULL #define PTE_EXECUTE_WRITECOPY 0x0000000000000200ULL #define PTE_PROTOTYPE 0x0000000000000400ULL // // State Flags // #define PTE_VALID 0x0000000000000001ULL #define PTE_ACCESSED 0x0000000000000020ULL #define PTE_DIRTY 0x0000000000000040ULL // // Cache flags // #define PTE_ENABLE_CACHE 0x0000000000000000ULL #define PTE_DISABLE_CACHE 0x0000000000000010ULL #define PTE_WRITECOMBINED_CACHE 0x0000000000000010ULL #define PTE_PROTECT_MASK 0x8000000000000612ULL #elif defined(_M_ARM) #define PTE_READONLY 0x200 #define PTE_EXECUTE 0 // Not worrying about NX yet #define PTE_EXECUTE_READ 0 // Not worrying about NX yet #define PTE_READWRITE 0 // Doesn't exist on ARM #define PTE_WRITECOPY 0 // Doesn't exist on ARM #define PTE_EXECUTE_READWRITE 0 // Not worrying about NX yet #define PTE_EXECUTE_WRITECOPY 0 // Not worrying about NX yet #define PTE_PROTOTYPE 0x400 // Using the Shared bit // // Cache flags // #define PTE_ENABLE_CACHE 0 #define PTE_DISABLE_CACHE 0x10 #define PTE_WRITECOMBINED_CACHE 0x10 #define PTE_PROTECT_MASK 0x610 #else #error Define these please! #endif // // Mask for image section page protection // #define IMAGE_SCN_PROTECTION_MASK (IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_EXECUTE) extern const ULONG_PTR MmProtectToPteMask[32]; extern const ULONG MmProtectToValue[32]; // // Assertions for session images, addresses, and PTEs // #define MI_IS_SESSION_IMAGE_ADDRESS(Address) \ (((Address) >= MiSessionImageStart) && ((Address) < MiSessionImageEnd)) #define MI_IS_SESSION_ADDRESS(Address) \ (((Address) >= MmSessionBase) && ((Address) < MiSessionSpaceEnd)) #define MI_IS_SESSION_PTE(Pte) \ ((((PMMPTE)Pte) >= MiSessionBasePte) && (((PMMPTE)Pte) < MiSessionLastPte)) #define MI_IS_PAGE_TABLE_ADDRESS(Address) \ (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)PTE_TOP)) #define MI_IS_SYSTEM_PAGE_TABLE_ADDRESS(Address) \ (((Address) >= (PVOID)MiAddressToPte(MmSystemRangeStart)) && ((Address) <= (PVOID)PTE_TOP)) #define MI_IS_PAGE_TABLE_OR_HYPER_ADDRESS(Address) \ (((PVOID)(Address) >= (PVOID)PTE_BASE) && ((PVOID)(Address) <= (PVOID)MmHyperSpaceEnd)) // // Creates a software PTE with the given protection // #define MI_MAKE_SOFTWARE_PTE(p, x) ((p)->u.Long = (x << MM_PTE_SOFTWARE_PROTECTION_BITS)) // // Marks a PTE as deleted // #define MI_SET_PFN_DELETED(x) ((x)->PteAddress = (PMMPTE)((ULONG_PTR)(x)->PteAddress | 1)) #define MI_IS_PFN_DELETED(x) ((ULONG_PTR)((x)->PteAddress) & 1) // // Special values for LoadedImports // #ifdef _WIN64 #define MM_SYSLDR_NO_IMPORTS (PVOID)0xFFFFFFFFFFFFFFFEULL #define MM_SYSLDR_BOOT_LOADED (PVOID)0xFFFFFFFFFFFFFFFFULL #else #define MM_SYSLDR_NO_IMPORTS (PVOID)0xFFFFFFFE #define MM_SYSLDR_BOOT_LOADED (PVOID)0xFFFFFFFF #endif #define MM_SYSLDR_SINGLE_ENTRY 0x1 // // Number of initial session IDs // #define MI_INITIAL_SESSION_IDS 64 #if defined(_M_IX86) || defined(_M_ARM) // // PFN List Sentinel // #define LIST_HEAD 0xFFFFFFFF // // Because GCC cannot automatically downcast 0xFFFFFFFF to lesser-width bits, // we need a manual definition suited to the number of bits in the PteFrame. // This is used as a LIST_HEAD for the colored list // #define COLORED_LIST_HEAD ((1 << 25) - 1) // 0x1FFFFFF #elif defined(_M_AMD64) #define LIST_HEAD 0xFFFFFFFFFFFFFFFFLL #define COLORED_LIST_HEAD ((1ULL << 57) - 1) // 0x1FFFFFFFFFFFFFFLL #else #error Define these please! #endif // // Special IRQL value (found in assertions) // #define MM_NOIRQL (KIRQL)0xFFFFFFFF // // Returns the color of a page // #define MI_GET_PAGE_COLOR(x) ((x) & MmSecondaryColorMask) #define MI_GET_NEXT_COLOR() (MI_GET_PAGE_COLOR(++MmSystemPageColor)) #define MI_GET_NEXT_PROCESS_COLOR(x) (MI_GET_PAGE_COLOR(++(x)->NextPageColor)) // // Prototype PTEs that don't yet have a pagefile association // #ifdef _WIN64 #define MI_PTE_LOOKUP_NEEDED 0xffffffffULL #else #define MI_PTE_LOOKUP_NEEDED 0xFFFFF #endif // // Number of session data and tag pages // #define MI_SESSION_DATA_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE) #define MI_SESSION_TAG_PAGES_MAXIMUM (MM_ALLOCATION_GRANULARITY / PAGE_SIZE) // // Used by MiCheckSecuredVad // #define MM_READ_WRITE_ALLOWED 11 #define MM_READ_ONLY_ALLOWED 10 #define MM_NO_ACCESS_ALLOWED 01 #define MM_DELETE_CHECK 85 // // System views are binned into 64K chunks // #define MI_SYSTEM_VIEW_BUCKET_SIZE _64K // // FIXFIX: These should go in ex.h after the pool merge // #ifdef _WIN64 #define POOL_BLOCK_SIZE 16 #else #define POOL_BLOCK_SIZE 8 #endif #define POOL_LISTS_PER_PAGE (PAGE_SIZE / POOL_BLOCK_SIZE) #define BASE_POOL_TYPE_MASK 1 #define POOL_MAX_ALLOC (PAGE_SIZE - (sizeof(POOL_HEADER) + POOL_BLOCK_SIZE)) // // Pool debugging/analysis/tracing flags // #define POOL_FLAG_CHECK_TIMERS 0x1 #define POOL_FLAG_CHECK_WORKERS 0x2 #define POOL_FLAG_CHECK_RESOURCES 0x4 #define POOL_FLAG_VERIFIER 0x8 #define POOL_FLAG_CHECK_DEADLOCK 0x10 #define POOL_FLAG_SPECIAL_POOL 0x20 #define POOL_FLAG_DBGPRINT_ON_FAILURE 0x40 #define POOL_FLAG_CRASH_ON_FAILURE 0x80 // // BAD_POOL_HEADER codes during pool bugcheck // #define POOL_CORRUPTED_LIST 3 #define POOL_SIZE_OR_INDEX_MISMATCH 5 #define POOL_ENTRIES_NOT_ALIGNED_PREVIOUS 6 #define POOL_HEADER_NOT_ALIGNED 7 #define POOL_HEADER_IS_ZERO 8 #define POOL_ENTRIES_NOT_ALIGNED_NEXT 9 #define POOL_ENTRY_NOT_FOUND 10 // // BAD_POOL_CALLER codes during pool bugcheck // #define POOL_ENTRY_CORRUPTED 1 #define POOL_ENTRY_ALREADY_FREE 6 #define POOL_ENTRY_NOT_ALLOCATED 7 #define POOL_ALLOC_IRQL_INVALID 8 #define POOL_FREE_IRQL_INVALID 9 #define POOL_BILLED_PROCESS_INVALID 13 #define POOL_HEADER_SIZE_INVALID 32 typedef struct _POOL_DESCRIPTOR { POOL_TYPE PoolType; ULONG PoolIndex; ULONG RunningAllocs; ULONG RunningDeAllocs; ULONG TotalPages; ULONG TotalBigPages; ULONG Threshold; PVOID LockAddress; PVOID PendingFrees; LONG PendingFreeDepth; SIZE_T TotalBytes; SIZE_T Spare0; LIST_ENTRY ListHeads[POOL_LISTS_PER_PAGE]; } POOL_DESCRIPTOR, *PPOOL_DESCRIPTOR; typedef struct _POOL_HEADER { union { struct { #ifdef _WIN64 USHORT PreviousSize:8; USHORT PoolIndex:8; USHORT BlockSize:8; USHORT PoolType:8; #else USHORT PreviousSize:9; USHORT PoolIndex:7; USHORT BlockSize:9; USHORT PoolType:7; #endif }; ULONG Ulong1; }; #ifdef _WIN64 ULONG PoolTag; #endif union { #ifdef _WIN64 PEPROCESS ProcessBilled; #else ULONG PoolTag; #endif struct { USHORT AllocatorBackTraceIndex; USHORT PoolTagHash; }; }; } POOL_HEADER, *PPOOL_HEADER; C_ASSERT(sizeof(POOL_HEADER) == POOL_BLOCK_SIZE); C_ASSERT(POOL_BLOCK_SIZE == sizeof(LIST_ENTRY)); typedef struct _POOL_TRACKER_TABLE { ULONG Key; LONG NonPagedAllocs; LONG NonPagedFrees; SIZE_T NonPagedBytes; LONG PagedAllocs; LONG PagedFrees; SIZE_T PagedBytes; } POOL_TRACKER_TABLE, *PPOOL_TRACKER_TABLE; typedef struct _POOL_TRACKER_BIG_PAGES { PVOID Va; ULONG Key; ULONG NumberOfPages; PVOID QuotaObject; } POOL_TRACKER_BIG_PAGES, *PPOOL_TRACKER_BIG_PAGES; extern ULONG ExpNumberOfPagedPools; extern POOL_DESCRIPTOR NonPagedPoolDescriptor; extern PPOOL_DESCRIPTOR ExpPagedPoolDescriptor[16 + 1]; extern PPOOL_TRACKER_TABLE PoolTrackTable; // // END FIXFIX // typedef struct _MI_LARGE_PAGE_DRIVER_ENTRY { LIST_ENTRY Links; UNICODE_STRING BaseName; } MI_LARGE_PAGE_DRIVER_ENTRY, *PMI_LARGE_PAGE_DRIVER_ENTRY; typedef enum _MMSYSTEM_PTE_POOL_TYPE { SystemPteSpace, NonPagedPoolExpansion, MaximumPtePoolTypes } MMSYSTEM_PTE_POOL_TYPE; typedef enum _MI_PFN_CACHE_ATTRIBUTE { MiNonCached, MiCached, MiWriteCombined, MiNotMapped } MI_PFN_CACHE_ATTRIBUTE, *PMI_PFN_CACHE_ATTRIBUTE; typedef struct _PHYSICAL_MEMORY_RUN { PFN_NUMBER BasePage; PFN_NUMBER PageCount; } PHYSICAL_MEMORY_RUN, *PPHYSICAL_MEMORY_RUN; typedef struct _PHYSICAL_MEMORY_DESCRIPTOR { ULONG NumberOfRuns; PFN_NUMBER NumberOfPages; PHYSICAL_MEMORY_RUN Run[1]; } PHYSICAL_MEMORY_DESCRIPTOR, *PPHYSICAL_MEMORY_DESCRIPTOR; typedef struct _MMCOLOR_TABLES { PFN_NUMBER Flink; PVOID Blink; PFN_NUMBER Count; } MMCOLOR_TABLES, *PMMCOLOR_TABLES; typedef struct _MI_LARGE_PAGE_RANGES { PFN_NUMBER StartFrame; PFN_NUMBER LastFrame; } MI_LARGE_PAGE_RANGES, *PMI_LARGE_PAGE_RANGES; typedef struct _MMVIEW { ULONG_PTR Entry; PCONTROL_AREA ControlArea; } MMVIEW, *PMMVIEW; typedef struct _MMSESSION { KGUARDED_MUTEX SystemSpaceViewLock; PKGUARDED_MUTEX SystemSpaceViewLockPointer; PCHAR SystemSpaceViewStart; PMMVIEW SystemSpaceViewTable; ULONG SystemSpaceHashSize; ULONG SystemSpaceHashEntries; ULONG SystemSpaceHashKey; ULONG BitmapFailures; PRTL_BITMAP SystemSpaceBitMap; } MMSESSION, *PMMSESSION; typedef struct _MM_SESSION_SPACE_FLAGS { ULONG Initialized:1; ULONG DeletePending:1; ULONG Filler:30; } MM_SESSION_SPACE_FLAGS; typedef struct _MM_SESSION_SPACE { struct _MM_SESSION_SPACE *GlobalVirtualAddress; LONG ReferenceCount; union { ULONG LongFlags; MM_SESSION_SPACE_FLAGS Flags; } u; ULONG SessionId; LIST_ENTRY ProcessList; LARGE_INTEGER LastProcessSwappedOutTime; PFN_NUMBER SessionPageDirectoryIndex; SIZE_T NonPageablePages; SIZE_T CommittedPages; PVOID PagedPoolStart; PVOID PagedPoolEnd; PMMPDE PagedPoolBasePde; ULONG Color; LONG ResidentProcessCount; ULONG SessionPoolAllocationFailures[4]; LIST_ENTRY ImageList; LCID LocaleId; ULONG AttachCount; KEVENT AttachEvent; PEPROCESS LastProcess; LONG ProcessReferenceToSession; LIST_ENTRY WsListEntry; GENERAL_LOOKASIDE Lookaside[SESSION_POOL_LOOKASIDES]; MMSESSION Session; KGUARDED_MUTEX PagedPoolMutex; MM_PAGED_POOL_INFO PagedPoolInfo; MMSUPPORT Vm; PMMWSLE Wsle; PDRIVER_UNLOAD Win32KDriverUnload; POOL_DESCRIPTOR PagedPool; #if defined (_M_AMD64) MMPDE PageDirectory; #else PMMPDE PageTables; #endif #if defined (_M_AMD64) PMMPTE SpecialPoolFirstPte; PMMPTE SpecialPoolLastPte; PMMPTE NextPdeForSpecialPoolExpansion; PMMPTE LastPdeForSpecialPoolExpansion; PFN_NUMBER SpecialPagesInUse; #endif LONG ImageLoadingCount; } MM_SESSION_SPACE, *PMM_SESSION_SPACE; extern PMM_SESSION_SPACE MmSessionSpace; extern MMPTE HyperTemplatePte; extern MMPDE ValidKernelPde; extern MMPTE ValidKernelPte; extern MMPDE ValidKernelPdeLocal; extern MMPTE ValidKernelPteLocal; extern MMPDE DemandZeroPde; extern MMPTE DemandZeroPte; extern MMPTE PrototypePte; extern MMPTE MmDecommittedPte; extern BOOLEAN MmLargeSystemCache; extern BOOLEAN MmZeroPageFile; extern BOOLEAN MmProtectFreedNonPagedPool; extern BOOLEAN MmTrackLockedPages; extern BOOLEAN MmTrackPtes; extern BOOLEAN MmDynamicPfn; extern BOOLEAN MmMirroring; extern BOOLEAN MmMakeLowMemory; extern BOOLEAN MmEnforceWriteProtection; extern SIZE_T MmAllocationFragment; extern ULONG MmConsumedPoolPercentage; extern ULONG MmVerifyDriverBufferType; extern ULONG MmVerifyDriverLevel; extern WCHAR MmVerifyDriverBuffer[512]; extern WCHAR MmLargePageDriverBuffer[512]; extern LIST_ENTRY MiLargePageDriverList; extern BOOLEAN MiLargePageAllDrivers; extern ULONG MmVerifyDriverBufferLength; extern ULONG MmLargePageDriverBufferLength; extern SIZE_T MmSizeOfNonPagedPoolInBytes; extern SIZE_T MmMaximumNonPagedPoolInBytes; extern PFN_NUMBER MmMaximumNonPagedPoolInPages; extern PFN_NUMBER MmSizeOfPagedPoolInPages; extern PVOID MmNonPagedSystemStart; extern PVOID MmNonPagedPoolStart; extern PVOID MmNonPagedPoolExpansionStart; extern PVOID MmNonPagedPoolEnd; extern SIZE_T MmSizeOfPagedPoolInBytes; extern PVOID MmPagedPoolStart; extern PVOID MmPagedPoolEnd; extern PVOID MmSessionBase; extern SIZE_T MmSessionSize; extern PMMPTE MmFirstReservedMappingPte, MmLastReservedMappingPte; extern PMMPTE MiFirstReservedZeroingPte; extern MI_PFN_CACHE_ATTRIBUTE MiPlatformCacheAttributes[2][MmMaximumCacheType]; extern PPHYSICAL_MEMORY_DESCRIPTOR MmPhysicalMemoryBlock; extern SIZE_T MmBootImageSize; extern PMMPTE MmSystemPtesStart[MaximumPtePoolTypes]; extern PMMPTE MmSystemPtesEnd[MaximumPtePoolTypes]; extern PMEMORY_ALLOCATION_DESCRIPTOR MxFreeDescriptor; extern MEMORY_ALLOCATION_DESCRIPTOR MxOldFreeDescriptor; extern ULONG_PTR MxPfnAllocation; extern MM_PAGED_POOL_INFO MmPagedPoolInfo; extern RTL_BITMAP MiPfnBitMap; extern KGUARDED_MUTEX MmPagedPoolMutex; extern KGUARDED_MUTEX MmSectionCommitMutex; extern PVOID MmPagedPoolStart; extern PVOID MmPagedPoolEnd; extern PVOID MmNonPagedSystemStart; extern PVOID MiSystemViewStart; extern SIZE_T MmSystemViewSize; extern PVOID MmSessionBase; extern PVOID MiSessionSpaceEnd; extern PMMPTE MiSessionImagePteStart; extern PMMPTE MiSessionImagePteEnd; extern PMMPTE MiSessionBasePte; extern PMMPTE MiSessionLastPte; extern SIZE_T MmSizeOfPagedPoolInBytes; extern PMMPDE MmSystemPagePtes; extern PVOID MmSystemCacheStart; extern PVOID MmSystemCacheEnd; extern MMSUPPORT MmSystemCacheWs; extern SIZE_T MmAllocatedNonPagedPool; extern ULONG MmSpecialPoolTag; extern PVOID MmHyperSpaceEnd; extern PMMWSL MmSystemCacheWorkingSetList; extern SIZE_T MmMinimumNonPagedPoolSize; extern ULONG MmMinAdditionNonPagedPoolPerMb; extern SIZE_T MmDefaultMaximumNonPagedPool; extern ULONG MmMaxAdditionNonPagedPoolPerMb; extern ULONG MmSecondaryColors; extern ULONG MmSecondaryColorMask; extern ULONG MmNumberOfSystemPtes; extern ULONG MmMaximumNonPagedPoolPercent; extern ULONG MmLargeStackSize; extern PMMCOLOR_TABLES MmFreePagesByColor[FreePageList + 1]; extern MMPFNLIST MmStandbyPageListByPriority[8]; extern ULONG MmProductType; extern MM_SYSTEMSIZE MmSystemSize; extern PKEVENT MiLowMemoryEvent; extern PKEVENT MiHighMemoryEvent; extern PKEVENT MiLowPagedPoolEvent; extern PKEVENT MiHighPagedPoolEvent; extern PKEVENT MiLowNonPagedPoolEvent; extern PKEVENT MiHighNonPagedPoolEvent; extern PFN_NUMBER MmLowMemoryThreshold; extern PFN_NUMBER MmHighMemoryThreshold; extern PFN_NUMBER MiLowPagedPoolThreshold; extern PFN_NUMBER MiHighPagedPoolThreshold; extern PFN_NUMBER MiLowNonPagedPoolThreshold; extern PFN_NUMBER MiHighNonPagedPoolThreshold; extern PFN_NUMBER MmMinimumFreePages; extern PFN_NUMBER MmPlentyFreePages; extern SIZE_T MmMinimumStackCommitInBytes; extern PFN_COUNT MiExpansionPoolPagesInitialCharge; extern PFN_NUMBER MmResidentAvailablePages; extern PFN_NUMBER MmResidentAvailableAtInit; extern ULONG MmTotalFreeSystemPtes[MaximumPtePoolTypes]; extern PFN_NUMBER MmTotalSystemDriverPages; extern ULONG MmCritsectTimeoutSeconds; extern PVOID MiSessionImageStart; extern PVOID MiSessionImageEnd; extern PMMPTE MiHighestUserPte; extern PMMPDE MiHighestUserPde; extern PFN_NUMBER MmSystemPageDirectory[PD_COUNT]; extern PMMPTE MmSharedUserDataPte; extern LIST_ENTRY MmProcessList; extern BOOLEAN MmZeroingPageThreadActive; extern KEVENT MmZeroingPageEvent; extern ULONG MmSystemPageColor; extern ULONG MmProcessColorSeed; extern PMMWSL MmWorkingSetList; extern PFN_NUMBER MiNumberOfFreePages; extern SIZE_T MmSessionViewSize; extern SIZE_T MmSessionPoolSize; extern SIZE_T MmSessionImageSize; extern PVOID MiSystemViewStart; extern PVOID MiSessionPoolEnd; // 0xBE000000 extern PVOID MiSessionPoolStart; // 0xBD000000 extern PVOID MiSessionViewStart; // 0xBE000000 extern PVOID MiSessionSpaceWs; extern ULONG MmMaximumDeadKernelStacks; extern SLIST_HEADER MmDeadStackSListHead; extern MM_AVL_TABLE MmSectionBasedRoot; extern KGUARDED_MUTEX MmSectionBasedMutex; extern PVOID MmHighSectionBase; extern SIZE_T MmSystemLockPagesCount; extern ULONG_PTR MmSubsectionBase; extern LARGE_INTEGER MmCriticalSectionTimeout; extern LIST_ENTRY MmWorkingSetExpansionHead; extern KSPIN_LOCK MmExpansionLock; extern PETHREAD MiExpansionLockOwner; FORCEINLINE BOOLEAN MiIsMemoryTypeFree(TYPE_OF_MEMORY MemoryType) { return ((MemoryType == LoaderFree) || (MemoryType == LoaderLoadedProgram) || (MemoryType == LoaderFirmwareTemporary) || (MemoryType == LoaderOsloaderStack)); } FORCEINLINE BOOLEAN MiIsMemoryTypeInvisible(TYPE_OF_MEMORY MemoryType) { return ((MemoryType == LoaderFirmwarePermanent) || (MemoryType == LoaderSpecialMemory) || (MemoryType == LoaderHALCachedMemory) || (MemoryType == LoaderBBTMemory)); } #ifdef _M_AMD64 FORCEINLINE BOOLEAN MiIsUserPxe(PVOID Address) { return ((ULONG_PTR)Address >> 7) == 0x1FFFFEDF6FB7DA0ULL; } FORCEINLINE BOOLEAN MiIsUserPpe(PVOID Address) { return ((ULONG_PTR)Address >> 16) == 0xFFFFF6FB7DA0ULL; } FORCEINLINE BOOLEAN MiIsUserPde(PVOID Address) { return ((ULONG_PTR)Address >> 25) == 0x7FFFFB7DA0ULL; } FORCEINLINE BOOLEAN MiIsUserPte(PVOID Address) { return ((ULONG_PTR)Address >> 34) == 0x3FFFFDA0ULL; } #else FORCEINLINE BOOLEAN MiIsUserPde(PVOID Address) { return ((Address >= (PVOID)MiAddressToPde(NULL)) && (Address <= (PVOID)MiHighestUserPde)); } FORCEINLINE BOOLEAN MiIsUserPte(PVOID Address) { return (Address <= (PVOID)MiHighestUserPte); } #endif // // Figures out the hardware bits for a PTE // FORCEINLINE ULONG_PTR MiDetermineUserGlobalPteMask(IN PVOID PointerPte) { MMPTE TempPte; /* Start fresh */ TempPte.u.Long = 0; /* Make it valid and accessed */ TempPte.u.Hard.Valid = TRUE; MI_MAKE_ACCESSED_PAGE(&TempPte); /* Is this for user-mode? */ if ( #if (_MI_PAGING_LEVELS == 4) MiIsUserPxe(PointerPte) || #endif #if (_MI_PAGING_LEVELS >= 3) MiIsUserPpe(PointerPte) || #endif MiIsUserPde(PointerPte) || MiIsUserPte(PointerPte)) { /* Set the owner bit */ MI_MAKE_OWNER_PAGE(&TempPte); } /* FIXME: We should also set the global bit */ /* Return the protection */ return TempPte.u.Long; } // // Creates a valid kernel PTE with the given protection // FORCEINLINE VOID MI_MAKE_HARDWARE_PTE_KERNEL(IN PMMPTE NewPte, IN PMMPTE MappingPte, IN ULONG_PTR ProtectionMask, IN PFN_NUMBER PageFrameNumber) { /* Only valid for kernel, non-session PTEs */ ASSERT(MappingPte > MiHighestUserPte); ASSERT(!MI_IS_SESSION_PTE(MappingPte)); ASSERT((MappingPte < (PMMPTE)PDE_BASE) || (MappingPte > (PMMPTE)PDE_TOP)); /* Start fresh */ *NewPte = ValidKernelPte; /* Set the protection and page */ NewPte->u.Hard.PageFrameNumber = PageFrameNumber; NewPte->u.Long |= MmProtectToPteMask[ProtectionMask]; } // // Creates a valid PTE with the given protection // FORCEINLINE VOID MI_MAKE_HARDWARE_PTE(IN PMMPTE NewPte, IN PMMPTE MappingPte, IN ULONG_PTR ProtectionMask, IN PFN_NUMBER PageFrameNumber) { /* Set the protection and page */ NewPte->u.Long = MiDetermineUserGlobalPteMask(MappingPte); NewPte->u.Long |= MmProtectToPteMask[ProtectionMask]; NewPte->u.Hard.PageFrameNumber = PageFrameNumber; } // // Creates a valid user PTE with the given protection // FORCEINLINE VOID MI_MAKE_HARDWARE_PTE_USER(IN PMMPTE NewPte, IN PMMPTE MappingPte, IN ULONG_PTR ProtectionMask, IN PFN_NUMBER PageFrameNumber) { /* Only valid for kernel, non-session PTEs */ ASSERT(MappingPte <= MiHighestUserPte); /* Start fresh */ NewPte->u.Long = 0; /* Set the protection and page */ NewPte->u.Hard.Valid = TRUE; NewPte->u.Hard.Owner = TRUE; NewPte->u.Hard.PageFrameNumber = PageFrameNumber; NewPte->u.Long |= MmProtectToPteMask[ProtectionMask]; } #ifndef _M_AMD64 // // Builds a Prototype PTE for the address of the PTE // FORCEINLINE VOID MI_MAKE_PROTOTYPE_PTE(IN PMMPTE NewPte, IN PMMPTE PointerPte) { ULONG_PTR Offset; /* Mark this as a prototype */ NewPte->u.Long = 0; NewPte->u.Proto.Prototype = 1; /* * Prototype PTEs are only valid in paged pool by design, this little trick * lets us only use 30 bits for the adress of the PTE, as long as the area * stays 1024MB At most. */ Offset = (ULONG_PTR)PointerPte - (ULONG_PTR)MmPagedPoolStart; /* * 7 bits go in the "low" (but we assume the bottom 2 are zero) * and the other 21 bits go in the "high" */ NewPte->u.Proto.ProtoAddressLow = (Offset & 0x1FC) >> 2; NewPte->u.Proto.ProtoAddressHigh = (Offset & 0x3FFFFE00) >> 9; } // // Builds a Subsection PTE for the address of the Segment // FORCEINLINE VOID MI_MAKE_SUBSECTION_PTE(IN PMMPTE NewPte, IN PVOID Segment) { ULONG_PTR Offset; /* Mark this as a prototype */ NewPte->u.Long = 0; NewPte->u.Subsect.Prototype = 1; /* * Segments are only valid either in nonpaged pool. We store the 20 bit * difference either from the top or bottom of nonpaged pool, giving a * maximum of 128MB to each delta, meaning nonpaged pool cannot exceed * 256MB. */ if ((ULONG_PTR)Segment < ((ULONG_PTR)MmSubsectionBase + (128 * _1MB))) { Offset = (ULONG_PTR)Segment - (ULONG_PTR)MmSubsectionBase; NewPte->u.Subsect.WhichPool = PagedPool; } else { Offset = (ULONG_PTR)MmNonPagedPoolEnd - (ULONG_PTR)Segment; NewPte->u.Subsect.WhichPool = NonPagedPool; } /* * 4 bits go in the "low" (but we assume the bottom 3 are zero) * and the other 20 bits go in the "high" */ NewPte->u.Subsect.SubsectionAddressLow = (Offset & 0x78) >> 3; NewPte->u.Subsect.SubsectionAddressHigh = (Offset & 0xFFFFF80) >> 7; } FORCEINLINE BOOLEAN MI_IS_MAPPED_PTE(PMMPTE PointerPte) { /// \todo Make this reasonable code, this is UGLY! return ((PointerPte->u.Long & 0xFFFFFC01) != 0); } #endif FORCEINLINE VOID MI_MAKE_TRANSITION_PTE(_Out_ PMMPTE NewPte, _In_ PFN_NUMBER Page, _In_ ULONG Protection) { NewPte->u.Long = 0; NewPte->u.Trans.Transition = 1; NewPte->u.Trans.Protection = Protection; NewPte->u.Trans.PageFrameNumber = Page; } // // Returns if the page is physically resident (ie: a large page) // FIXFIX: CISC/x86 only? // FORCEINLINE BOOLEAN MI_IS_PHYSICAL_ADDRESS(IN PVOID Address) { PMMPDE PointerPde; /* Large pages are never paged out, always physically resident */ PointerPde = MiAddressToPde(Address); return ((PointerPde->u.Hard.LargePage) && (PointerPde->u.Hard.Valid)); } // // Writes a valid PTE // FORCEINLINE VOID MI_WRITE_VALID_PTE(IN PMMPTE PointerPte, IN MMPTE TempPte) { /* Write the valid PTE */ ASSERT(PointerPte->u.Hard.Valid == 0); ASSERT(TempPte.u.Hard.Valid == 1); *PointerPte = TempPte; } // // Updates a valid PTE // FORCEINLINE VOID MI_UPDATE_VALID_PTE(IN PMMPTE PointerPte, IN MMPTE TempPte) { /* Write the valid PTE */ ASSERT(PointerPte->u.Hard.Valid == 1); ASSERT(TempPte.u.Hard.Valid == 1); ASSERT(PointerPte->u.Hard.PageFrameNumber == TempPte.u.Hard.PageFrameNumber); *PointerPte = TempPte; } // // Writes an invalid PTE // FORCEINLINE VOID MI_WRITE_INVALID_PTE(IN PMMPTE PointerPte, IN MMPTE InvalidPte) { /* Write the invalid PTE */ ASSERT(InvalidPte.u.Hard.Valid == 0); ASSERT(InvalidPte.u.Long != 0); *PointerPte = InvalidPte; } // // Erase the PTE completely // FORCEINLINE VOID MI_ERASE_PTE(IN PMMPTE PointerPte) { /* Zero out the PTE */ ASSERT(PointerPte->u.Long != 0); PointerPte->u.Long = 0; } // // Writes a valid PDE // FORCEINLINE VOID MI_WRITE_VALID_PDE(IN PMMPDE PointerPde, IN MMPDE TempPde) { /* Write the valid PDE */ ASSERT(PointerPde->u.Hard.Valid == 0); ASSERT(TempPde.u.Hard.Valid == 1); *PointerPde = TempPde; } // // Writes an invalid PDE // FORCEINLINE VOID MI_WRITE_INVALID_PDE(IN PMMPDE PointerPde, IN MMPDE InvalidPde) { /* Write the invalid PDE */ ASSERT(InvalidPde.u.Hard.Valid == 0); ASSERT(InvalidPde.u.Long != 0); *PointerPde = InvalidPde; } // // Checks if the thread already owns a working set // FORCEINLINE BOOLEAN MM_ANY_WS_LOCK_HELD(IN PETHREAD Thread) { /* If any of these are held, return TRUE */ return ((Thread->OwnsProcessWorkingSetExclusive) || (Thread->OwnsProcessWorkingSetShared) || (Thread->OwnsSystemWorkingSetExclusive) || (Thread->OwnsSystemWorkingSetShared) || (Thread->OwnsSessionWorkingSetExclusive) || (Thread->OwnsSessionWorkingSetShared)); } // // Checks if the process owns the working set lock // FORCEINLINE BOOLEAN MI_WS_OWNER(IN PEPROCESS Process) { /* Check if this process is the owner, and that the thread owns the WS */ if (PsGetCurrentThread()->OwnsProcessWorkingSetExclusive == 0) { DPRINT("Thread: %p is not an owner\n", PsGetCurrentThread()); } if (KeGetCurrentThread()->ApcState.Process != &Process->Pcb) { DPRINT("Current thread %p is attached to another process %p\n", PsGetCurrentThread(), Process); } return ((KeGetCurrentThread()->ApcState.Process == &Process->Pcb) && ((PsGetCurrentThread()->OwnsProcessWorkingSetExclusive) || (PsGetCurrentThread()->OwnsProcessWorkingSetShared))); } // // New ARM3<->RosMM PAGE Architecture // FORCEINLINE BOOLEAN MiIsRosSectionObject(IN PVOID Section) { PROS_SECTION_OBJECT RosSection = Section; if ((RosSection->Type == 'SC') && (RosSection->Size == 'TN')) return TRUE; return FALSE; } #define MI_IS_ROS_PFN(x) ((x)->u4.AweAllocation == TRUE) VOID NTAPI MiDecrementReferenceCount( IN PMMPFN Pfn1, IN PFN_NUMBER PageFrameIndex ); FORCEINLINE BOOLEAN MI_IS_WS_UNSAFE(IN PEPROCESS Process) { return (Process->Vm.Flags.AcquiredUnsafe == TRUE); } // // Locks the working set for the given process // FORCEINLINE VOID MiLockProcessWorkingSet(IN PEPROCESS Process, IN PETHREAD Thread) { /* Shouldn't already be owning the process working set */ ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE); ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); /* Block APCs, make sure that still nothing is already held */ KeEnterGuardedRegion(); ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); /* Lock the working set */ ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex); /* Now claim that we own the lock */ ASSERT(!MI_IS_WS_UNSAFE(Process)); ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); Thread->OwnsProcessWorkingSetExclusive = TRUE; } FORCEINLINE VOID MiLockProcessWorkingSetShared(IN PEPROCESS Process, IN PETHREAD Thread) { /* Shouldn't already be owning the process working set */ ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE); ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); /* Block APCs, make sure that still nothing is already held */ KeEnterGuardedRegion(); ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); /* Lock the working set */ ExAcquirePushLockShared(&Process->Vm.WorkingSetMutex); /* Now claim that we own the lock */ ASSERT(!MI_IS_WS_UNSAFE(Process)); ASSERT(Thread->OwnsProcessWorkingSetShared == FALSE); ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); Thread->OwnsProcessWorkingSetShared = TRUE; } FORCEINLINE VOID MiLockProcessWorkingSetUnsafe(IN PEPROCESS Process, IN PETHREAD Thread) { /* Shouldn't already be owning the process working set */ ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); /* APCs must be blocked, make sure that still nothing is already held */ ASSERT(KeAreAllApcsDisabled() == TRUE); ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); /* Lock the working set */ ExAcquirePushLockExclusive(&Process->Vm.WorkingSetMutex); /* Now claim that we own the lock */ ASSERT(!MI_IS_WS_UNSAFE(Process)); Process->Vm.Flags.AcquiredUnsafe = 1; ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); Thread->OwnsProcessWorkingSetExclusive = TRUE; } // // Unlocks the working set for the given process // FORCEINLINE VOID MiUnlockProcessWorkingSet(IN PEPROCESS Process, IN PETHREAD Thread) { /* Make sure we are the owner of a safe acquisition */ ASSERT(MI_WS_OWNER(Process)); ASSERT(!MI_IS_WS_UNSAFE(Process)); /* The thread doesn't own it anymore */ ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE); Thread->OwnsProcessWorkingSetExclusive = FALSE; /* Release the lock and re-enable APCs */ ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex); KeLeaveGuardedRegion(); } // // Unlocks the working set for the given process // FORCEINLINE VOID MiUnlockProcessWorkingSetShared(IN PEPROCESS Process, IN PETHREAD Thread) { /* Make sure we are the owner of a safe acquisition (because shared) */ ASSERT(MI_WS_OWNER(Process)); ASSERT(!MI_IS_WS_UNSAFE(Process)); /* Ensure we are in a shared acquisition */ ASSERT(Thread->OwnsProcessWorkingSetShared == TRUE); ASSERT(Thread->OwnsProcessWorkingSetExclusive == FALSE); /* Don't claim the lock anylonger */ Thread->OwnsProcessWorkingSetShared = FALSE; /* Release the lock and re-enable APCs */ ExReleasePushLockShared(&Process->Vm.WorkingSetMutex); KeLeaveGuardedRegion(); } // // Unlocks the working set for the given process // FORCEINLINE VOID MiUnlockProcessWorkingSetUnsafe(IN PEPROCESS Process, IN PETHREAD Thread) { /* Make sure we are the owner of an unsafe acquisition */ ASSERT(KeGetCurrentIrql() <= APC_LEVEL); ASSERT(KeAreAllApcsDisabled() == TRUE); ASSERT(MI_WS_OWNER(Process)); ASSERT(MI_IS_WS_UNSAFE(Process)); /* No longer unsafe */ Process->Vm.Flags.AcquiredUnsafe = 0; /* The thread doesn't own it anymore */ ASSERT(Thread->OwnsProcessWorkingSetExclusive == TRUE); Thread->OwnsProcessWorkingSetExclusive = FALSE; /* Release the lock but don't touch APC state */ ExReleasePushLockExclusive(&Process->Vm.WorkingSetMutex); ASSERT(KeGetCurrentIrql() <= APC_LEVEL); } // // Locks the working set // FORCEINLINE VOID MiLockWorkingSet(IN PETHREAD Thread, IN PMMSUPPORT WorkingSet) { /* Block APCs */ KeEnterGuardedRegion(); /* Working set should be in global memory */ ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE); /* Thread shouldn't already be owning something */ ASSERT(!MM_ANY_WS_LOCK_HELD(Thread)); /* Lock this working set */ ExAcquirePushLockExclusive(&WorkingSet->WorkingSetMutex); /* Which working set is this? */ if (WorkingSet == &MmSystemCacheWs) { /* Own the system working set */ ASSERT((Thread->OwnsSystemWorkingSetExclusive == FALSE) && (Thread->OwnsSystemWorkingSetShared == FALSE)); Thread->OwnsSystemWorkingSetExclusive = TRUE; } else if (WorkingSet->Flags.SessionSpace) { /* Own the session working set */ ASSERT((Thread->OwnsSessionWorkingSetExclusive == FALSE) && (Thread->OwnsSessionWorkingSetShared == FALSE)); Thread->OwnsSessionWorkingSetExclusive = TRUE; } else { /* Own the process working set */ ASSERT((Thread->OwnsProcessWorkingSetExclusive == FALSE) && (Thread->OwnsProcessWorkingSetShared == FALSE)); Thread->OwnsProcessWorkingSetExclusive = TRUE; } } // // Unlocks the working set // FORCEINLINE VOID MiUnlockWorkingSet(IN PETHREAD Thread, IN PMMSUPPORT WorkingSet) { /* Working set should be in global memory */ ASSERT(MI_IS_SESSION_ADDRESS((PVOID)WorkingSet) == FALSE); /* Which working set is this? */ if (WorkingSet == &MmSystemCacheWs) { /* Release the system working set */ ASSERT((Thread->OwnsSystemWorkingSetExclusive == TRUE) || (Thread->OwnsSystemWorkingSetShared == TRUE)); Thread->OwnsSystemWorkingSetExclusive = FALSE; } else if (WorkingSet->Flags.SessionSpace) { /* Release the session working set */ ASSERT((Thread->OwnsSessionWorkingSetExclusive == TRUE) || (Thread->OwnsSessionWorkingSetShared == TRUE)); Thread->OwnsSessionWorkingSetExclusive = 0; } else { /* Release the process working set */ ASSERT((Thread->OwnsProcessWorkingSetExclusive) || (Thread->OwnsProcessWorkingSetShared)); Thread->OwnsProcessWorkingSetExclusive = FALSE; } /* Release the working set lock */ ExReleasePushLockExclusive(&WorkingSet->WorkingSetMutex); /* Unblock APCs */ KeLeaveGuardedRegion(); } FORCEINLINE VOID MiUnlockProcessWorkingSetForFault(IN PEPROCESS Process, IN PETHREAD Thread, OUT PBOOLEAN Safe, OUT PBOOLEAN Shared) { ASSERT(MI_WS_OWNER(Process)); /* Check if the current owner is unsafe */ if (MI_IS_WS_UNSAFE(Process)) { /* Release unsafely */ MiUnlockProcessWorkingSetUnsafe(Process, Thread); *Safe = FALSE; *Shared = FALSE; } else if (Thread->OwnsProcessWorkingSetExclusive == 1) { /* Owner is safe and exclusive, release normally */ MiUnlockProcessWorkingSet(Process, Thread); *Safe = TRUE; *Shared = FALSE; } else { /* Owner is shared (implies safe), release normally */ MiUnlockProcessWorkingSetShared(Process, Thread); *Safe = TRUE; *Shared = TRUE; } } FORCEINLINE VOID MiLockProcessWorkingSetForFault(IN PEPROCESS Process, IN PETHREAD Thread, IN BOOLEAN Safe, IN BOOLEAN Shared) { /* Check if this was a safe lock or not */ if (Safe) { if (Shared) { /* Reacquire safely & shared */ MiLockProcessWorkingSetShared(Process, Thread); } else { /* Reacquire safely */ MiLockProcessWorkingSet(Process, Thread); } } else { /* Unsafe lock cannot be shared */ ASSERT(Shared == FALSE); /* Reacquire unsafely */ MiLockProcessWorkingSetUnsafe(Process, Thread); } } FORCEINLINE KIRQL MiAcquireExpansionLock(VOID) { KIRQL OldIrql; ASSERT(KeGetCurrentIrql() <= APC_LEVEL); KeAcquireSpinLock(&MmExpansionLock, &OldIrql); ASSERT(MiExpansionLockOwner == NULL); MiExpansionLockOwner = PsGetCurrentThread(); return OldIrql; } FORCEINLINE VOID MiReleaseExpansionLock(KIRQL OldIrql) { ASSERT(MiExpansionLockOwner == PsGetCurrentThread()); MiExpansionLockOwner = NULL; KeReleaseSpinLock(&MmExpansionLock, OldIrql); ASSERT(KeGetCurrentIrql() <= APC_LEVEL); } // // Returns the ProtoPTE inside a VAD for the given VPN // FORCEINLINE PMMPTE MI_GET_PROTOTYPE_PTE_FOR_VPN(IN PMMVAD Vad, IN ULONG_PTR Vpn) { PMMPTE ProtoPte; /* Find the offset within the VAD's prototype PTEs */ ProtoPte = Vad->FirstPrototypePte + (Vpn - Vad->StartingVpn); ASSERT(ProtoPte <= Vad->LastContiguousPte); return ProtoPte; } // // Returns the PFN Database entry for the given page number // Warning: This is not necessarily a valid PFN database entry! // FORCEINLINE PMMPFN MI_PFN_ELEMENT(IN PFN_NUMBER Pfn) { /* Get the entry */ return &MmPfnDatabase[Pfn]; }; // // Drops a locked page without dereferencing it // FORCEINLINE VOID MiDropLockCount(IN PMMPFN Pfn1) { /* This page shouldn't be locked, but it should be valid */ ASSERT(Pfn1->u3.e2.ReferenceCount != 0); ASSERT(Pfn1->u2.ShareCount == 0); /* Is this the last reference to the page */ if (Pfn1->u3.e2.ReferenceCount == 1) { /* It better not be valid */ ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid); /* Is it a prototype PTE? */ if ((Pfn1->u3.e1.PrototypePte == 1) && (Pfn1->OriginalPte.u.Soft.Prototype == 1)) { /* FIXME: We should return commit */ DPRINT1("Not returning commit for prototype PTE\n"); } /* Update the counter */ InterlockedDecrementSizeT(&MmSystemLockPagesCount); } } // // Drops a locked page and dereferences it // FORCEINLINE VOID MiDereferencePfnAndDropLockCount(IN PMMPFN Pfn1) { USHORT RefCount, OldRefCount; PFN_NUMBER PageFrameIndex; /* Loop while we decrement the page successfully */ do { /* There should be at least one reference */ OldRefCount = Pfn1->u3.e2.ReferenceCount; ASSERT(OldRefCount != 0); /* Are we the last one */ if (OldRefCount == 1) { /* The page shoudln't be shared not active at this point */ ASSERT(Pfn1->u3.e2.ReferenceCount == 1); ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid); ASSERT(Pfn1->u2.ShareCount == 0); /* Is it a prototype PTE? */ if ((Pfn1->u3.e1.PrototypePte == 1) && (Pfn1->OriginalPte.u.Soft.Prototype == 1)) { /* FIXME: We should return commit */ DPRINT1("Not returning commit for prototype PTE\n"); } /* Update the counter, and drop a reference the long way */ InterlockedDecrementSizeT(&MmSystemLockPagesCount); PageFrameIndex = MiGetPfnEntryIndex(Pfn1); MiDecrementReferenceCount(Pfn1, PageFrameIndex); return; } /* Drop a reference the short way, and that's it */ RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount, OldRefCount - 1, OldRefCount); ASSERT(RefCount != 0); } while (OldRefCount != RefCount); /* If we got here, there should be more than one reference */ ASSERT(RefCount > 1); if (RefCount == 2) { /* Is it still being shared? */ if (Pfn1->u2.ShareCount >= 1) { /* Then it should be valid */ ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); /* Is it a prototype PTE? */ if ((Pfn1->u3.e1.PrototypePte == 1) && (Pfn1->OriginalPte.u.Soft.Prototype == 1)) { /* We don't handle ethis */ ASSERT(FALSE); } /* Update the counter */ InterlockedDecrementSizeT(&MmSystemLockPagesCount); } } } // // References a locked page and updates the counter // Used in MmProbeAndLockPages to handle different edge cases // FORCEINLINE VOID MiReferenceProbedPageAndBumpLockCount(IN PMMPFN Pfn1) { USHORT RefCount, OldRefCount; /* Sanity check */ ASSERT(Pfn1->u3.e2.ReferenceCount != 0); /* Does ARM3 own the page? */ if (MI_IS_ROS_PFN(Pfn1)) { /* ReactOS Mm doesn't track share count */ ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); } else { /* On ARM3 pages, we should see a valid share count */ ASSERT((Pfn1->u2.ShareCount != 0) && (Pfn1->u3.e1.PageLocation == ActiveAndValid)); /* Is it a prototype PTE? */ if ((Pfn1->u3.e1.PrototypePte == 1) && (Pfn1->OriginalPte.u.Soft.Prototype == 1)) { /* FIXME: We should charge commit */ DPRINT1("Not charging commit for prototype PTE\n"); } } /* More locked pages! */ InterlockedIncrementSizeT(&MmSystemLockPagesCount); /* Loop trying to update the reference count */ do { /* Get the current reference count, make sure it's valid */ OldRefCount = Pfn1->u3.e2.ReferenceCount; ASSERT(OldRefCount != 0); ASSERT(OldRefCount < 2500); /* Bump it up by one */ RefCount = InterlockedCompareExchange16((PSHORT)&Pfn1->u3.e2.ReferenceCount, OldRefCount + 1, OldRefCount); ASSERT(RefCount != 0); } while (OldRefCount != RefCount); /* Was this the first lock attempt? If not, undo our bump */ if (OldRefCount != 1) InterlockedDecrementSizeT(&MmSystemLockPagesCount); } // // References a locked page and updates the counter // Used in all other cases except MmProbeAndLockPages // FORCEINLINE VOID MiReferenceUsedPageAndBumpLockCount(IN PMMPFN Pfn1) { USHORT NewRefCount; /* Is it a prototype PTE? */ if ((Pfn1->u3.e1.PrototypePte == 1) && (Pfn1->OriginalPte.u.Soft.Prototype == 1)) { /* FIXME: We should charge commit */ DPRINT1("Not charging commit for prototype PTE\n"); } /* More locked pages! */ InterlockedIncrementSizeT(&MmSystemLockPagesCount); /* Update the reference count */ NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount); if (NewRefCount == 2) { /* Is it locked or shared? */ if (Pfn1->u2.ShareCount) { /* It's shared, so make sure it's active */ ASSERT(Pfn1->u3.e1.PageLocation == ActiveAndValid); } else { /* It's locked, so we shouldn't lock again */ InterlockedDecrementSizeT(&MmSystemLockPagesCount); } } else { /* Someone had already locked the page, so undo our bump */ ASSERT(NewRefCount < 2500); InterlockedDecrementSizeT(&MmSystemLockPagesCount); } } // // References a locked page and updates the counter // Used in all other cases except MmProbeAndLockPages // FORCEINLINE VOID MiReferenceUnusedPageAndBumpLockCount(IN PMMPFN Pfn1) { USHORT NewRefCount; /* Make sure the page isn't used yet */ ASSERT(Pfn1->u2.ShareCount == 0); ASSERT(Pfn1->u3.e1.PageLocation != ActiveAndValid); /* Is it a prototype PTE? */ if ((Pfn1->u3.e1.PrototypePte == 1) && (Pfn1->OriginalPte.u.Soft.Prototype == 1)) { /* FIXME: We should charge commit */ DPRINT1("Not charging commit for prototype PTE\n"); } /* More locked pages! */ InterlockedIncrementSizeT(&MmSystemLockPagesCount); /* Update the reference count */ NewRefCount = InterlockedIncrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount); if (NewRefCount != 1) { /* Someone had already locked the page, so undo our bump */ ASSERT(NewRefCount < 2500); InterlockedDecrementSizeT(&MmSystemLockPagesCount); } } FORCEINLINE VOID MiIncrementPageTableReferences(IN PVOID Address) { PUSHORT RefCount; RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]; *RefCount += 1; ASSERT(*RefCount <= PTE_PER_PAGE); } FORCEINLINE VOID MiDecrementPageTableReferences(IN PVOID Address) { PUSHORT RefCount; RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]; *RefCount -= 1; ASSERT(*RefCount < PTE_PER_PAGE); } FORCEINLINE USHORT MiQueryPageTableReferences(IN PVOID Address) { PUSHORT RefCount; RefCount = &MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]; return *RefCount; } INIT_FUNCTION BOOLEAN NTAPI MmArmInitSystem( IN ULONG Phase, IN PLOADER_PARAMETER_BLOCK LoaderBlock ); INIT_FUNCTION VOID NTAPI MiInitializeSessionSpaceLayout(VOID); INIT_FUNCTION NTSTATUS NTAPI MiInitMachineDependent( IN PLOADER_PARAMETER_BLOCK LoaderBlock ); INIT_FUNCTION VOID NTAPI MiComputeColorInformation( VOID ); INIT_FUNCTION VOID NTAPI MiMapPfnDatabase( IN PLOADER_PARAMETER_BLOCK LoaderBlock ); INIT_FUNCTION VOID NTAPI MiInitializeColorTables( VOID ); INIT_FUNCTION VOID NTAPI MiInitializePfnDatabase( IN PLOADER_PARAMETER_BLOCK LoaderBlock ); VOID NTAPI MiInitializeSessionWsSupport( VOID ); VOID NTAPI MiInitializeSessionIds( VOID ); INIT_FUNCTION BOOLEAN NTAPI MiInitializeMemoryEvents( VOID ); INIT_FUNCTION PFN_NUMBER NTAPI MxGetNextPage( IN PFN_NUMBER PageCount ); INIT_FUNCTION PPHYSICAL_MEMORY_DESCRIPTOR NTAPI MmInitializeMemoryLimits( IN PLOADER_PARAMETER_BLOCK LoaderBlock, IN PBOOLEAN IncludeType ); PFN_NUMBER NTAPI MiPagesInLoaderBlock( IN PLOADER_PARAMETER_BLOCK LoaderBlock, IN PBOOLEAN IncludeType ); VOID FASTCALL MiSyncARM3WithROS( IN PVOID AddressStart, IN PVOID AddressEnd ); NTSTATUS NTAPI MiRosProtectVirtualMemory( IN PEPROCESS Process, IN OUT PVOID *BaseAddress, IN OUT PSIZE_T NumberOfBytesToProtect, IN ULONG NewAccessProtection, OUT PULONG OldAccessProtection OPTIONAL ); NTSTATUS NTAPI MmArmAccessFault( IN ULONG FaultCode, IN PVOID Address, IN KPROCESSOR_MODE Mode, IN PVOID TrapInformation ); NTSTATUS FASTCALL MiCheckPdeForPagedPool( IN PVOID Address ); INIT_FUNCTION VOID NTAPI MiInitializeNonPagedPool( VOID ); INIT_FUNCTION VOID NTAPI MiInitializeNonPagedPoolThresholds( VOID ); INIT_FUNCTION VOID NTAPI MiInitializePoolEvents( VOID ); INIT_FUNCTION VOID // NTAPI // InitializePool( // IN POOL_TYPE PoolType,// FIXFIX: This should go in ex.h after the pool merge IN ULONG Threshold // ); // // FIXFIX: THIS ONE TOO INIT_FUNCTION VOID NTAPI ExInitializePoolDescriptor( IN PPOOL_DESCRIPTOR PoolDescriptor, IN POOL_TYPE PoolType, IN ULONG PoolIndex, IN ULONG Threshold, IN PVOID PoolLock ); NTSTATUS NTAPI MiInitializeSessionPool( VOID ); INIT_FUNCTION VOID NTAPI MiInitializeSystemPtes( IN PMMPTE StartingPte, IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE PoolType ); PMMPTE NTAPI MiReserveSystemPtes( IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType ); VOID NTAPI MiReleaseSystemPtes( IN PMMPTE StartingPte, IN ULONG NumberOfPtes, IN MMSYSTEM_PTE_POOL_TYPE SystemPtePoolType ); PFN_NUMBER NTAPI MiFindContiguousPages( IN PFN_NUMBER LowestPfn, IN PFN_NUMBER HighestPfn, IN PFN_NUMBER BoundaryPfn, IN PFN_NUMBER SizeInPages, IN MEMORY_CACHING_TYPE CacheType ); PVOID NTAPI MiCheckForContiguousMemory( IN PVOID BaseAddress, IN PFN_NUMBER BaseAddressPages, IN PFN_NUMBER SizeInPages, IN PFN_NUMBER LowestPfn, IN PFN_NUMBER HighestPfn, IN PFN_NUMBER BoundaryPfn, IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute ); PMDL NTAPI MiAllocatePagesForMdl( IN PHYSICAL_ADDRESS LowAddress, IN PHYSICAL_ADDRESS HighAddress, IN PHYSICAL_ADDRESS SkipBytes, IN SIZE_T TotalBytes, IN MI_PFN_CACHE_ATTRIBUTE CacheAttribute, IN ULONG Flags ); VOID NTAPI MiInsertPageInList( IN PMMPFNLIST ListHead, IN PFN_NUMBER PageFrameIndex ); VOID NTAPI MiUnlinkFreeOrZeroedPage( IN PMMPFN Entry ); VOID NTAPI MiUnlinkPageFromList( IN PMMPFN Pfn ); VOID NTAPI MiInitializePfn( IN PFN_NUMBER PageFrameIndex, IN PMMPTE PointerPte, IN BOOLEAN Modified ); NTSTATUS NTAPI MiInitializeAndChargePfn( OUT PPFN_NUMBER PageFrameIndex, IN PMMPDE PointerPde, IN PFN_NUMBER ContainingPageFrame, IN BOOLEAN SessionAllocation ); VOID NTAPI MiInitializePfnAndMakePteValid( IN PFN_NUMBER PageFrameIndex, IN PMMPTE PointerPte, IN MMPTE TempPte ); VOID NTAPI MiInitializePfnForOtherProcess( IN PFN_NUMBER PageFrameIndex, IN PVOID PteAddress, IN PFN_NUMBER PteFrame ); VOID NTAPI MiDecrementShareCount( IN PMMPFN Pfn1, IN PFN_NUMBER PageFrameIndex ); PFN_NUMBER NTAPI MiRemoveAnyPage( IN ULONG Color ); PFN_NUMBER NTAPI MiRemoveZeroPage( IN ULONG Color ); VOID NTAPI MiZeroPhysicalPage( IN PFN_NUMBER PageFrameIndex ); VOID NTAPI MiInsertPageInFreeList( IN PFN_NUMBER PageFrameIndex ); PFN_COUNT NTAPI MiDeleteSystemPageableVm( IN PMMPTE PointerPte, IN PFN_NUMBER PageCount, IN ULONG Flags, OUT PPFN_NUMBER ValidPages ); ULONG NTAPI MiGetPageProtection( IN PMMPTE PointerPte ); PLDR_DATA_TABLE_ENTRY NTAPI MiLookupDataTableEntry( IN PVOID Address ); INIT_FUNCTION VOID NTAPI MiInitializeDriverLargePageList( VOID ); INIT_FUNCTION VOID NTAPI MiInitializeLargePageSupport( VOID ); INIT_FUNCTION VOID NTAPI MiSyncCachedRanges( VOID ); BOOLEAN NTAPI MiIsPfnInUse( IN PMMPFN Pfn1 ); PMMVAD NTAPI MiLocateAddress( IN PVOID VirtualAddress ); TABLE_SEARCH_RESULT NTAPI MiCheckForConflictingNode( IN ULONG_PTR StartVpn, IN ULONG_PTR EndVpn, IN PMM_AVL_TABLE Table, OUT PMMADDRESS_NODE *NodeOrParent ); TABLE_SEARCH_RESULT NTAPI MiFindEmptyAddressRangeDownTree( IN SIZE_T Length, IN ULONG_PTR BoundaryAddress, IN ULONG_PTR Alignment, IN PMM_AVL_TABLE Table, OUT PULONG_PTR Base, OUT PMMADDRESS_NODE *Parent ); NTSTATUS NTAPI MiFindEmptyAddressRangeDownBasedTree( IN SIZE_T Length, IN ULONG_PTR BoundaryAddress, IN ULONG_PTR Alignment, IN PMM_AVL_TABLE Table, OUT PULONG_PTR Base ); TABLE_SEARCH_RESULT NTAPI MiFindEmptyAddressRangeInTree( IN SIZE_T Length, IN ULONG_PTR Alignment, IN PMM_AVL_TABLE Table, OUT PMMADDRESS_NODE *PreviousVad, OUT PULONG_PTR Base ); NTSTATUS NTAPI MiCheckSecuredVad( IN PMMVAD Vad, IN PVOID Base, IN SIZE_T Size, IN ULONG ProtectionMask ); VOID NTAPI MiInsertVad( _Inout_ PMMVAD Vad, _Inout_ PMM_AVL_TABLE VadRoot); NTSTATUS NTAPI MiInsertVadEx( _Inout_ PMMVAD Vad, _In_ ULONG_PTR *BaseAddress, _In_ SIZE_T ViewSize, _In_ ULONG_PTR HighestAddress, _In_ ULONG_PTR Alignment, _In_ ULONG AllocationType); VOID NTAPI MiInsertBasedSection( IN PSECTION Section ); NTSTATUS NTAPI MiUnmapViewOfSection( IN PEPROCESS Process, IN PVOID BaseAddress, IN ULONG Flags ); NTSTATUS NTAPI MiRosUnmapViewOfSection( IN PEPROCESS Process, IN PVOID BaseAddress, IN BOOLEAN SkipDebuggerNotify ); VOID NTAPI MiInsertNode( IN PMM_AVL_TABLE Table, IN PMMADDRESS_NODE NewNode, PMMADDRESS_NODE Parent, TABLE_SEARCH_RESULT Result ); VOID NTAPI MiRemoveNode( IN PMMADDRESS_NODE Node, IN PMM_AVL_TABLE Table ); PMMADDRESS_NODE NTAPI MiGetPreviousNode( IN PMMADDRESS_NODE Node ); PMMADDRESS_NODE NTAPI MiGetNextNode( IN PMMADDRESS_NODE Node ); BOOLEAN NTAPI MiInitializeSystemSpaceMap( IN PMMSESSION InputSession OPTIONAL ); VOID NTAPI MiSessionRemoveProcess( VOID ); VOID NTAPI MiReleaseProcessReferenceToSessionDataPage( IN PMM_SESSION_SPACE SessionGlobal ); VOID NTAPI MiSessionAddProcess( IN PEPROCESS NewProcess ); NTSTATUS NTAPI MiSessionCommitPageTables( IN PVOID StartVa, IN PVOID EndVa ); ULONG NTAPI MiMakeProtectionMask( IN ULONG Protect ); VOID NTAPI MiDeleteVirtualAddresses( IN ULONG_PTR Va, IN ULONG_PTR EndingAddress, IN PMMVAD Vad ); VOID NTAPI MiDeletePte( IN PMMPTE PointerPte, IN PVOID VirtualAddress, IN PEPROCESS CurrentProcess, IN PMMPTE PrototypePte ); ULONG NTAPI MiMakeSystemAddressValid( IN PVOID PageTableVirtualAddress, IN PEPROCESS CurrentProcess ); ULONG NTAPI MiMakeSystemAddressValidPfn( IN PVOID VirtualAddress, IN KIRQL OldIrql ); VOID NTAPI MiRemoveMappedView( IN PEPROCESS CurrentProcess, IN PMMVAD Vad ); PSUBSECTION NTAPI MiLocateSubsection( IN PMMVAD Vad, IN ULONG_PTR Vpn ); VOID NTAPI MiDeleteARM3Section( PVOID ObjectBody ); NTSTATUS NTAPI MiQueryMemorySectionName( IN HANDLE ProcessHandle, IN PVOID BaseAddress, OUT PVOID MemoryInformation, IN SIZE_T MemoryInformationLength, OUT PSIZE_T ReturnLength ); NTSTATUS NTAPI MiRosUnmapViewInSystemSpace( IN PVOID MappedBase ); POOL_TYPE NTAPI MmDeterminePoolType( IN PVOID PoolAddress ); VOID NTAPI MiMakePdeExistAndMakeValid( IN PMMPDE PointerPde, IN PEPROCESS TargetProcess, IN KIRQL OldIrql ); VOID NTAPI MiWriteProtectSystemImage( _In_ PVOID ImageBase); // // MiRemoveZeroPage will use inline code to zero out the page manually if only // free pages are available. In some scenarios, we don't/can't run that piece of // code and would rather only have a real zero page. If we can't have a zero page, // then we'd like to have our own code to grab a free page and zero it out, by // using MiRemoveAnyPage. This macro implements this. // FORCEINLINE PFN_NUMBER MiRemoveZeroPageSafe(IN ULONG Color) { if (MmFreePagesByColor[ZeroedPageList][Color].Flink != LIST_HEAD) return MiRemoveZeroPage(Color); return 0; } #if (_MI_PAGING_LEVELS == 2) FORCEINLINE BOOLEAN MiSynchronizeSystemPde(PMMPDE PointerPde) { MMPDE SystemPde; ULONG Index; /* Get the Index from the PDE */ Index = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE); /* Copy the PDE from the double-mapped system page directory */ SystemPde = MmSystemPagePtes[Index]; *PointerPde = SystemPde; /* Make sure we re-read the PDE and PTE */ KeMemoryBarrierWithoutFence(); /* Return, if we had success */ return SystemPde.u.Hard.Valid != 0; } #endif /* EOF */