From d8cdb89fb03006595dc40ac23db5267b8d9d9c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Gardou?= Date: Fri, 6 Nov 2020 09:39:31 +0100 Subject: [PATCH] [NTOSKRNL] Overhaul Cc and Mm relationship Previously, when creating a file section, Mm requested Cc to cache the file, then Cc would request pages from Mm, then Mm would request them back to serve its file-mapping role Now, Mm does it all by itself. If file cahcing is requested by the FS driver, then Cc creates a file mapping and uses that to serve its purpose. This is a rewrite of Cc --- ntoskrnl/cache/section/sptab.c | 3 - ntoskrnl/cc/copy.c | 561 +++------ ntoskrnl/cc/fs.c | 58 +- ntoskrnl/cc/pin.c | 236 ++-- ntoskrnl/cc/view.c | 495 +++----- ntoskrnl/ex/init.c | 2 - ntoskrnl/ex/sysinfo.c | 12 +- ntoskrnl/include/internal/cc.h | 30 +- ntoskrnl/include/internal/mm.h | 82 +- ntoskrnl/mm/ARM3/section.c | 32 +- ntoskrnl/mm/ARM3/virtual.c | 8 +- ntoskrnl/mm/ARM3/zeropage.c | 2 +- ntoskrnl/mm/balance.c | 12 - ntoskrnl/mm/pagefile.c | 5 +- ntoskrnl/mm/rmap.c | 317 ++--- ntoskrnl/mm/section.c | 1986 +++++++++++++------------------- ntoskrnl/po/power.c | 4 +- 17 files changed, 1499 insertions(+), 2346 deletions(-) diff --git a/ntoskrnl/cache/section/sptab.c b/ntoskrnl/cache/section/sptab.c index 454fb25af42..7a2481a4752 100644 --- a/ntoskrnl/cache/section/sptab.c +++ b/ntoskrnl/cache/section/sptab.c @@ -187,9 +187,6 @@ _MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, ASSERT(Segment->Locked); ASSERT(!IS_SWAP_FROM_SSE(Entry) || !IS_DIRTY_SSE(Entry)); - if (Entry && !IS_SWAP_FROM_SSE(Entry)) - MmGetRmapListHeadPage(PFN_FROM_SSE(Entry)); - PageTable = MiSectionPageTableGetOrAllocate(&Segment->PageTable, Offset); if (!PageTable) return STATUS_NO_MEMORY; diff --git a/ntoskrnl/cc/copy.c b/ntoskrnl/cc/copy.c index 1f731716b1f..5bb14cc210a 100644 --- a/ntoskrnl/cc/copy.c +++ b/ntoskrnl/cc/copy.c @@ -20,13 +20,6 @@ static PFN_NUMBER CcZeroPage = 0; #define MAX_ZERO_LENGTH (256 * 1024) -typedef enum _CC_COPY_OPERATION -{ - CcOperationRead, - CcOperationWrite, - CcOperationZero -} CC_COPY_OPERATION; - typedef enum _CC_CAN_WRITE_RETRY { FirstTry = 0, @@ -35,7 +28,7 @@ typedef enum _CC_CAN_WRITE_RETRY RetryMasterLocked = 255, } CC_CAN_WRITE_RETRY; -ULONG CcRosTraceLevel = 0; +ULONG CcRosTraceLevel = CC_API_DEBUG; ULONG CcFastMdlReadWait; ULONG CcFastMdlReadNotPossible; ULONG CcFastReadNotPossible; @@ -76,338 +69,6 @@ CcInitCacheZeroPage ( MiZeroPhysicalPage(CcZeroPage); } -NTSTATUS -NTAPI -CcReadVirtualAddress ( - PROS_VACB Vacb) -{ - ULONG Size; - PMDL Mdl; - NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - ULARGE_INTEGER LargeSize; - - LargeSize.QuadPart = Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; - if (LargeSize.QuadPart > VACB_MAPPING_GRANULARITY) - { - LargeSize.QuadPart = VACB_MAPPING_GRANULARITY; - } - Size = LargeSize.LowPart; - - Size = ROUND_TO_PAGES(Size); - ASSERT(Size <= VACB_MAPPING_GRANULARITY); - ASSERT(Size > 0); - - Mdl = IoAllocateMdl(Vacb->BaseAddress, Size, FALSE, FALSE, NULL); - if (!Mdl) - { - return STATUS_INSUFFICIENT_RESOURCES; - } - - Status = STATUS_SUCCESS; - _SEH2_TRY - { - MmProbeAndLockPages(Mdl, KernelMode, IoWriteAccess); - } - _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - DPRINT1("MmProbeAndLockPages failed with: %lx for %p (%p, %p)\n", Status, Mdl, Vacb, Vacb->BaseAddress); - KeBugCheck(CACHE_MANAGER); - } _SEH2_END; - - if (NT_SUCCESS(Status)) - { - Mdl->MdlFlags |= MDL_IO_PAGE_READ; - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoPageRead(Vacb->SharedCacheMap->FileObject, Mdl, &Vacb->FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } - - MmUnlockPages(Mdl); - } - - IoFreeMdl(Mdl); - - if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE)) - { - DPRINT1("IoPageRead failed, Status %x\n", Status); - return Status; - } - - if (Size < VACB_MAPPING_GRANULARITY) - { - RtlZeroMemory((char*)Vacb->BaseAddress + Size, - VACB_MAPPING_GRANULARITY - Size); - } - - return STATUS_SUCCESS; -} - -NTSTATUS -NTAPI -CcWriteVirtualAddress ( - PROS_VACB Vacb) -{ - ULONG Size; - PMDL Mdl; - NTSTATUS Status; - IO_STATUS_BLOCK IoStatus; - KEVENT Event; - ULARGE_INTEGER LargeSize; - - LargeSize.QuadPart = Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; - if (LargeSize.QuadPart > VACB_MAPPING_GRANULARITY) - { - LargeSize.QuadPart = VACB_MAPPING_GRANULARITY; - } - Size = LargeSize.LowPart; - // - // Nonpaged pool PDEs in ReactOS must actually be synchronized between the - // MmGlobalPageDirectory and the real system PDE directory. What a mess... - // - { - ULONG i = 0; - do - { - MmGetPfnForProcess(NULL, (PVOID)((ULONG_PTR)Vacb->BaseAddress + (i << PAGE_SHIFT))); - } while (++i < (Size >> PAGE_SHIFT)); - } - - ASSERT(Size <= VACB_MAPPING_GRANULARITY); - ASSERT(Size > 0); - - Mdl = IoAllocateMdl(Vacb->BaseAddress, Size, FALSE, FALSE, NULL); - if (!Mdl) - { - return STATUS_INSUFFICIENT_RESOURCES; - } - - Status = STATUS_SUCCESS; - _SEH2_TRY - { - MmProbeAndLockPages(Mdl, KernelMode, IoReadAccess); - } - _SEH2_EXCEPT (EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - DPRINT1("MmProbeAndLockPages failed with: %lx for %p (%p, %p)\n", Status, Mdl, Vacb, Vacb->BaseAddress); - KeBugCheck(CACHE_MANAGER); - } _SEH2_END; - - if (NT_SUCCESS(Status)) - { - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoSynchronousPageWrite(Vacb->SharedCacheMap->FileObject, Mdl, &Vacb->FileOffset, &Event, &IoStatus); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = IoStatus.Status; - } - - MmUnlockPages(Mdl); - } - IoFreeMdl(Mdl); - if (!NT_SUCCESS(Status) && (Status != STATUS_END_OF_FILE)) - { - DPRINT1("IoPageWrite failed, Status %x\n", Status); - return Status; - } - - return STATUS_SUCCESS; -} - -NTSTATUS -ReadWriteOrZero( - _Inout_ PVOID BaseAddress, - _Inout_opt_ PVOID Buffer, - _In_ ULONG Length, - _In_ CC_COPY_OPERATION Operation) -{ - NTSTATUS Status = STATUS_SUCCESS; - - if (Operation == CcOperationZero) - { - /* Zero */ - RtlZeroMemory(BaseAddress, Length); - } - else - { - _SEH2_TRY - { - if (Operation == CcOperationWrite) - RtlCopyMemory(BaseAddress, Buffer, Length); - else - RtlCopyMemory(Buffer, BaseAddress, Length); - } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - Status = _SEH2_GetExceptionCode(); - } - _SEH2_END; - } - return Status; -} - -BOOLEAN -CcCopyData ( - _In_ PFILE_OBJECT FileObject, - _In_ LONGLONG FileOffset, - _Inout_ PVOID Buffer, - _In_ LONGLONG Length, - _In_ CC_COPY_OPERATION Operation, - _In_ BOOLEAN Wait, - _Out_ PIO_STATUS_BLOCK IoStatus) -{ - NTSTATUS Status; - LONGLONG CurrentOffset; - ULONG BytesCopied; - KIRQL OldIrql; - PROS_SHARED_CACHE_MAP SharedCacheMap; - PLIST_ENTRY ListEntry; - PROS_VACB Vacb; - ULONG PartialLength; - PVOID BaseAddress; - BOOLEAN Valid; - PPRIVATE_CACHE_MAP PrivateCacheMap; - - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - PrivateCacheMap = FileObject->PrivateCacheMap; - CurrentOffset = FileOffset; - BytesCopied = 0; - - if (!Wait) - { - /* test if the requested data is available */ - KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &OldIrql); - /* FIXME: this loop doesn't take into account areas that don't have - * a VACB in the list yet */ - ListEntry = SharedCacheMap->CacheMapVacbListHead.Flink; - while (ListEntry != &SharedCacheMap->CacheMapVacbListHead) - { - Vacb = CONTAINING_RECORD(ListEntry, - ROS_VACB, - CacheMapVacbListEntry); - ListEntry = ListEntry->Flink; - if (!Vacb->Valid && - DoRangesIntersect(Vacb->FileOffset.QuadPart, - VACB_MAPPING_GRANULARITY, - CurrentOffset, Length)) - { - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); - /* data not available */ - return FALSE; - } - if (Vacb->FileOffset.QuadPart >= CurrentOffset + Length) - break; - } - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); - } - - PartialLength = CurrentOffset % VACB_MAPPING_GRANULARITY; - if (PartialLength != 0) - { - PartialLength = min(Length, VACB_MAPPING_GRANULARITY - PartialLength); - Status = CcRosRequestVacb(SharedCacheMap, - ROUND_DOWN(CurrentOffset, - VACB_MAPPING_GRANULARITY), - &BaseAddress, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - ExRaiseStatus(Status); - if (!Valid) - { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - ExRaiseStatus(Status); - } - } - Status = ReadWriteOrZero((PUCHAR)BaseAddress + CurrentOffset % VACB_MAPPING_GRANULARITY, - Buffer, - PartialLength, - Operation); - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, Operation != CcOperationRead, FALSE); - - if (!NT_SUCCESS(Status)) - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - - Length -= PartialLength; - CurrentOffset += PartialLength; - BytesCopied += PartialLength; - - if (Operation != CcOperationZero) - Buffer = (PVOID)((ULONG_PTR)Buffer + PartialLength); - } - - while (Length > 0) - { - ASSERT(CurrentOffset % VACB_MAPPING_GRANULARITY == 0); - PartialLength = min(VACB_MAPPING_GRANULARITY, Length); - Status = CcRosRequestVacb(SharedCacheMap, - CurrentOffset, - &BaseAddress, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - ExRaiseStatus(Status); - if (!Valid && - (Operation == CcOperationRead || - PartialLength < VACB_MAPPING_GRANULARITY)) - { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - ExRaiseStatus(Status); - } - } - Status = ReadWriteOrZero(BaseAddress, Buffer, PartialLength, Operation); - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, Operation != CcOperationRead, FALSE); - - if (!NT_SUCCESS(Status)) - ExRaiseStatus(STATUS_INVALID_USER_BUFFER); - - Length -= PartialLength; - CurrentOffset += PartialLength; - BytesCopied += PartialLength; - - if (Operation != CcOperationZero) - Buffer = (PVOID)((ULONG_PTR)Buffer + PartialLength); - } - - /* If that was a successful sync read operation, let's handle read ahead */ - if (Operation == CcOperationRead && Length == 0 && Wait) - { - /* If file isn't random access and next read may get us cross VACB boundary, - * schedule next read - */ - if (!BooleanFlagOn(FileObject->Flags, FO_RANDOM_ACCESS) && - (CurrentOffset - 1) / VACB_MAPPING_GRANULARITY != (CurrentOffset + BytesCopied - 1) / VACB_MAPPING_GRANULARITY) - { - CcScheduleReadAhead(FileObject, (PLARGE_INTEGER)&FileOffset, BytesCopied); - } - - /* And update read history in private cache map */ - PrivateCacheMap->FileOffset1.QuadPart = PrivateCacheMap->FileOffset2.QuadPart; - PrivateCacheMap->BeyondLastByte1.QuadPart = PrivateCacheMap->BeyondLastByte2.QuadPart; - PrivateCacheMap->FileOffset2.QuadPart = FileOffset; - PrivateCacheMap->BeyondLastByte2.QuadPart = FileOffset + BytesCopied; - } - - IoStatus->Status = STATUS_SUCCESS; - IoStatus->Information = BytesCopied; - return TRUE; -} - VOID CcPostDeferredWrites(VOID) { @@ -492,8 +153,6 @@ CcPerformReadAhead( PROS_SHARED_CACHE_MAP SharedCacheMap; PROS_VACB Vacb; ULONG PartialLength; - PVOID BaseAddress; - BOOLEAN Valid; ULONG Length; PPRIVATE_CACHE_MAP PrivateCacheMap; BOOLEAN Locked; @@ -556,10 +215,7 @@ CcPerformReadAhead( { PartialLength = min(Length, VACB_MAPPING_GRANULARITY - PartialLength); Status = CcRosRequestVacb(SharedCacheMap, - ROUND_DOWN(CurrentOffset, - VACB_MAPPING_GRANULARITY), - &BaseAddress, - &Valid, + ROUND_DOWN(CurrentOffset, VACB_MAPPING_GRANULARITY), &Vacb); if (!NT_SUCCESS(Status)) { @@ -567,15 +223,13 @@ CcPerformReadAhead( goto Clear; } - if (!Valid) + Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, + CurrentOffset % VACB_MAPPING_GRANULARITY, PartialLength); + if (!NT_SUCCESS(Status)) { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - DPRINT1("Failed to read data: %lx!\n", Status); - goto Clear; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); + DPRINT1("Failed to read data: %lx!\n", Status); + goto Clear; } CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); @@ -590,8 +244,6 @@ CcPerformReadAhead( PartialLength = min(VACB_MAPPING_GRANULARITY, Length); Status = CcRosRequestVacb(SharedCacheMap, CurrentOffset, - &BaseAddress, - &Valid, &Vacb); if (!NT_SUCCESS(Status)) { @@ -599,15 +251,12 @@ CcPerformReadAhead( goto Clear; } - if (!Valid) + Status = CcRosEnsureVacbResident(Vacb, TRUE, FALSE, 0, PartialLength); + if (!NT_SUCCESS(Status)) { - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - DPRINT1("Failed to read data: %lx!\n", Status); - goto Clear; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); + DPRINT1("Failed to read data: %lx!\n", Status); + goto Clear; } CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); @@ -811,6 +460,12 @@ CcCopyRead ( OUT PVOID Buffer, OUT PIO_STATUS_BLOCK IoStatus) { + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; + NTSTATUS Status; + LONGLONG CurrentOffset; + LONGLONG ReadEnd = FileOffset->QuadPart + Length; + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d\n", FileObject, FileOffset->QuadPart, Length, Wait); @@ -819,13 +474,58 @@ CcCopyRead ( FileObject, FileOffset->QuadPart, Length, Wait, Buffer, IoStatus); - return CcCopyData(FileObject, - FileOffset->QuadPart, - Buffer, - Length, - CcOperationRead, - Wait, - IoStatus); + if (!SharedCacheMap) + return FALSE; + + /* Documented to ASSERT, but KMTests test this case... */ + // ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); + + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = 0; + + CurrentOffset = FileOffset->QuadPart; + while(CurrentOffset < ReadEnd) + { + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + SIZE_T CopyLength = VacbLength; + + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + return FALSE; + + /* Do not copy past the section */ + if (CurrentOffset + VacbLength > SharedCacheMap->SectionSize.QuadPart) + CopyLength = SharedCacheMap->SectionSize.QuadPart - CurrentOffset; + if (CopyLength != 0) + RtlCopyMemory(Buffer, (PUCHAR)Vacb->BaseAddress + VacbOffset, CopyLength); + + /* Zero-out the buffer tail if needed */ + if (CopyLength < VacbLength) + RtlZeroMemory((PUCHAR)Buffer + CopyLength, VacbLength - CopyLength); + + IoStatus->Information += VacbLength; + + Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); + CurrentOffset += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + } + _SEH2_END; + } + + return TRUE; } /* @@ -840,7 +540,11 @@ CcCopyWrite ( IN BOOLEAN Wait, IN PVOID Buffer) { - IO_STATUS_BLOCK IoStatus; + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; + NTSTATUS Status; + LONGLONG CurrentOffset; + LONGLONG WriteEnd = FileOffset->QuadPart + Length; CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%I64d Length=%lu Wait=%d Buffer=%p\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); @@ -849,13 +553,48 @@ CcCopyWrite ( "Length %lu, Wait %u, Buffer 0x%p)\n", FileObject, FileOffset->QuadPart, Length, Wait, Buffer); - return CcCopyData(FileObject, - FileOffset->QuadPart, - Buffer, - Length, - CcOperationWrite, - Wait, - &IoStatus); + if (!SharedCacheMap) + return FALSE; + + /* FIXME: Honor FileObject FO_WRITE_THROUGH flag */ + + ASSERT((FileOffset->QuadPart + Length) <= SharedCacheMap->FileSize.QuadPart); + + CurrentOffset = FileOffset->QuadPart; + while(CurrentOffset < WriteEnd) + { + ULONG VacbOffset = CurrentOffset % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + + Status = CcRosGetVacb(SharedCacheMap, CurrentOffset, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; + } + + RtlCopyMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), Buffer, VacbLength); + + Buffer = (PVOID)((ULONG_PTR)Buffer + VacbLength); + CurrentOffset += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, TRUE, FALSE); + } + _SEH2_END; + } + + return TRUE; } /* @@ -999,11 +738,8 @@ CcZeroData ( NTSTATUS Status; LARGE_INTEGER WriteOffset; LONGLONG Length; - ULONG CurrentLength; - PMDL Mdl; - ULONG i; - IO_STATUS_BLOCK Iosb; - KEVENT Event; + PROS_VACB Vacb; + PROS_SHARED_CACHE_MAP SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; CCTRACE(CC_API_DEBUG, "FileObject=%p StartOffset=%I64u EndOffset=%I64u Wait=%d\n", FileObject, StartOffset->QuadPart, EndOffset->QuadPart, Wait); @@ -1015,9 +751,14 @@ CcZeroData ( Length = EndOffset->QuadPart - StartOffset->QuadPart; WriteOffset.QuadPart = StartOffset->QuadPart; - if (FileObject->SectionObjectPointer->SharedCacheMap == NULL) + if (!SharedCacheMap || (FileObject->Flags & FO_WRITE_THROUGH)) { - /* File is not cached */ + /* Make this a non-cached write */ + IO_STATUS_BLOCK Iosb; + KEVENT Event; + PMDL Mdl; + ULONG i; + ULONG CurrentLength; Mdl = _alloca(MmSizeOfMdl(NULL, MAX_ZERO_LENGTH)); @@ -1032,7 +773,7 @@ CcZeroData ( CurrentLength = Length; } MmInitializeMdl(Mdl, (PVOID)(ULONG_PTR)WriteOffset.QuadPart, CurrentLength); - Mdl->MdlFlags |= (MDL_PAGES_LOCKED | MDL_IO_PAGE_READ); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; for (i = 0; i < ((Mdl->Size - sizeof(MDL)) / sizeof(ULONG)); i++) { ((PPFN_NUMBER)(Mdl + 1))[i] = CcZeroPage; @@ -1055,18 +796,42 @@ CcZeroData ( WriteOffset.QuadPart += CurrentLength; Length -= CurrentLength; } - } - else - { - IO_STATUS_BLOCK IoStatus; - return CcCopyData(FileObject, - WriteOffset.QuadPart, - NULL, - Length, - CcOperationZero, - Wait, - &IoStatus); + return TRUE; + } + + ASSERT(EndOffset->QuadPart <= SharedCacheMap->FileSize.QuadPart); + + while(WriteOffset.QuadPart < EndOffset->QuadPart) + { + ULONG VacbOffset = WriteOffset.QuadPart % VACB_MAPPING_GRANULARITY; + ULONG VacbLength = min(Length, VACB_MAPPING_GRANULARITY - VacbOffset); + + Status = CcRosGetVacb(SharedCacheMap, WriteOffset.QuadPart, &Vacb); + if (!NT_SUCCESS(Status)) + { + ExRaiseStatus(Status); + return FALSE; + } + + _SEH2_TRY + { + if (!CcRosEnsureVacbResident(Vacb, Wait, FALSE, VacbOffset, VacbLength)) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; + } + + RtlZeroMemory((PVOID)((ULONG_PTR)Vacb->BaseAddress + VacbOffset), VacbLength); + + WriteOffset.QuadPart += VacbLength; + Length -= VacbLength; + } + _SEH2_FINALLY + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, TRUE, FALSE); + } + _SEH2_END; } return TRUE; diff --git a/ntoskrnl/cc/fs.c b/ntoskrnl/cc/fs.c index 5798f56a91a..5a74ecee48c 100644 --- a/ntoskrnl/cc/fs.c +++ b/ntoskrnl/cc/fs.c @@ -10,13 +10,10 @@ /* INCLUDES ******************************************************************/ #include + #define NDEBUG #include -/* GLOBALS *****************************************************************/ - -NTSTATUS CcRosInternalFreeVacb(PROS_VACB Vacb); - /* FUNCTIONS *****************************************************************/ /* @@ -272,8 +269,9 @@ CcSetFileSizes ( IN PFILE_OBJECT FileObject, IN PCC_FILE_SIZES FileSizes) { - KIRQL oldirql; + KIRQL OldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + LARGE_INTEGER OldSectionSize; CCTRACE(CC_API_DEBUG, "FileObject=%p FileSizes=%p\n", FileObject, FileSizes); @@ -294,7 +292,14 @@ CcSetFileSizes ( if (SharedCacheMap == NULL) return; - if (FileSizes->AllocationSize.QuadPart < SharedCacheMap->SectionSize.QuadPart) + /* Update the relevant fields */ + KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &OldIrql); + OldSectionSize = SharedCacheMap->SectionSize; + SharedCacheMap->SectionSize = FileSizes->AllocationSize; + SharedCacheMap->FileSize = FileSizes->FileSize; + KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, OldIrql); + + if (FileSizes->AllocationSize.QuadPart < OldSectionSize.QuadPart) { CcPurgeCacheSection(FileObject->SectionObjectPointer, &FileSizes->AllocationSize, @@ -303,46 +308,9 @@ CcSetFileSizes ( } else { - PROS_VACB LastVacb; - - /* - * If file (allocation) size has increased, then we need to check whether - * it just grows in a single VACB (the last one). - * If so, we must mark the VACB as invalid to trigger a read to the - * FSD at the next VACB usage, and thus avoid returning garbage - */ - - /* Check for allocation size and the last VACB */ - if (SharedCacheMap->SectionSize.QuadPart < FileSizes->AllocationSize.QuadPart && - SharedCacheMap->SectionSize.QuadPart % VACB_MAPPING_GRANULARITY) - { - LastVacb = CcRosLookupVacb(SharedCacheMap, - SharedCacheMap->SectionSize.QuadPart); - if (LastVacb != NULL) - { - /* Mark it as invalid */ - CcRosReleaseVacb(SharedCacheMap, LastVacb, LastVacb->Dirty ? LastVacb->Valid : FALSE, FALSE, FALSE); - } - } - - /* Check for file size and the last VACB */ - if (SharedCacheMap->FileSize.QuadPart < FileSizes->FileSize.QuadPart && - SharedCacheMap->FileSize.QuadPart % VACB_MAPPING_GRANULARITY) - { - LastVacb = CcRosLookupVacb(SharedCacheMap, - SharedCacheMap->FileSize.QuadPart); - if (LastVacb != NULL) - { - /* Mark it as invalid */ - CcRosReleaseVacb(SharedCacheMap, LastVacb, LastVacb->Dirty ? LastVacb->Valid : FALSE, FALSE, FALSE); - } - } + /* Extend our section object */ + MmExtendSection(SharedCacheMap->Section, &SharedCacheMap->SectionSize); } - - KeAcquireSpinLock(&SharedCacheMap->CacheMapLock, &oldirql); - SharedCacheMap->SectionSize = FileSizes->AllocationSize; - SharedCacheMap->FileSize = FileSizes->FileSize; - KeReleaseSpinLock(&SharedCacheMap->CacheMapLock, oldirql); } /* diff --git a/ntoskrnl/cc/pin.c b/ntoskrnl/cc/pin.c index b839e71dafd..189ba480305 100644 --- a/ntoskrnl/cc/pin.c +++ b/ntoskrnl/cc/pin.c @@ -67,91 +67,6 @@ CcpFindBcb( return (Found ? Bcb : NULL); } -static -BOOLEAN -NTAPI -CcpMapData( - IN PROS_SHARED_CACHE_MAP SharedCacheMap, - IN PLARGE_INTEGER FileOffset, - IN ULONG Length, - IN ULONG Flags, - OUT PROS_VACB *pVacb, - OUT PVOID *pBuffer) -{ - LONGLONG ReadOffset, BaseOffset; - BOOLEAN Valid; - PROS_VACB Vacb; - NTSTATUS Status; - LONGLONG ROffset; - - ReadOffset = FileOffset->QuadPart; - - DPRINT("SectionSize %I64x, FileSize %I64x\n", - SharedCacheMap->SectionSize.QuadPart, - SharedCacheMap->FileSize.QuadPart); - - if (ReadOffset % VACB_MAPPING_GRANULARITY + Length > VACB_MAPPING_GRANULARITY) - { - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - return FALSE; - } - - if (!BooleanFlagOn(Flags, MAP_NO_READ)) - { - static int Warned = 0; - - SetFlag(Flags, MAP_NO_READ); - if (!Warned) - { - DPRINT1("Mapping/pinning with no read not implemented. Forcing read, might fail if wait not allowed\n"); - Warned++; - } - } - - /* Properly round offset and call internal helper for getting a VACB */ - ROffset = ROUND_DOWN(ReadOffset, VACB_MAPPING_GRANULARITY); - Status = CcRosGetVacb(SharedCacheMap, - ROffset, - &BaseOffset, - pBuffer, - &Valid, - &Vacb); - if (!NT_SUCCESS(Status)) - { - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - ExRaiseStatus(Status); - return FALSE; - } - - if (!Valid && BooleanFlagOn(Flags, MAP_NO_READ)) - { - if (!BooleanFlagOn(Flags, MAP_WAIT)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - return FALSE; - } - - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", - SharedCacheMap->FileObject, FileOffset, Length, Flags); - ExRaiseStatus(Status); - return FALSE; - } - } - - *pBuffer = (PUCHAR)*pBuffer + ReadOffset % VACB_MAPPING_GRANULARITY; - *pVacb = Vacb; - - return TRUE; -} - static VOID CcpDereferenceBcb( @@ -304,44 +219,44 @@ CcpPinData( OUT PVOID * Buffer) { PINTERNAL_BCB NewBcb; - BOOLEAN Result; - PROS_VACB Vacb; KIRQL OldIrql; - ULONG MapFlags; + ULONG VacbOffset; + NTSTATUS Status; + BOOLEAN Result; + + VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); + /* This seems to be valid, according to KMTests */ + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + Length = VACB_MAPPING_GRANULARITY - VacbOffset; KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); NewBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, TRUE); if (NewBcb != NULL) { + BOOLEAN Result; + ++NewBcb->RefCount; KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); if (BooleanFlagOn(Flags, PIN_EXCLUSIVE)) - { Result = ExAcquireResourceExclusiveLite(&NewBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT)); - } else - { Result = ExAcquireSharedStarveExclusive(&NewBcb->Lock, BooleanFlagOn(Flags, PIN_WAIT)); - } if (!Result) { CcpDereferenceBcb(SharedCacheMap, NewBcb); - NewBcb = NULL; - } - else - { - NewBcb->PinCount++; - *Bcb = NewBcb; - *Buffer = (PUCHAR)NewBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY; + return FALSE; } - return Result; + NewBcb->PinCount++; } else { + LONGLONG ROffset; + PROS_VACB Vacb; + KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); if (BooleanFlagOn(Flags, PIN_IF_BCB)) @@ -349,29 +264,49 @@ CcpPinData( return FALSE; } - MapFlags = Flags & PIN_WAIT; - if (BooleanFlagOn(Flags, PIN_NO_READ)) + /* Properly round offset and call internal helper for getting a VACB */ + ROffset = ROUND_DOWN(FileOffset->QuadPart, VACB_MAPPING_GRANULARITY); + Status = CcRosGetVacb(SharedCacheMap, ROffset, &Vacb); + if (!NT_SUCCESS(Status)) { - SetFlag(MapFlags, MAP_NO_READ); + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + ExRaiseStatus(Status); + return FALSE; } - Result = CcpMapData(SharedCacheMap, FileOffset, Length, MapFlags, &Vacb, Buffer); - if (Result) + NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); + if (NewBcb == NULL) { - NewBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, Flags, TRUE); - if (NewBcb == NULL) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Result = FALSE; - } - else - { - *Bcb = NewBcb; - } + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + return FALSE; } } - return Result; + Result = FALSE; + _SEH2_TRY + { + /* Ensure the pages are resident */ + Result = CcRosEnsureVacbResident(NewBcb->Vacb, + BooleanFlagOn(Flags, PIN_WAIT), + BooleanFlagOn(Flags, PIN_NO_READ), + VacbOffset, Length); + } + _SEH2_FINALLY + { + if (!Result) + { + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + CcUnpinData(NewBcb); + return FALSE; + } + } + _SEH2_END; + + *Bcb = NewBcb; + *Buffer = (PVOID)((ULONG_PTR)NewBcb->Vacb->BaseAddress + VacbOffset); + return TRUE; } /* @@ -387,13 +322,15 @@ CcMapData ( OUT PVOID *pBcb, OUT PVOID *pBuffer) { - BOOLEAN Ret; KIRQL OldIrql; PINTERNAL_BCB iBcb; PROS_VACB Vacb; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG VacbOffset; + NTSTATUS Status; + BOOLEAN Result; - DPRINT("CcMapData(FileObject 0x%p, FileOffset %I64x, Length %lu, Flags 0x%lx," + CCTRACE(CC_API_DEBUG, "CcMapData(FileObject 0x%p, FileOffset 0x%I64x, Length %lu, Flags 0x%lx," " pBcb 0x%p, pBuffer 0x%p)\n", FileObject, FileOffset->QuadPart, Length, Flags, pBcb, pBuffer); @@ -413,6 +350,11 @@ CcMapData ( ++CcMapDataNoWait; } + VacbOffset = (ULONG)(FileOffset->QuadPart % VACB_MAPPING_GRANULARITY); + /* KMTests seem to show that it is allowed to call accross mapping granularity */ + if ((VacbOffset + Length) > VACB_MAPPING_GRANULARITY) + Length = VACB_MAPPING_GRANULARITY - VacbOffset; + KeAcquireSpinLock(&SharedCacheMap->BcbSpinLock, &OldIrql); iBcb = CcpFindBcb(SharedCacheMap, FileOffset, Length, FALSE); @@ -420,34 +362,54 @@ CcMapData ( { KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - Ret = CcpMapData(SharedCacheMap, FileOffset, Length, Flags, &Vacb, pBuffer); - if (Ret) + /* Call internal helper for getting a VACB */ + Status = CcRosGetVacb(SharedCacheMap, FileOffset->QuadPart, &Vacb); + if (!NT_SUCCESS(Status)) { - iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); - if (iBcb == NULL) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Ret = FALSE; - } - else - { - *pBcb = iBcb; - } + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + ExRaiseStatus(Status); + return FALSE; + } + + iBcb = CcpGetAppropriateBcb(SharedCacheMap, Vacb, FileOffset, Length, 0, FALSE); + if (iBcb == NULL) + { + CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> FALSE\n", + SharedCacheMap->FileObject, FileOffset, Length, Flags); + return FALSE; } } else { ++iBcb->RefCount; KeReleaseSpinLock(&SharedCacheMap->BcbSpinLock, OldIrql); - - *pBcb = iBcb; - *pBuffer = (PUCHAR)iBcb->Vacb->BaseAddress + FileOffset->QuadPart % VACB_MAPPING_GRANULARITY; - Ret = TRUE; } - CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> %d Bcb=%p\n", - FileObject, FileOffset, Length, Flags, Ret, *pBcb); - return Ret; + _SEH2_TRY + { + Result = FALSE; + /* Ensure the pages are resident */ + Result = CcRosEnsureVacbResident(iBcb->Vacb, BooleanFlagOn(Flags, MAP_WAIT), + BooleanFlagOn(Flags, MAP_NO_READ), VacbOffset, Length); + } + _SEH2_FINALLY + { + if (!Result) + { + CcpDereferenceBcb(SharedCacheMap, iBcb); + return FALSE; + } + } + _SEH2_END; + + *pBcb = iBcb; + *pBuffer = (PVOID)((ULONG_PTR)iBcb->Vacb->BaseAddress + VacbOffset); + + CCTRACE(CC_API_DEBUG, "FileObject=%p FileOffset=%p Length=%lu Flags=0x%lx -> TRUE Bcb=%p, Buffer %p\n", + FileObject, FileOffset, Length, Flags, *pBcb, *pBuffer); + return Result; } /* diff --git a/ntoskrnl/cc/view.c b/ntoskrnl/cc/view.c index adb28bb84ab..ddbe77e22af 100644 --- a/ntoskrnl/cc/view.c +++ b/ntoskrnl/cc/view.c @@ -156,20 +156,29 @@ CcRosTraceCacheMap ( #endif } +NTSTATUS +NTAPI +MmFlushVirtualMemory(IN PEPROCESS Process, + IN OUT PVOID *BaseAddress, + IN OUT PSIZE_T RegionSize, + OUT PIO_STATUS_BLOCK IoStatusBlock); + NTSTATUS NTAPI CcRosFlushVacb ( PROS_VACB Vacb) { + IO_STATUS_BLOCK Iosb; + SIZE_T FlushSize = min(VACB_MAPPING_GRANULARITY, + Vacb->SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart); NTSTATUS Status; CcRosUnmarkDirtyVacb(Vacb, TRUE); - Status = CcWriteVirtualAddress(Vacb); + Status = MmFlushVirtualMemory(NULL, &Vacb->BaseAddress, &FlushSize, &Iosb); + if (!NT_SUCCESS(Status)) - { CcRosMarkDirtyVacb(Vacb); - } return Status; } @@ -234,6 +243,8 @@ CcRosFlushDirtyPages ( current->SharedCacheMap->LazyWriteContext, Wait); if (!Locked) { + DPRINT("Not locked!"); + ASSERT(!Wait); OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); CcRosVacbDecRefCount(current); continue; @@ -264,15 +275,18 @@ CcRosFlushDirtyPages ( PagesFreed = VACB_MAPPING_GRANULARITY / PAGE_SIZE; (*Count) += PagesFreed; - /* Make sure we don't overflow target! */ - if (Target < PagesFreed) + if (!Wait) { - /* If we would have, jump to zero directly */ - Target = 0; - } - else - { - Target -= PagesFreed; + /* Make sure we don't overflow target! */ + if (Target < PagesFreed) + { + /* If we would have, jump to zero directly */ + Target = 0; + } + else + { + Target -= PagesFreed; + } } } @@ -286,136 +300,6 @@ CcRosFlushDirtyPages ( return STATUS_SUCCESS; } -NTSTATUS -CcRosTrimCache ( - ULONG Target, - ULONG Priority, - PULONG NrFreed) -/* - * FUNCTION: Try to free some memory from the file cache. - * ARGUMENTS: - * Target - The number of pages to be freed. - * Priority - The priority of free (currently unused). - * NrFreed - Points to a variable where the number of pages - * actually freed is returned. - */ -{ - PLIST_ENTRY current_entry; - PROS_VACB current; - ULONG PagesFreed; - KIRQL oldIrql; - LIST_ENTRY FreeList; - PFN_NUMBER Page; - ULONG i; - BOOLEAN FlushedPages = FALSE; - - DPRINT("CcRosTrimCache(Target %lu)\n", Target); - - InitializeListHead(&FreeList); - - *NrFreed = 0; - -retry: - oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - - current_entry = VacbLruListHead.Flink; - while (current_entry != &VacbLruListHead) - { - ULONG Refs; - - current = CONTAINING_RECORD(current_entry, - ROS_VACB, - VacbLruListEntry); - current_entry = current_entry->Flink; - - KeAcquireSpinLockAtDpcLevel(¤t->SharedCacheMap->CacheMapLock); - - /* Reference the VACB */ - CcRosVacbIncRefCount(current); - - /* Check if it's mapped and not dirty */ - if (InterlockedCompareExchange((PLONG)¤t->MappedCount, 0, 0) > 0 && !current->Dirty) - { - /* We have to break these locks because Cc sucks */ - KeReleaseSpinLockFromDpcLevel(¤t->SharedCacheMap->CacheMapLock); - KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Page out the VACB */ - for (i = 0; i < VACB_MAPPING_GRANULARITY / PAGE_SIZE; i++) - { - Page = (PFN_NUMBER)(MmGetPhysicalAddress((PUCHAR)current->BaseAddress + (i * PAGE_SIZE)).QuadPart >> PAGE_SHIFT); - - MmPageOutPhysicalAddress(Page); - } - - /* Reacquire the locks */ - oldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - KeAcquireSpinLockAtDpcLevel(¤t->SharedCacheMap->CacheMapLock); - } - - /* Dereference the VACB */ - Refs = CcRosVacbDecRefCount(current); - - /* Check if we can free this entry now */ - if (Refs < 2) - { - ASSERT(!current->Dirty); - ASSERT(!current->MappedCount); - ASSERT(Refs == 1); - - RemoveEntryList(¤t->CacheMapVacbListEntry); - RemoveEntryList(¤t->VacbLruListEntry); - InitializeListHead(¤t->VacbLruListEntry); - InsertHeadList(&FreeList, ¤t->CacheMapVacbListEntry); - - /* Calculate how many pages we freed for Mm */ - PagesFreed = min(VACB_MAPPING_GRANULARITY / PAGE_SIZE, Target); - Target -= PagesFreed; - (*NrFreed) += PagesFreed; - } - - KeReleaseSpinLockFromDpcLevel(¤t->SharedCacheMap->CacheMapLock); - } - - KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); - - /* Try flushing pages if we haven't met our target */ - if ((Target > 0) && !FlushedPages) - { - /* Flush dirty pages to disk */ - CcRosFlushDirtyPages(Target, &PagesFreed, FALSE, FALSE); - FlushedPages = TRUE; - - /* We can only swap as many pages as we flushed */ - if (PagesFreed < Target) Target = PagesFreed; - - /* Check if we flushed anything */ - if (PagesFreed != 0) - { - /* Try again after flushing dirty pages */ - DPRINT("Flushed %lu dirty cache pages to disk\n", PagesFreed); - goto retry; - } - } - - while (!IsListEmpty(&FreeList)) - { - ULONG Refs; - - current_entry = RemoveHeadList(&FreeList); - current = CONTAINING_RECORD(current_entry, - ROS_VACB, - CacheMapVacbListEntry); - InitializeListHead(¤t->CacheMapVacbListEntry); - Refs = CcRosVacbDecRefCount(current); - ASSERT(Refs == 0); - } - - DPRINT("Evicted %lu cache pages\n", (*NrFreed)); - - return STATUS_SUCCESS; -} - NTSTATUS NTAPI CcRosReleaseVacb ( @@ -504,6 +388,7 @@ CcRosMarkDirtyVacb ( { KIRQL oldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG Length = VACB_MAPPING_GRANULARITY; SharedCacheMap = Vacb->SharedCacheMap; @@ -513,8 +398,12 @@ CcRosMarkDirtyVacb ( ASSERT(!Vacb->Dirty); InsertTailList(&DirtyVacbListHead, &Vacb->DirtyVacbListEntry); - CcTotalDirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; - Vacb->SharedCacheMap->DirtyPages += VACB_MAPPING_GRANULARITY / PAGE_SIZE; +#if 0 + if (Vacb->FileOffset.QuadPart + Length > SharedCacheMap->SectionSize.QuadPart) + Length = SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; +#endif + CcTotalDirtyPages += PAGE_ROUND_UP(Length) / PAGE_SIZE; + Vacb->SharedCacheMap->DirtyPages += PAGE_ROUND_UP(Length) / PAGE_SIZE; CcRosVacbIncRefCount(Vacb); /* Move to the tail of the LRU list */ @@ -531,6 +420,9 @@ CcRosMarkDirtyVacb ( CcScheduleLazyWriteScan(FALSE); } KeReleaseQueuedSpinLock(LockQueueMasterLock, oldIrql); + + /* Tell Mm */ + MmMakePagesDirty(NULL, Vacb->BaseAddress, Length); } VOID @@ -541,6 +433,7 @@ CcRosUnmarkDirtyVacb ( { KIRQL oldIrql; PROS_SHARED_CACHE_MAP SharedCacheMap; + ULONG Length = VACB_MAPPING_GRANULARITY; SharedCacheMap = Vacb->SharedCacheMap; @@ -556,8 +449,14 @@ CcRosUnmarkDirtyVacb ( RemoveEntryList(&Vacb->DirtyVacbListEntry); InitializeListHead(&Vacb->DirtyVacbListEntry); - CcTotalDirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; - Vacb->SharedCacheMap->DirtyPages -= VACB_MAPPING_GRANULARITY / PAGE_SIZE; + +#if 0 + if (Vacb->FileOffset.QuadPart + Length > SharedCacheMap->SectionSize.QuadPart) + Length = SharedCacheMap->SectionSize.QuadPart - Vacb->FileOffset.QuadPart; +#endif + CcTotalDirtyPages -= PAGE_ROUND_UP(Length) / PAGE_SIZE; + Vacb->SharedCacheMap->DirtyPages -= PAGE_ROUND_UP(Length) / PAGE_SIZE; + CcRosVacbDecRefCount(Vacb); if (LockViews) @@ -626,73 +525,6 @@ CcRosUnmapVacb ( return STATUS_SUCCESS; } -static -NTSTATUS -CcRosMapVacbInKernelSpace( - PROS_VACB Vacb) -{ - ULONG i; - NTSTATUS Status; - ULONG_PTR NumberOfPages; - PVOID BaseAddress = NULL; - - /* Create a memory area. */ - MmLockAddressSpace(MmGetKernelAddressSpace()); - Status = MmCreateMemoryArea(MmGetKernelAddressSpace(), - 0, // nothing checks for VACB mareas, so set to 0 - &BaseAddress, - VACB_MAPPING_GRANULARITY, - PAGE_READWRITE, - (PMEMORY_AREA*)&Vacb->MemoryArea, - 0, - PAGE_SIZE); - ASSERT(Vacb->BaseAddress == NULL); - Vacb->BaseAddress = BaseAddress; - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MmCreateMemoryArea failed with %lx for VACB %p\n", Status, Vacb); - return Status; - } - - ASSERT(((ULONG_PTR)Vacb->BaseAddress % PAGE_SIZE) == 0); - ASSERT((ULONG_PTR)Vacb->BaseAddress > (ULONG_PTR)MmSystemRangeStart); - ASSERT((ULONG_PTR)Vacb->BaseAddress + VACB_MAPPING_GRANULARITY - 1 > (ULONG_PTR)MmSystemRangeStart); - - /* Create a virtual mapping for this memory area */ - NumberOfPages = BYTES_TO_PAGES(VACB_MAPPING_GRANULARITY); - for (i = 0; i < NumberOfPages; i++) - { - PFN_NUMBER PageFrameNumber; - - MI_SET_USAGE(MI_USAGE_CACHE); - Status = MmRequestPageMemoryConsumer(MC_CACHE, TRUE, &PageFrameNumber); - if (PageFrameNumber == 0) - { - DPRINT1("Unable to allocate page\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - - ASSERT(BaseAddress == Vacb->BaseAddress); - ASSERT(i * PAGE_SIZE < VACB_MAPPING_GRANULARITY); - ASSERT((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE) >= (ULONG_PTR)BaseAddress); - ASSERT((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE) > (ULONG_PTR)MmSystemRangeStart); - - Status = MmCreateVirtualMapping(NULL, - (PVOID)((ULONG_PTR)Vacb->BaseAddress + (i * PAGE_SIZE)), - PAGE_READWRITE, - &PageFrameNumber, - 1); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Unable to create virtual mapping\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - - return STATUS_SUCCESS; -} - static BOOLEAN CcRosFreeUnusedVacb ( @@ -789,6 +621,7 @@ CcRosCreateVacb ( KIRQL oldIrql; ULONG Refs; BOOLEAN Retried; + SIZE_T ViewSize = VACB_MAPPING_GRANULARITY; ASSERT(SharedCacheMap); @@ -823,8 +656,9 @@ CcRosCreateVacb ( Retried = FALSE; Retry: - /* Map VACB in kernel space */ - Status = CcRosMapVacbInKernelSpace(current); + /* Map VACB in system space */ + Status = MmMapViewInSystemSpaceEx(SharedCacheMap->Section, ¤t->BaseAddress, &ViewSize, ¤t->FileOffset); + if (!NT_SUCCESS(Status)) { ULONG Freed; @@ -932,14 +766,50 @@ Retry: return Status; } +BOOLEAN +NTAPI +CcRosEnsureVacbResident( + _In_ PROS_VACB Vacb, + _In_ BOOLEAN Wait, + _In_ BOOLEAN NoRead, + _In_ ULONG Offset, + _In_ ULONG Length +) +{ + PVOID BaseAddress; + + ASSERT((Offset + Length) <= VACB_MAPPING_GRANULARITY); + + if ((Vacb->FileOffset.QuadPart + Offset) > Vacb->SharedCacheMap->FileSize.QuadPart) + return FALSE; + + BaseAddress = (PVOID)((ULONG_PTR)Vacb->BaseAddress + Offset); + + /* Check if the pages are resident */ + if (!MmArePagesResident(NULL, BaseAddress, Length)) + { + if (!Wait) + { + return FALSE; + } + + if (!NoRead) + { + NTSTATUS Status = MmMakePagesResident(NULL, BaseAddress, Length); + if (!NT_SUCCESS(Status)) + ExRaiseStatus(Status); + } + } + + return TRUE; +} + + NTSTATUS NTAPI CcRosGetVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PLONGLONG BaseOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb) { PROS_VACB current; @@ -978,13 +848,9 @@ CcRosGetVacb ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); /* - * Return information about the VACB to the caller. + * Return the VACB to the caller. */ - *UptoDate = current->Valid; - *BaseAddress = current->BaseAddress; - DPRINT("*BaseAddress %p\n", *BaseAddress); *Vacb = current; - *BaseOffset = current->FileOffset.QuadPart; ASSERT(Refs > 1); @@ -996,14 +862,11 @@ NTAPI CcRosRequestVacb ( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb) /* * FUNCTION: Request a page mapping for a shared cache map */ { - LONGLONG BaseOffset; ASSERT(SharedCacheMap); @@ -1016,30 +879,9 @@ CcRosRequestVacb ( return CcRosGetVacb(SharedCacheMap, FileOffset, - &BaseOffset, - BaseAddress, - UptoDate, Vacb); } -static -VOID -CcFreeCachePage ( - PVOID Context, - MEMORY_AREA* MemoryArea, - PVOID Address, - PFN_NUMBER Page, - SWAPENTRY SwapEntry, - BOOLEAN Dirty) -{ - ASSERT(SwapEntry == 0); - if (Page != 0) - { - ASSERT(MmGetReferenceCountPage(Page) == 1); - MmReleasePageMemoryConsumer(MC_CACHE, Page); - } -} - NTSTATUS CcRosInternalFreeVacb ( PROS_VACB Vacb) @@ -1047,6 +889,8 @@ CcRosInternalFreeVacb ( * FUNCTION: Releases a VACB associated with a shared cache map */ { + NTSTATUS Status; + DPRINT("Freeing VACB 0x%p\n", Vacb); #if DBG if (Vacb->SharedCacheMap->Trace) @@ -1055,12 +899,14 @@ CcRosInternalFreeVacb ( } #endif - MmLockAddressSpace(MmGetKernelAddressSpace()); - MmFreeMemoryArea(MmGetKernelAddressSpace(), - Vacb->MemoryArea, - CcFreeCachePage, - NULL); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); + /* Delete the mapping */ + Status = MmUnmapViewInSystemSpace(Vacb->BaseAddress); + if (!NT_SUCCESS(Status)) + { + DPRINT1("Failed to unmap VACB from System address space! Status 0x%08X\n", Status); + ASSERT(FALSE); + /* Proceed with the deĺetion anyway */ + } if (Vacb->ReferenceCount != 0) { @@ -1097,11 +943,8 @@ CcFlushCache ( PROS_VACB current; NTSTATUS Status; - CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=%p Length=%lu\n", - SectionObjectPointers, FileOffset, Length); - - DPRINT("CcFlushCache(SectionObjectPointers 0x%p, FileOffset 0x%p, Length %lu, IoStatus 0x%p)\n", - SectionObjectPointers, FileOffset, Length, IoStatus); + CCTRACE(CC_API_DEBUG, "SectionObjectPointers=%p FileOffset=0x%I64X Length=%lu\n", + SectionObjectPointers, FileOffset ? FileOffset->QuadPart : 0LL, Length); if (SectionObjectPointers && SectionObjectPointers->SharedCacheMap) { @@ -1217,6 +1060,8 @@ CcRosDeleteFileCache ( KeReleaseSpinLockFromDpcLevel(&SharedCacheMap->CacheMapLock); KeReleaseQueuedSpinLock(LockQueueMasterLock, *OldIrql); + if(SharedCacheMap->Section) + ObDereferenceObject(SharedCacheMap->Section); ObDereferenceObject(SharedCacheMap->FileObject); while (!IsListEmpty(&FreeList)) @@ -1299,36 +1144,6 @@ CcRosRemoveIfClosed ( KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); } - -VOID -NTAPI -CcRosDereferenceCache ( - PFILE_OBJECT FileObject) -{ - PROS_SHARED_CACHE_MAP SharedCacheMap; - KIRQL OldIrql; - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - ASSERT(SharedCacheMap); - if (SharedCacheMap->OpenCount > 0) - { - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) - { - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - MmFreeSectionSegments(SharedCacheMap->FileObject); - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - - return; - } - } - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); -} - NTSTATUS NTAPI CcRosReleaseFileCache ( @@ -1373,20 +1188,12 @@ CcRosReleaseFileCache ( PrivateMap->NodeTypeCode = 0; } - if (SharedCacheMap->OpenCount > 0) + ASSERT(SharedCacheMap->OpenCount > 0); + + SharedCacheMap->OpenCount--; + if (SharedCacheMap->OpenCount == 0) { - SharedCacheMap->OpenCount--; - if (SharedCacheMap->OpenCount == 0) - { - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - MmFreeSectionSegments(SharedCacheMap->FileObject); - - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); - KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); - - return STATUS_SUCCESS; - } + CcRosDeleteFileCache(FileObject, SharedCacheMap, &OldIrql); } } } @@ -1412,6 +1219,8 @@ CcRosInitializeFileCache ( DPRINT("CcRosInitializeFileCache(FileObject 0x%p)\n", FileObject); + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + Allocated = FALSE; SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; if (SharedCacheMap == NULL) @@ -1437,27 +1246,37 @@ CcRosInitializeFileCache ( KeInitializeSpinLock(&SharedCacheMap->CacheMapLock); InitializeListHead(&SharedCacheMap->CacheMapVacbListHead); InitializeListHead(&SharedCacheMap->BcbList); - } - OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); - if (Allocated) + SharedCacheMap->Flags = SHARED_CACHE_MAP_IN_CREATION; + + ObReferenceObjectByPointer(FileObject, + FILE_ALL_ACCESS, + NULL, + KernelMode); + + FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; + + // CcRosTraceCacheMap(SharedCacheMap, TRUE); + } + else if (SharedCacheMap->Flags & SHARED_CACHE_MAP_IN_CREATION) { - if (FileObject->SectionObjectPointer->SharedCacheMap == NULL) - { - ObReferenceObjectByPointer(FileObject, - FILE_ALL_ACCESS, - NULL, - KernelMode); - FileObject->SectionObjectPointer->SharedCacheMap = SharedCacheMap; + /* The shared cache map is being created somewhere else. Wait for that to happen */ + KEVENT Waiter; + PKEVENT PreviousWaiter = SharedCacheMap->CreateEvent; - InsertTailList(&CcCleanSharedCacheMapList, &SharedCacheMap->SharedCacheMapLinks); - } - else - { - ExFreeToNPagedLookasideList(&SharedCacheMapLookasideList, SharedCacheMap); - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - } + KeInitializeEvent(&Waiter, NotificationEvent, FALSE); + SharedCacheMap->CreateEvent = &Waiter; + + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + + KeWaitForSingleObject(&Waiter, Executive, KernelMode, FALSE, NULL); + + if (PreviousWaiter) + KeSetEvent(PreviousWaiter, IO_NO_INCREMENT, FALSE); + + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); } + if (FileObject->PrivateCacheMap == NULL) { PPRIVATE_CACHE_MAP PrivateMap; @@ -1503,8 +1322,48 @@ CcRosInitializeFileCache ( FileObject->PrivateCacheMap = PrivateMap; SharedCacheMap->OpenCount++; } + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + /* Create the section */ + if (Allocated) + { + NTSTATUS Status; + + ASSERT(SharedCacheMap->Section == NULL); + + Status = MmCreateSection( + &SharedCacheMap->Section, + SECTION_ALL_ACCESS, + NULL, + &SharedCacheMap->SectionSize, + PAGE_READWRITE, + 0, + NULL, + FileObject); + + ASSERT(NT_SUCCESS(Status)); + + if (!NT_SUCCESS(Status)) + { + CcRosReleaseFileCache(FileObject); + return Status; + } + + OldIrql = KeAcquireQueuedSpinLock(LockQueueMasterLock); + + InsertTailList(&CcCleanSharedCacheMapList, &SharedCacheMap->SharedCacheMapLinks); + SharedCacheMap->Flags &= ~SHARED_CACHE_MAP_IN_CREATION; + + if (SharedCacheMap->CreateEvent) + { + KeSetEvent(SharedCacheMap->CreateEvent, IO_NO_INCREMENT, FALSE); + SharedCacheMap->CreateEvent = NULL; + } + + KeReleaseQueuedSpinLock(LockQueueMasterLock, OldIrql); + } + return STATUS_SUCCESS; } @@ -1564,8 +1423,6 @@ CcInitView ( TAG_VACB, 20); - MmInitializeMemoryConsumer(MC_CACHE, CcRosTrimCache); - CcInitCacheZeroPage(); } diff --git a/ntoskrnl/ex/init.c b/ntoskrnl/ex/init.c index 8eb867b09cd..fc98cccd430 100644 --- a/ntoskrnl/ex/init.c +++ b/ntoskrnl/ex/init.c @@ -1970,7 +1970,6 @@ Phase1InitializationDiscard(IN PVOID Context) InbvEnableDisplayString(TRUE); /* Launch initial process */ - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); ProcessInfo = &InitBuffer->ProcessInfo; ExpLoadInitialProcess(InitBuffer, &ProcessParameters, &Environment); @@ -2009,7 +2008,6 @@ Phase1InitializationDiscard(IN PVOID Context) /* Free the boot buffer */ ExFreePoolWithTag(InitBuffer, TAG_INIT); - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); } VOID diff --git a/ntoskrnl/ex/sysinfo.c b/ntoskrnl/ex/sysinfo.c index 64cbffc5ba1..f595623b4c8 100644 --- a/ntoskrnl/ex/sysinfo.c +++ b/ntoskrnl/ex/sysinfo.c @@ -719,7 +719,6 @@ QSI_DEF(SystemPerformanceInformation) * Not sure this is right. 8^\ */ Spi->CommittedPages = MiMemoryConsumers[MC_SYSTEM].PagesUsed + - MiMemoryConsumers[MC_CACHE].PagesUsed + MiMemoryConsumers[MC_USER].PagesUsed + MiUsedSwapPages; /* @@ -767,7 +766,7 @@ QSI_DEF(SystemPerformanceInformation) Spi->TotalSystemDriverPages = 0; /* FIXME */ Spi->Spare3Count = 0; /* FIXME */ - Spi->ResidentSystemCachePage = MiMemoryConsumers[MC_CACHE].PagesUsed; + Spi->ResidentSystemCachePage = 0; /* FIXME */ Spi->ResidentPagedPoolPage = 0; /* FIXME */ Spi->ResidentSystemDriverPage = 0; /* FIXME */ @@ -1477,13 +1476,10 @@ QSI_DEF(SystemFileCacheInformation) RtlZeroMemory(Sci, sizeof(SYSTEM_FILECACHE_INFORMATION)); /* Return the Byte size not the page size. */ - Sci->CurrentSize = - MiMemoryConsumers[MC_CACHE].PagesUsed * PAGE_SIZE; - Sci->PeakSize = - MiMemoryConsumers[MC_CACHE].PagesUsed * PAGE_SIZE; /* FIXME */ + Sci->CurrentSize = 0; /* FIXME */ + Sci->PeakSize = 0; /* FIXME */ /* Taskmgr multiplies this one by page size right away */ - Sci->CurrentSizeIncludingTransitionInPages = - MiMemoryConsumers[MC_CACHE].PagesUsed; /* FIXME: Should be */ + Sci->CurrentSizeIncludingTransitionInPages = 0; /* FIXME: Should be */ /* system working set and standby pages. */ Sci->PageFaultCount = 0; /* FIXME */ Sci->MinimumWorkingSet = 0; /* FIXME */ diff --git a/ntoskrnl/include/internal/cc.h b/ntoskrnl/include/internal/cc.h index 18c0b9effa2..9e186a61d11 100644 --- a/ntoskrnl/include/internal/cc.h +++ b/ntoskrnl/include/internal/cc.h @@ -3,7 +3,7 @@ // // Define this if you want debugging support // -#define _CC_DEBUG_ 0x00 +#define _CC_DEBUG_ 0x0 // // These define the Debug Masks Supported @@ -179,6 +179,8 @@ typedef struct _ROS_SHARED_CACHE_MAP ULONG DirtyPages; LIST_ENTRY SharedCacheMapLinks; ULONG Flags; + PVOID Section; + PKEVENT CreateEvent; PCACHE_MANAGER_CALLBACKS Callbacks; PVOID LazyWriteContext; LIST_ENTRY PrivateList; @@ -197,13 +199,12 @@ typedef struct _ROS_SHARED_CACHE_MAP #define READAHEAD_DISABLED 0x1 #define WRITEBEHIND_DISABLED 0x2 +#define SHARED_CACHE_MAP_IN_CREATION 0x4 typedef struct _ROS_VACB { /* Base address of the region where the view's data is mapped. */ PVOID BaseAddress; - /* Memory area representing the region where the view's data is mapped. */ - struct _MEMORY_AREA* MemoryArea; /* Are the contents of the view valid. */ BOOLEAN Valid; /* Are the contents of the view newer than those on disk. */ @@ -316,12 +317,19 @@ NTAPI CcRosGetVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PLONGLONG BaseOffset, - PVOID *BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb ); +BOOLEAN +NTAPI +CcRosEnsureVacbResident( + _In_ PROS_VACB Vacb, + _In_ BOOLEAN Wait, + _In_ BOOLEAN NoRead, + _In_ ULONG Offset, + _In_ ULONG Length +); + VOID NTAPI CcInitView(VOID); @@ -330,14 +338,6 @@ VOID NTAPI CcShutdownLazyWriter(VOID); -NTSTATUS -NTAPI -CcReadVirtualAddress(PROS_VACB Vacb); - -NTSTATUS -NTAPI -CcWriteVirtualAddress(PROS_VACB Vacb); - BOOLEAN NTAPI CcInitializeCacheManager(VOID); @@ -415,8 +415,6 @@ NTAPI CcRosRequestVacb( PROS_SHARED_CACHE_MAP SharedCacheMap, LONGLONG FileOffset, - PVOID* BaseAddress, - PBOOLEAN UptoDate, PROS_VACB *Vacb ); diff --git a/ntoskrnl/include/internal/mm.h b/ntoskrnl/include/internal/mm.h index be8b0acc4b0..55e9fcfc29c 100644 --- a/ntoskrnl/include/internal/mm.h +++ b/ntoskrnl/include/internal/mm.h @@ -89,12 +89,9 @@ typedef ULONG_PTR SWAPENTRY; #define SEC_PHYSICALMEMORY (0x80000000) -#define MM_DATAFILE_SEGMENT (0x2) - -#define MC_CACHE (0) -#define MC_USER (1) -#define MC_SYSTEM (2) -#define MC_MAXIMUM (3) +#define MC_USER (0) +#define MC_SYSTEM (1) +#define MC_MAXIMUM (2) #define PAGED_POOL_MASK 1 #define MUST_SUCCEED_POOL_MASK 2 @@ -171,10 +168,10 @@ typedef struct _MM_SECTION_SEGMENT FAST_MUTEX Lock; /* lock which protects the page directory */ LARGE_INTEGER RawLength; /* length of the segment which is part of the mapped file */ LARGE_INTEGER Length; /* absolute length of the segment */ - ULONG ReferenceCount; - ULONG CacheCount; + PULONG ReferenceCount; + ULONG SectionCount; ULONG Protection; - ULONG Flags; + PULONG Flags; BOOLEAN WriteCopy; BOOLEAN Locked; @@ -185,6 +182,9 @@ typedef struct _MM_SECTION_SEGMENT ULONG Characteristics; } Image; + ULONG RefCount; + ULONG SegFlags; + LIST_ENTRY ListOfSegments; RTL_GENERIC_TABLE PageTable; } MM_SECTION_SEGMENT, *PMM_SECTION_SEGMENT; @@ -193,12 +193,20 @@ typedef struct _MM_IMAGE_SECTION_OBJECT { PFILE_OBJECT FileObject; + ULONG RefCount; + ULONG SegFlags; + SECTION_IMAGE_INFORMATION ImageInformation; PVOID BasedAddress; ULONG NrSegments; PMM_SECTION_SEGMENT Segments; } MM_IMAGE_SECTION_OBJECT, *PMM_IMAGE_SECTION_OBJECT; +#define MM_DATAFILE_SEGMENT (0x2) +#define MM_SEGMENT_INDELETE (0x4) +#define MM_SEGMENT_INCREATE (0x8) + + #define MA_GetStartingAddress(_MemoryArea) ((_MemoryArea)->VadNode.StartingVpn << PAGE_SHIFT) #define MA_GetEndingAddress(_MemoryArea) (((_MemoryArea)->VadNode.EndingVpn + 1) << PAGE_SHIFT) @@ -862,11 +870,6 @@ MmInitializeRmapList(VOID); VOID NTAPI MmSetCleanAllRmaps(PFN_NUMBER Page); - -VOID -NTAPI -MmSetDirtyAllRmaps(PFN_NUMBER Page); - BOOLEAN NTAPI MmIsDirtyPageRmap(PFN_NUMBER Page); @@ -1288,15 +1291,6 @@ MmNotPresentFaultSectionView( BOOLEAN Locked ); -NTSTATUS -NTAPI -MmPageOutSectionView( - PMMSUPPORT AddressSpace, - PMEMORY_AREA MemoryArea, - PVOID Address, - ULONG_PTR Entry -); - NTSTATUS NTAPI MmCreatePhysicalMemorySection(VOID); @@ -1337,6 +1331,48 @@ MmMakePagesResident( _In_ PVOID Address, _In_ ULONG Length); +NTSTATUS +NTAPI +MmMakePagesDirty( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length); + +NTSTATUS +NTAPI +MmRosFlushVirtualMemory( + _In_ PEPROCESS Process, + _Inout_ PVOID* Address, + _Inout_ PSIZE_T Length, + _Out_ PIO_STATUS_BLOCK Iosb); + +BOOLEAN +NTAPI +MmCheckDirtySegment( + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN ForceDirty, + BOOLEAN PageOut); + +BOOLEAN +NTAPI +MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN Dirty, + BOOLEAN PageOut, + ULONG_PTR *InEntry); + +VOID +NTAPI +MmDereferenceSegment(PMM_SECTION_SEGMENT Segment); + +NTSTATUS +NTAPI +MmExtendSection( + _In_ PVOID Section, + _Inout_ PLARGE_INTEGER NewSize); + /* sysldr.c ******************************************************************/ VOID diff --git a/ntoskrnl/mm/ARM3/section.c b/ntoskrnl/mm/ARM3/section.c index 938a08d6037..9ea36637a22 100644 --- a/ntoskrnl/mm/ARM3/section.c +++ b/ntoskrnl/mm/ARM3/section.c @@ -2934,7 +2934,7 @@ MmMapViewOfArm3Section(IN PVOID SectionObject, if (!(*ViewSize)) { /* Compute it for the caller */ - CalculatedViewSize = Section->SizeOfSection.QuadPart - + CalculatedViewSize = Section->SizeOfSection.QuadPart - SectionOffset->QuadPart; /* Check if it's larger than 4GB or overflows into kernel-mode */ @@ -3891,30 +3891,24 @@ NtExtendSection(IN HANDLE SectionHandle, NULL); if (!NT_SUCCESS(Status)) return Status; - /* Really this should go in MmExtendSection */ - if (!Section->u.Flags.File || Section->u.Flags.Image) - { - DPRINT1("Not extending a file\n"); - ObDereferenceObject(Section); - return STATUS_SECTION_NOT_EXTENDED; - } - - /* FIXME: Do the work */ + Status = MmExtendSection(Section, &SafeNewMaximumSize); /* Dereference the section */ ObDereferenceObject(Section); - /* Enter SEH */ - _SEH2_TRY + if (NT_SUCCESS(Status)) { - /* Write back the new size */ - *NewMaximumSize = SafeNewMaximumSize; + _SEH2_TRY + { + /* Write back the new size */ + *NewMaximumSize = SafeNewMaximumSize; + } + _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) + { + Status = _SEH2_GetExceptionCode(); + } + _SEH2_END; } - _SEH2_EXCEPT(EXCEPTION_EXECUTE_HANDLER) - { - /* Nothing to do */ - } - _SEH2_END; /* Return the status */ return STATUS_NOT_IMPLEMENTED; diff --git a/ntoskrnl/mm/ARM3/virtual.c b/ntoskrnl/mm/ARM3/virtual.c index 19eb8a69b01..00271e03e7b 100644 --- a/ntoskrnl/mm/ARM3/virtual.c +++ b/ntoskrnl/mm/ARM3/virtual.c @@ -1356,12 +1356,8 @@ MmFlushVirtualMemory(IN PEPROCESS Process, OUT PIO_STATUS_BLOCK IoStatusBlock) { PAGED_CODE(); - UNIMPLEMENTED; - - // - // Fake success - // - return STATUS_SUCCESS; + /* For now we call the old Mm */ + return MmRosFlushVirtualMemory(Process, BaseAddress, RegionSize, IoStatusBlock); } ULONG diff --git a/ntoskrnl/mm/ARM3/zeropage.c b/ntoskrnl/mm/ARM3/zeropage.c index 5fe22294199..80725e8a5a7 100644 --- a/ntoskrnl/mm/ARM3/zeropage.c +++ b/ntoskrnl/mm/ARM3/zeropage.c @@ -46,7 +46,7 @@ MmZeroPageThread(VOID) /* Get the discardable sections to free them */ MiFindInitializationCode(&StartAddress, &EndAddress); if (StartAddress) MiFreeInitializationCode(StartAddress, EndAddress); - DPRINT("Free non-cache pages: %lx\n", MmAvailablePages + MiMemoryConsumers[MC_CACHE].PagesUsed); + DPRINT("Free pages: %lx\n", MmAvailablePages); /* Set our priority to 0 */ Thread->BasePriority = 0; diff --git a/ntoskrnl/mm/balance.c b/ntoskrnl/mm/balance.c index 5cd5e267a23..29c0dfc6f47 100644 --- a/ntoskrnl/mm/balance.c +++ b/ntoskrnl/mm/balance.c @@ -54,18 +54,6 @@ MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages) /* Set up targets. */ MiMinimumAvailablePages = 256; MiMinimumPagesPerRun = 256; - if ((NrAvailablePages + NrSystemPages) >= 8192) - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 4 * 3; - } - else if ((NrAvailablePages + NrSystemPages) >= 4096) - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 3 * 2; - } - else - { - MiMemoryConsumers[MC_CACHE].PagesTarget = NrAvailablePages / 8; - } MiMemoryConsumers[MC_USER].PagesTarget = NrAvailablePages - MiMinimumAvailablePages; } diff --git a/ntoskrnl/mm/pagefile.c b/ntoskrnl/mm/pagefile.c index 37870520998..e0f9e087150 100644 --- a/ntoskrnl/mm/pagefile.c +++ b/ntoskrnl/mm/pagefile.c @@ -94,9 +94,6 @@ NTAPI MmBuildMdlFromPages(PMDL Mdl, PPFN_NUMBER Pages) { memcpy(Mdl + 1, Pages, sizeof(PFN_NUMBER) * (PAGE_ROUND_UP(Mdl->ByteOffset+Mdl->ByteCount)/PAGE_SIZE)); - - /* FIXME: this flag should be set by the caller perhaps? */ - Mdl->MdlFlags |= MDL_IO_PAGE_READ; } @@ -230,7 +227,7 @@ MiReadPageFile( MmInitializeMdl(Mdl, NULL, PAGE_SIZE); MmBuildMdlFromPages(Mdl, &Page); - Mdl->MdlFlags |= MDL_PAGES_LOCKED; + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; file_offset.QuadPart = PageFileOffset * PAGE_SIZE; diff --git a/ntoskrnl/mm/rmap.c b/ntoskrnl/mm/rmap.c index 9f5938d7496..d1cfb584dce 100644 --- a/ntoskrnl/mm/rmap.c +++ b/ntoskrnl/mm/rmap.c @@ -19,7 +19,6 @@ /* GLOBALS ******************************************************************/ static NPAGED_LOOKASIDE_LIST RmapLookasideList; -FAST_MUTEX RmapListLock; /* FUNCTIONS ****************************************************************/ @@ -38,7 +37,6 @@ VOID NTAPI MmInitializeRmapList(VOID) { - ExInitializeFastMutex(&RmapListLock); ExInitializeNPagedLookasideList (&RmapLookasideList, NULL, RmapListFree, @@ -55,37 +53,27 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) PMM_RMAP_ENTRY entry; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace; - ULONG Type; PVOID Address; PEPROCESS Process; - ULONGLONG Offset; NTSTATUS Status = STATUS_SUCCESS; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset; + KIRQL OldIrql; + + OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); entry = MmGetRmapListHeadPage(Page); -#ifdef NEWCC - // Special case for NEWCC: we can have a page that's only in a segment - // page table - if (entry && RMAP_IS_SEGMENT(entry->Address) && entry->Next == NULL) - { - /* NEWCC does locking itself */ - ExReleaseFastMutex(&RmapListLock); - return MmpPageOutPhysicalAddress(Page); - } -#endif - while (entry && RMAP_IS_SEGMENT(entry->Address)) entry = entry->Next; if (entry == NULL) { - ExReleaseFastMutex(&RmapListLock); - return(STATUS_UNSUCCESSFUL); + MiReleasePfnLock(OldIrql); + return STATUS_UNSUCCESSFUL; } Process = entry->Process; - Address = entry->Address; if ((((ULONG_PTR)Address) & 0xFFF) != 0) @@ -97,12 +85,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) { if (!ExAcquireRundownProtection(&Process->RundownProtect)) { - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); return STATUS_PROCESS_IS_TERMINATING; } Status = ObReferenceObjectByPointer(Process, PROCESS_ALL_ACCESS, NULL, KernelMode); - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); if (!NT_SUCCESS(Status)) { ExReleaseRundownProtection(&Process->RundownProtect); @@ -112,11 +100,12 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) } else { - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); AddressSpace = MmGetKernelAddressSpace(); } MmLockAddressSpace(AddressSpace); + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL || MemoryArea->DeleteInProgress) { @@ -128,23 +117,27 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) } return(STATUS_UNSUCCESSFUL); } - Type = MemoryArea->Type; - if (Type == MEMORY_AREA_SECTION_VIEW) + + if (MemoryArea->Type == MEMORY_AREA_SECTION_VIEW) { ULONG_PTR Entry; - Offset = MemoryArea->SectionData.ViewOffset.QuadPart + + BOOLEAN Dirty; + PFN_NUMBER MapPage; + LARGE_INTEGER Offset; + BOOLEAN Released; + + Offset.QuadPart = MemoryArea->SectionData.ViewOffset.QuadPart + ((ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea)); - MmLockSectionSegment(MemoryArea->SectionData.Segment); + Segment = MemoryArea->SectionData.Segment; - /* - * Get or create a pageop - */ - Entry = MmGetPageEntrySectionSegment(MemoryArea->SectionData.Segment, - (PLARGE_INTEGER)&Offset); + MmLockSectionSegment(Segment); + + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry && MM_IS_WAIT_PTE(Entry)) { - MmUnlockSectionSegment(MemoryArea->SectionData.Segment); + /* The segment is being read or something. Give up */ + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); if (Address < MmSystemRangeStart) { @@ -154,18 +147,101 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) return(STATUS_UNSUCCESSFUL); } - MmSetPageEntrySectionSegment(MemoryArea->SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + /* Delete this virtual mapping in the process */ + MmDeleteVirtualMapping(Process, Address, &Dirty, &MapPage); + ASSERT(MapPage == Page); - /* - * Release locks now we have a page op. - */ - MmUnlockSectionSegment(MemoryArea->SectionData.Segment); + if (Page != PFN_FROM_SSE(Entry)) + { + SWAPENTRY SwapEntry; + + /* This page is private to the process */ + MmUnlockSectionSegment(Segment); + + /* Check if we should write it back to the page file */ + SwapEntry = MmGetSavedSwapEntryPage(Page); + + if ((SwapEntry == 0) && Dirty) + { + /* We don't have a Swap entry, yet the page is dirty. Get one */ + SwapEntry = MmAllocSwapPage(); + if (!SwapEntry) + { + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + + /* We can't, so let this page in the Process VM */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmSetDirtyPage(Process, Address); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_UNSUCCESSFUL; + } + } + + if (Dirty) + { + Status = MmWriteToSwapPage(SwapEntry, Page); + if (!NT_SUCCESS(Status)) + { + /* We failed at saving the content of this page. Keep it in */ + PMM_REGION Region = MmFindRegion((PVOID)MA_GetStartingAddress(MemoryArea), + &MemoryArea->SectionData.RegionListHead, + Address, NULL); + + /* This Swap Entry is useless to us */ + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); + + /* We can't, so let this page in the Process VM */ + MmCreateVirtualMapping(Process, Address, Region->Protect, &Page, 1); + MmSetDirtyPage(Process, Address); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_UNSUCCESSFUL; + } + } + + if (SwapEntry) + { + /* Keep this in the process VM */ + MmCreatePageFileMapping(Process, Address, SwapEntry); + MmSetSavedSwapEntryPage(Page, 0); + } + + MmUnlockAddressSpace(AddressSpace); + + /* We can finally let this page go */ + MmDeleteRmap(Page, Process, Address); + MmReleasePageMemoryConsumer(MC_USER, Page); + + ASSERT(MmGetRmapListHeadPage(Page) == NULL); + + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + return STATUS_SUCCESS; + } + + /* Delete this RMAP */ + MmDeleteRmap(Page, Process, Address); + + /* One less mapping referencing this segment */ + Released = MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); + + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); - /* - * Do the actual page out work. - */ - Status = MmPageOutSectionView(AddressSpace, MemoryArea, Address, Entry); + if (Address < MmSystemRangeStart) + { + ExReleaseRundownProtection(&Process->RundownProtect); + ObDereferenceObject(Process); + } + + if (Released) return STATUS_SUCCESS; } #ifdef NEWCC else if (Type == MEMORY_AREA_CACHE) @@ -185,7 +261,29 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) ExReleaseRundownProtection(&Process->RundownProtect); ObDereferenceObject(Process); } - return(Status); + + /* Now write this page to file, if needed */ + Segment = MmGetSectionAssociation(Page, &SegmentOffset); + if (Segment) + { + BOOLEAN Released; + + MmLockSectionSegment(Segment); + + Released = MmCheckDirtySegment(Segment, &SegmentOffset, FALSE, TRUE); + + MmUnlockSectionSegment(Segment); + + MmDereferenceSegment(Segment); + + if (Released) + { + return STATUS_SUCCESS; + } + } + + /* If we are here, then we didn't release the page */ + return STATUS_UNSUCCESSFUL; } VOID @@ -193,12 +291,13 @@ NTAPI MmSetCleanAllRmaps(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); if (current_entry == NULL) { - DPRINT1("MmIsDirtyRmap: No rmaps.\n"); + DPRINT1("MmSetCleanAllRmaps: No rmaps.\n"); KeBugCheck(MEMORY_MANAGEMENT); } while (current_entry != NULL) @@ -207,29 +306,7 @@ MmSetCleanAllRmaps(PFN_NUMBER Page) MmSetCleanPage(current_entry->Process, current_entry->Address); current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); -} - -VOID -NTAPI -MmSetDirtyAllRmaps(PFN_NUMBER Page) -{ - PMM_RMAP_ENTRY current_entry; - - ExAcquireFastMutex(&RmapListLock); - current_entry = MmGetRmapListHeadPage(Page); - if (current_entry == NULL) - { - DPRINT1("MmIsDirtyRmap: No rmaps.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - while (current_entry != NULL) - { - if (!RMAP_IS_SEGMENT(current_entry->Address)) - MmSetDirtyPage(current_entry->Process, current_entry->Address); - current_entry = current_entry->Next; - } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); } BOOLEAN @@ -237,27 +314,31 @@ NTAPI MmIsDirtyPageRmap(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry; + KIRQL OldIrql; + BOOLEAN Dirty = FALSE; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); if (current_entry == NULL) { - ExReleaseFastMutex(&RmapListLock); - return(FALSE); + DPRINT1("MmIsDirtyPageRmap: No rmaps.\n"); + KeBugCheck(MEMORY_MANAGEMENT); } while (current_entry != NULL) { - if ( - !RMAP_IS_SEGMENT(current_entry->Address) && - MmIsDirtyPage(current_entry->Process, current_entry->Address)) + if (!RMAP_IS_SEGMENT(current_entry->Address)) { - ExReleaseFastMutex(&RmapListLock); - return(TRUE); + if (MmIsDirtyPage(current_entry->Process, current_entry->Address)) + { + Dirty = TRUE; + break; + } } current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); - return(FALSE); + MiReleasePfnLock(OldIrql); + + return Dirty; } VOID @@ -268,6 +349,8 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, PMM_RMAP_ENTRY current_entry; PMM_RMAP_ENTRY new_entry; ULONG PrevSize; + KIRQL OldIrql; + if (!RMAP_IS_SEGMENT(Address)) Address = (PVOID)PAGE_ROUND_DOWN(Address); @@ -298,7 +381,7 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, KeBugCheck(MEMORY_MANAGEMENT); } - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); current_entry = MmGetRmapListHeadPage(Page); new_entry->Next = current_entry; #if DBG @@ -318,7 +401,8 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, } #endif MmSetRmapListHeadPage(Page, new_entry); - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); + if (!RMAP_IS_SEGMENT(Address)) { if (Process == NULL) @@ -336,63 +420,15 @@ MmInsertRmap(PFN_NUMBER Page, PEPROCESS Process, } } -VOID -NTAPI -MmDeleteAllRmaps(PFN_NUMBER Page, PVOID Context, - VOID (*DeleteMapping)(PVOID Context, PEPROCESS Process, - PVOID Address)) -{ - PMM_RMAP_ENTRY current_entry; - PMM_RMAP_ENTRY previous_entry; - PEPROCESS Process; - - ExAcquireFastMutex(&RmapListLock); - current_entry = MmGetRmapListHeadPage(Page); - if (current_entry == NULL) - { - DPRINT1("MmDeleteAllRmaps: No rmaps.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - MmSetRmapListHeadPage(Page, NULL); - ExReleaseFastMutex(&RmapListLock); - - while (current_entry != NULL) - { - previous_entry = current_entry; - current_entry = current_entry->Next; - if (!RMAP_IS_SEGMENT(previous_entry->Address)) - { - if (DeleteMapping) - { - DeleteMapping(Context, previous_entry->Process, - previous_entry->Address); - } - Process = previous_entry->Process; - ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); - if (Process == NULL) - { - Process = PsInitialSystemProcess; - } - if (Process) - { - (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); - } - } - else - { - ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); - } - } -} - VOID NTAPI MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, PVOID Address) { PMM_RMAP_ENTRY current_entry, previous_entry; + KIRQL OldIrql; - ExAcquireFastMutex(&RmapListLock); + OldIrql = MiAcquirePfnLock(); previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); @@ -409,7 +445,8 @@ MmDeleteRmap(PFN_NUMBER Page, PEPROCESS Process, { previous_entry->Next = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); + ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); if (!RMAP_IS_SEGMENT(Address)) { @@ -450,8 +487,8 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) { PCACHE_SECTION_PAGE_TABLE Result = NULL; PMM_RMAP_ENTRY current_entry;//, previous_entry; + KIRQL OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); //previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); while (current_entry != NULL) @@ -460,14 +497,20 @@ MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) { Result = (PCACHE_SECTION_PAGE_TABLE)current_entry->Process; *RawOffset = (ULONG_PTR)current_entry->Address & ~RMAP_SEGMENT_MASK; - InterlockedIncrementUL(&Result->Segment->ReferenceCount); - ExReleaseFastMutex(&RmapListLock); + if (*Result->Segment->Flags & MM_SEGMENT_INDELETE) + { + MiReleasePfnLock(OldIrql); + return NULL; + } + + InterlockedIncrementUL(Result->Segment->ReferenceCount); + MiReleasePfnLock(OldIrql); return Result; } //previous_entry = current_entry; current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); return NULL; } @@ -482,8 +525,8 @@ NTAPI MmDeleteSectionAssociation(PFN_NUMBER Page) { PMM_RMAP_ENTRY current_entry, previous_entry; + KIRQL OldIrql = MiAcquirePfnLock(); - ExAcquireFastMutex(&RmapListLock); previous_entry = NULL; current_entry = MmGetRmapListHeadPage(Page); while (current_entry != NULL) @@ -498,12 +541,12 @@ MmDeleteSectionAssociation(PFN_NUMBER Page) { previous_entry->Next = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); return; } previous_entry = current_entry; current_entry = current_entry->Next; } - ExReleaseFastMutex(&RmapListLock); + MiReleasePfnLock(OldIrql); } diff --git a/ntoskrnl/mm/section.c b/ntoskrnl/mm/section.c index 4d29f5a9ff4..39ae8bf75f5 100644 --- a/ntoskrnl/mm/section.c +++ b/ntoskrnl/mm/section.c @@ -193,6 +193,58 @@ static GENERIC_MAPPING MmpSectionMapping = /* FUNCTIONS *****************************************************************/ + +NTSTATUS +NTAPI +MiWritePage(PMM_SECTION_SEGMENT Segment, + LONGLONG SegOffset, + PFN_NUMBER Page) +/* + * FUNCTION: write a page for a section backed memory area. + * PARAMETERS: + * MemoryArea - Memory area to write the page for. + * Offset - Offset of the page to write. + * Page - Page which contains the data to write. + */ +{ + NTSTATUS Status; + IO_STATUS_BLOCK IoStatus; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; + PMDL Mdl = (PMDL)MdlBase; + PFILE_OBJECT FileObject = Segment->FileObject; + LARGE_INTEGER FileOffset; + PFSRTL_COMMON_FCB_HEADER Fcb = FileObject->FsContext; + + FileOffset.QuadPart = Segment->Image.FileOffset + SegOffset; + + /* Check if we are not writing off-limit */ + if (FileOffset.QuadPart >= Fcb->AllocationSize.QuadPart) + { + return STATUS_SUCCESS; + } + + RtlZeroMemory(MdlBase, sizeof(MdlBase)); + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, &Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; + + KeInitializeEvent(&Event, NotificationEvent, FALSE); + Status = IoSynchronousPageWrite(FileObject, Mdl, &FileOffset, &Event, &IoStatus); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); + Status = IoStatus.Status; + } + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); + } + + return Status; +} + + /* References: [1] Microsoft Corporation, "Microsoft Portable Executable and Common Object @@ -713,6 +765,8 @@ l_ReadHeaderFromFile: nStatus = STATUS_INVALID_IMAGE_FORMAT; + ASSERT(ImageSectionObject->RefCount > 0); + /* convert the executable sections into segments. See also [1], section 4 */ for(i = 0; i < ImageSectionObject->NrSegments - 1; ++ i) { @@ -824,51 +878,146 @@ MmspWaitForFileLock(PFILE_OBJECT File) //return KeWaitForSingleObject(&File->Lock, 0, KernelMode, FALSE, NULL); } + + VOID NTAPI -MmFreeSectionSegments(PFILE_OBJECT FileObject) +MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) { - if (FileObject->SectionObjectPointer->ImageSectionObject != NULL) + ULONG Length; + LARGE_INTEGER Offset; + ULONG_PTR Entry; + SWAPENTRY SavedSwapEntry; + PFN_NUMBER Page; + + Page = 0; + + MmLockSectionSegment(Segment); + + Length = PAGE_ROUND_UP(Segment->Length.QuadPart); + for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) { - PMM_IMAGE_SECTION_OBJECT ImageSectionObject; + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + if (Entry) + { + MmSetPageEntrySectionSegment(Segment, &Offset, 0); + if (IS_SWAP_FROM_SSE(Entry)) + { + MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); + } + else + { + Page = PFN_FROM_SSE(Entry); + SavedSwapEntry = MmGetSavedSwapEntryPage(Page); + if (SavedSwapEntry != 0) + { + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SavedSwapEntry); + } + MmReleasePageMemoryConsumer(MC_USER, Page); + } + } + } + + MmUnlockSectionSegment(Segment); +} + +static +VOID +NTAPI +FreeSegmentPage(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset) +{ + ULONG_PTR Entry; + PFN_NUMBER Page; + + MmLockSectionSegment(Segment); + + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + + MmUnlockSectionSegment(Segment); + + /* This must be either a valid entry or nothing */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* There should be no reference anymore */ + ASSERT(SHARE_COUNT_FROM_SSE(Entry) == 0); + + Page = PFN_FROM_SSE(Entry); + /* If there is a page, this must be because it's still dirty */ + ASSERT(Page != 0); + + /* Write the page */ + if (IS_DIRTY_SSE(Entry)) + MiWritePage(Segment, Offset->QuadPart, Page); + + MmReleasePageMemoryConsumer(MC_USER, Page); +} + +VOID +NTAPI +MmDereferenceSegment(PMM_SECTION_SEGMENT Segment) +{ + KIRQL OldIrql; + + /* Lock the PFN lock because we mess around with SectionObjectPointers */ + OldIrql = MiAcquirePfnLock(); + + if (InterlockedDecrementUL(Segment->ReferenceCount) > 0) + { + /* Nothing to do yet */ + MiReleasePfnLock(OldIrql); + return; + } + + *Segment->Flags |= MM_SEGMENT_INDELETE; + + MiReleasePfnLock(OldIrql); + + /* Flush the segment */ + if (*Segment->Flags & MM_DATAFILE_SEGMENT) + { + /* Free the page table. This will flush any remaining dirty data */ + MmFreePageTablesSectionSegment(Segment, FreeSegmentPage); + + OldIrql = MiAcquirePfnLock(); + /* Delete the pointer on the file */ + ASSERT(Segment->FileObject->SectionObjectPointer->DataSectionObject == Segment); + Segment->FileObject->SectionObjectPointer->DataSectionObject = NULL; + MiReleasePfnLock(OldIrql); + ObDereferenceObject(Segment->FileObject); + + ExFreePoolWithTag(Segment, TAG_MM_SECTION_SEGMENT); + } + else + { + /* Most grotesque thing ever */ + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = CONTAINING_RECORD(Segment->ReferenceCount, MM_IMAGE_SECTION_OBJECT, RefCount); PMM_SECTION_SEGMENT SectionSegments; ULONG NrSegments; ULONG i; - ImageSectionObject = (PMM_IMAGE_SECTION_OBJECT)FileObject->SectionObjectPointer->ImageSectionObject; + OldIrql = MiAcquirePfnLock(); + /* Delete the pointer on the file */ + ASSERT(ImageSectionObject->FileObject->SectionObjectPointer->ImageSectionObject == ImageSectionObject); + ImageSectionObject->FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + + ObDereferenceObject(ImageSectionObject->FileObject); + NrSegments = ImageSectionObject->NrSegments; SectionSegments = ImageSectionObject->Segments; for (i = 0; i < NrSegments; i++) { - if (SectionSegments[i].ReferenceCount != 0) + if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) { - DPRINT1("Image segment %lu still referenced (was %lu)\n", i, - SectionSegments[i].ReferenceCount); - KeBugCheck(MEMORY_MANAGEMENT); + MmpFreePageFileSegment(&SectionSegments[i]); } + MmFreePageTablesSectionSegment(&SectionSegments[i], NULL); } - ObDereferenceObject(ImageSectionObject->FileObject); - ExFreePool(ImageSectionObject->Segments); - ExFreePool(ImageSectionObject); - FileObject->SectionObjectPointer->ImageSectionObject = NULL; - } - if (FileObject->SectionObjectPointer->DataSectionObject != NULL) - { - PMM_SECTION_SEGMENT Segment; - Segment = (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> - DataSectionObject; - - if (Segment->ReferenceCount != 0) - { - DPRINT1("Data segment still referenced\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - ObDereferenceObject(Segment->FileObject); - MmFreePageTablesSectionSegment(Segment, NULL); - ExFreePool(Segment); - FileObject->SectionObjectPointer->DataSectionObject = NULL; + ExFreePoolWithTag(ImageSectionObject->Segments, TAG_MM_SECTION_SEGMENT); + ExFreePoolWithTag(ImageSectionObject, TAG_MM_SECTION_SEGMENT); } } @@ -908,7 +1057,9 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, ULONG_PTR *InEntry) { ULONG_PTR Entry = InEntry ? *InEntry : MmGetPageEntrySectionSegment(Segment, Offset); - BOOLEAN IsDirectMapped = FALSE; + PFN_NUMBER Page = PFN_FROM_SSE(Entry); + ULONG_PTR NewEntry = 0; + SWAPENTRY SwapEntry; if (Entry == 0) { @@ -924,142 +1075,66 @@ MmUnsharePageEntrySectionSegment(PMEMORY_AREA MemoryArea, { KeBugCheck(MEMORY_MANAGEMENT); } + Dirty = Dirty || IS_DIRTY_SSE(Entry); Entry = MAKE_SSE(PAGE_FROM_SSE(Entry), SHARE_COUNT_FROM_SSE(Entry) - 1); - /* - * If we reducing the share count of this entry to zero then set the entry - * to zero and tell the cache the page is no longer mapped. - */ - if (SHARE_COUNT_FROM_SSE(Entry) == 0) - { - PFILE_OBJECT FileObject; - SWAPENTRY SavedSwapEntry; - PFN_NUMBER Page; -#ifndef NEWCC - PROS_SHARED_CACHE_MAP SharedCacheMap; - BOOLEAN IsImageSection; - LARGE_INTEGER FileOffset; - - FileOffset.QuadPart = Offset->QuadPart + Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; -#endif - - Page = PFN_FROM_SSE(Entry); - FileObject = Segment->FileObject; - if (FileObject != NULL && - !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - -#ifndef NEWCC - if ((FileOffset.QuadPart % PAGE_SIZE) == 0 && - (Offset->QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) - { - NTSTATUS Status; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - IsDirectMapped = TRUE; -#ifndef NEWCC - Status = CcRosUnmapVacb(SharedCacheMap, FileOffset.QuadPart, Dirty); -#else - Status = STATUS_SUCCESS; -#endif - if (!NT_SUCCESS(Status)) - { - DPRINT1("CcRosUnmapVacb failed, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } -#endif - } - - SavedSwapEntry = MmGetSavedSwapEntryPage(Page); - if (SavedSwapEntry == 0) - { - if (!PageOut && (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - /* - * FIXME: - * Try to page out this page and set the swap entry - * within the section segment. There exist no rmap entry - * for this page. The pager thread can't page out a - * page without a rmap entry. - */ - MmSetPageEntrySectionSegment(Segment, Offset, Entry); - if (InEntry) *InEntry = Entry; - MiSetPageEvent(NULL, NULL); - } - else - { - MmSetPageEntrySectionSegment(Segment, Offset, 0); - if (InEntry) *InEntry = 0; - MiSetPageEvent(NULL, NULL); - if (!IsDirectMapped) - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } - } - } - else - { - if (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - if (!PageOut) - { - if (Dirty) - { - /* - * FIXME: - * We hold all locks. Nobody can do something with the current - * process and the current segment (also not within an other process). - */ - NTSTATUS Status; - Status = MmWriteToSwapPage(SavedSwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - MmSetPageEntrySectionSegment(Segment, Offset, MAKE_SWAP_SSE(SavedSwapEntry)); - if (InEntry) *InEntry = MAKE_SWAP_SSE(SavedSwapEntry); - MmSetSavedSwapEntryPage(Page, 0); - MiSetPageEvent(NULL, NULL); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - } - else - { - DPRINT1("Found a swapentry for a non private page in an image or data file sgment\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - } - else + if (Dirty) Entry = DIRTY_SSE(Entry); + + if (SHARE_COUNT_FROM_SSE(Entry) > 0) { + /* Update the page mapping in the segment and we're done */ if (InEntry) *InEntry = Entry; else MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; } - return(SHARE_COUNT_FROM_SSE(Entry) > 0); -} -BOOLEAN MiIsPageFromCache(PMEMORY_AREA MemoryArea, - LONGLONG SegOffset) -{ -#ifndef NEWCC - if (!(MemoryArea->SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + if (IS_DIRTY_SSE(Entry) && !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { - PROS_SHARED_CACHE_MAP SharedCacheMap; - PROS_VACB Vacb; - SharedCacheMap = MemoryArea->SectionData.Segment->FileObject->SectionObjectPointer->SharedCacheMap; - Vacb = CcRosLookupVacb(SharedCacheMap, SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset); - if (Vacb) + ASSERT(!Segment->WriteCopy); + ASSERT(MmGetSavedSwapEntryPage(Page) == 0); + + /* The entry must be written back to the disk, so let this in the segment, the page-out thread will take care of this */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } + + SwapEntry = MmGetSavedSwapEntryPage(Page); + if (Dirty && !SwapEntry) + { + SwapEntry = MmAllocSwapPage(); + if (!SwapEntry) { - CcRosReleaseVacb(SharedCacheMap, Vacb, Vacb->Valid, FALSE, TRUE); - return TRUE; + /* We can't have a swap entry for this page. Let the segment keep it */ + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; } } -#endif - return FALSE; + + if (Dirty) + { + NTSTATUS Status = MmWriteToSwapPage(SwapEntry, Page); + if (!NT_SUCCESS(Status)) + { + /* We failed. Clean up */ + MmSetSavedSwapEntryPage(Page, 0); + MmFreeSwapPage(SwapEntry); + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + return FALSE; + } + } + + if (SwapEntry) + { + NewEntry = MAKE_SWAP_SSE(SwapEntry); + MmSetSavedSwapEntryPage(Page, 0); + } + + /* We can let this go */ + MmSetPageEntrySectionSegment(Segment, Offset, NewEntry); + MmReleasePageMemoryConsumer(MC_USER, Page); + MiSetPageEvent(NULL, NULL); + return TRUE; } NTSTATUS @@ -1097,175 +1172,46 @@ MiReadPage(PMEMORY_AREA MemoryArea, * Page - Variable that receives a page contains the read data. */ { - LONGLONG BaseOffset; - LONGLONG FileOffset; - PVOID BaseAddress; - BOOLEAN UptoDate; - PROS_VACB Vacb; - PFILE_OBJECT FileObject; NTSTATUS Status; - LONGLONG RawLength; - PROS_SHARED_CACHE_MAP SharedCacheMap; - BOOLEAN IsImageSection; - LONGLONG Length; + IO_STATUS_BLOCK IoStatus; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(PFN_NUMBER)]; + PMDL Mdl = (PMDL)MdlBase; + PFILE_OBJECT FileObject = MemoryArea->SectionData.Segment->FileObject; + LARGE_INTEGER FileOffset; + PFSRTL_COMMON_FCB_HEADER Fcb = FileObject->FsContext; - FileObject = MemoryArea->SectionData.Segment->FileObject; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - RawLength = MemoryArea->SectionData.Segment->RawLength.QuadPart; - FileOffset = SegOffset + MemoryArea->SectionData.Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; + FileOffset.QuadPart = MemoryArea->SectionData.Segment->Image.FileOffset + SegOffset; - ASSERT(SharedCacheMap); + Status = MmRequestPageMemoryConsumer(MC_USER, FALSE, Page); + if (!NT_SUCCESS(Status)) + return Status; - DPRINT("%S %I64x\n", FileObject->FileName.Buffer, FileOffset); + /* Check if we are beyond the file */ + if (FileOffset.QuadPart > Fcb->FileSize.QuadPart) + return STATUS_SUCCESS; - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then get the related VACB. - */ - if (((FileOffset % PAGE_SIZE) == 0) && - ((SegOffset + PAGE_SIZE <= RawLength) || !IsImageSection) && - !(MemoryArea->SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) + RtlZeroMemory(MdlBase, sizeof(MdlBase)); + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED | MDL_IO_PAGE_READ; + + KeInitializeEvent(&Event, NotificationEvent, FALSE); + + Status = IoPageRead(FileObject, Mdl, &FileOffset, &Event, &IoStatus); + if (Status == STATUS_PENDING) { - - /* - * Get the related VACB; we use a lower level interface than - * filesystems do because it is safe for us to use an offset with an - * alignment less than the file system block size. - */ - Status = CcRosGetVacb(SharedCacheMap, - FileOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } - - /* Probe the page, since it's PDE might not be synced */ - (void)*((volatile char*)BaseAddress + FileOffset - BaseOffset); - - /* - * Retrieve the page from the view that we actually want. - */ - (*Page) = MmGetPhysicalAddress((char*)BaseAddress + - FileOffset - BaseOffset).LowPart >> PAGE_SHIFT; - - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, TRUE); + KeWaitForSingleObject(&Event, WrPageIn, KernelMode, FALSE, NULL); + Status = IoStatus.Status; } - else + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) { - PEPROCESS Process; - KIRQL Irql; - PVOID PageAddr; - LONGLONG VacbOffset; - - /* - * Allocate a page, this is rather complicated by the possibility - * we might have to move other things out of memory - */ - MI_SET_USAGE(MI_USAGE_SECTION); - MI_SET_PROCESS2(PsGetCurrentProcess()->ImageFileName); - Status = MmRequestPageMemoryConsumer(MC_USER, TRUE, Page); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - Status = CcRosGetVacb(SharedCacheMap, - FileOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } - - Process = PsGetCurrentProcess(); - PageAddr = MiMapPageInHyperSpace(Process, *Page, &Irql); - VacbOffset = BaseOffset + VACB_MAPPING_GRANULARITY - FileOffset; - Length = RawLength - SegOffset; - if (Length <= VacbOffset && Length <= PAGE_SIZE) - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, Length); - } - else if (VacbOffset >= PAGE_SIZE) - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, PAGE_SIZE); - } - else - { - memcpy(PageAddr, (char*)BaseAddress + FileOffset - BaseOffset, VacbOffset); - MiUnmapPageInHyperSpace(Process, PageAddr, Irql); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); - Status = CcRosGetVacb(SharedCacheMap, - FileOffset + VacbOffset, - &BaseOffset, - &BaseAddress, - &UptoDate, - &Vacb); - if (!NT_SUCCESS(Status)) - { - return(Status); - } - if (!UptoDate) - { - /* - * If the VACB isn't up to date then call the file - * system to read in the data. - */ - Status = CcReadVirtualAddress(Vacb); - if (!NT_SUCCESS(Status)) - { - CcRosReleaseVacb(SharedCacheMap, Vacb, FALSE, FALSE, FALSE); - return Status; - } - } - PageAddr = MiMapPageInHyperSpace(Process, *Page, &Irql); - if (Length < PAGE_SIZE) - { - memcpy((char*)PageAddr + VacbOffset, BaseAddress, Length - VacbOffset); - } - else - { - memcpy((char*)PageAddr + VacbOffset, BaseAddress, PAGE_SIZE - VacbOffset); - } - } - MiUnmapPageInHyperSpace(Process, PageAddr, Irql); - CcRosReleaseVacb(SharedCacheMap, Vacb, TRUE, FALSE, FALSE); + MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); } - return(STATUS_SUCCESS); + + return Status; } + #else NTSTATUS NTAPI @@ -1571,7 +1517,7 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, /* * Add the page to the process's working set */ - MmInsertRmap(Page, Process, Address); + if (Process) MmInsertRmap(Page, Process, Address); /* * Finish the operation */ @@ -1684,7 +1630,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, KeBugCheck(MEMORY_MANAGEMENT); } ASSERT(MmIsPagePresent(Process, PAddress)); - MmInsertRmap(Page, Process, Address); + if (Process) + MmInsertRmap(Page, Process, Address); /* Set this section offset has being backed by our new page. */ Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); @@ -1765,7 +1712,8 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, DPRINT1("Unable to create virtual mapping\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmInsertRmap(Page, Process, Address); + if (Process) + MmInsertRmap(Page, Process, Address); /* * Mark the offset within the section as having valid, in-memory @@ -1794,7 +1742,9 @@ MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, DPRINT1("Unable to create virtual mapping\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmInsertRmap(Page, Process, Address); + + if (Process) + MmInsertRmap(Page, Process, Address); /* Take a reference on it */ MmSharePageEntrySectionSegment(Segment, &Offset); @@ -1905,7 +1855,8 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, */ DPRINT("Swapping page (Old %x New %x)\n", OldPage, NewPage); MmDeleteVirtualMapping(Process, PAddress, NULL, NULL); - MmDeleteRmap(OldPage, Process, PAddress); + if (Process) + MmDeleteRmap(OldPage, Process, PAddress); MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, FALSE, FALSE, NULL); MmUnlockSectionSegment(Segment); @@ -1923,567 +1874,15 @@ MmAccessFaultSectionView(PMMSUPPORT AddressSpace, KeBugCheck(MEMORY_MANAGEMENT); return(Status); } - MmInsertRmap(NewPage, Process, PAddress); + + if (Process) + MmInsertRmap(NewPage, Process, PAddress); MiSetPageEvent(Process, Address); DPRINT("Address 0x%p\n", Address); return(STATUS_SUCCESS); } -VOID -MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address) -{ - MM_SECTION_PAGEOUT_CONTEXT* PageOutContext; - BOOLEAN WasDirty; - PFN_NUMBER Page = 0; - - PageOutContext = (MM_SECTION_PAGEOUT_CONTEXT*)Context; - if (Process) - { - MmLockAddressSpace(&Process->Vm); - } - - MmDeleteVirtualMapping(Process, - Address, - &WasDirty, - &Page); - if (WasDirty) - { - PageOutContext->WasDirty = TRUE; - } - if (!PageOutContext->Private) - { - MmLockSectionSegment(PageOutContext->Segment); - MmUnsharePageEntrySectionSegment(PageOutContext->MemoryArea, - PageOutContext->Segment, - &PageOutContext->Offset, - PageOutContext->WasDirty, - TRUE, - &PageOutContext->SectionEntry); - MmUnlockSectionSegment(PageOutContext->Segment); - } - if (Process) - { - MmUnlockAddressSpace(&Process->Vm); - } - - if (PageOutContext->Private) - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } -} - -NTSTATUS -NTAPI -MmPageOutSectionView(PMMSUPPORT AddressSpace, - MEMORY_AREA* MemoryArea, - PVOID Address, ULONG_PTR Entry) -{ - PFN_NUMBER Page; - MM_SECTION_PAGEOUT_CONTEXT Context; - SWAPENTRY SwapEntry; - NTSTATUS Status; -#ifndef NEWCC - ULONGLONG FileOffset; - PFILE_OBJECT FileObject; - PROS_SHARED_CACHE_MAP SharedCacheMap = NULL; - BOOLEAN IsImageSection; -#endif - BOOLEAN DirectMapped; - PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - KIRQL OldIrql; - - Address = (PVOID)PAGE_ROUND_DOWN(Address); - - /* - * Get the segment and section. - */ - Context.Segment = MemoryArea->SectionData.Segment; - Context.MemoryArea = MemoryArea; - Context.SectionEntry = Entry; - Context.CallingProcess = Process; - - Context.Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - - DirectMapped = FALSE; - - MmLockSectionSegment(Context.Segment); - -#ifndef NEWCC - FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; - FileObject = Context.Segment->FileObject; - - if (FileObject != NULL && - !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then note this is a direct mapped page. - */ - if ((FileOffset % PAGE_SIZE) == 0 && - (Context.Offset.QuadPart + PAGE_SIZE <= Context.Segment->RawLength.QuadPart || !IsImageSection)) - { - DirectMapped = TRUE; - } - } -#endif - - /* - * Get the section segment entry and the physical address. - */ - if (!MmIsPagePresent(Process, Address)) - { - DPRINT1("Trying to page out not-present page at (%p,0x%p).\n", - Process ? Process->UniqueProcessId : 0, Address); - KeBugCheck(MEMORY_MANAGEMENT); - } - Page = MmGetPfnForProcess(Process, Address); - SwapEntry = MmGetSavedSwapEntryPage(Page); - - /* - * Check the reference count to ensure this page can be paged out - */ - if (MmGetReferenceCountPage(Page) != 1) - { - DPRINT("Cannot page out locked section page: 0x%lu (RefCount: %lu)\n", - Page, MmGetReferenceCountPage(Page)); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - return STATUS_UNSUCCESSFUL; - } - - /* - * Prepare the context structure for the rmap delete call. - */ - MmUnlockSectionSegment(Context.Segment); - Context.WasDirty = FALSE; - if (IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) - { - Context.Private = TRUE; - } - else - { - Context.Private = FALSE; - } - - /* - * Take an additional reference to the page or the VACB. - */ - if (DirectMapped && !Context.Private) - { - if(!MiIsPageFromCache(MemoryArea, Context.Offset.QuadPart)) - { - DPRINT1("Direct mapped non private page is not associated with the cache.\n"); - KeBugCheck(MEMORY_MANAGEMENT); - } - } - else - { - OldIrql = MiAcquirePfnLock(); - MmReferencePage(Page); - MiReleasePfnLock(OldIrql); - } - - MmDeleteAllRmaps(Page, (PVOID)&Context, MmPageOutDeleteMapping); - - /* Since we passed in a surrogate, we'll get back the page entry - * state in our context. This is intended to make intermediate - * decrements of share count not release the wait entry. - */ - Entry = Context.SectionEntry; - - /* - * If this wasn't a private page then we should have reduced the entry to - * zero by deleting all the rmaps. - */ - if (!Context.Private && Entry != 0) - { - if (!(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { - KeBugCheckEx(MEMORY_MANAGEMENT, Entry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - } - - /* - * If the page wasn't dirty then we can just free it as for a readonly page. - * Since we unmapped all the mappings above we know it will not suddenly - * become dirty. - * If the page is from a pagefile section and has no swap entry, - * we can't free the page at this point. - */ - SwapEntry = MmGetSavedSwapEntryPage(Page); - if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - if (Context.Private) - { - DPRINT1("Found a %s private page (address %p) in a shared section segment.\n", - Context.WasDirty ? "dirty" : "clean", Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Page, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); - } - if (!Context.WasDirty || SwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - if (SwapEntry != 0) - { - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - } - else if (!Context.Private && DirectMapped) - { - if (SwapEntry != 0) - { - DPRINT1("Found a swapentry for a non private and direct mapped page (address %p)\n", - Address); - KeBugCheckEx(MEMORY_MANAGEMENT, STATUS_UNSUCCESSFUL, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address); - } -#ifndef NEWCC - Status = CcRosUnmapVacb(SharedCacheMap, FileOffset, FALSE); -#else - Status = STATUS_SUCCESS; -#endif -#ifndef NEWCC - if (!NT_SUCCESS(Status)) - { - DPRINT1("CcRosUnmapVacb failed, status = %x\n", Status); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)SharedCacheMap, (ULONG_PTR)FileOffset, (ULONG_PTR)Address); - } -#endif - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - else if (!Context.WasDirty && !DirectMapped && !Context.Private) - { - if (SwapEntry != 0) - { - DPRINT1("Found a swap entry for a non dirty, non private and not direct mapped page (address %p)\n", - Address); - KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process, (ULONG_PTR)Address); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - else if (!Context.WasDirty && Context.Private && SwapEntry != 0) - { - DPRINT("Not dirty and private and not swapped (%p:%p)\n", Process, Address); - MmSetSavedSwapEntryPage(Page, 0); - MmLockAddressSpace(AddressSpace); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); - MmUnlockAddressSpace(AddressSpace); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Status %x Swapping out %p:%p\n", Status, Process, Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - - /* - * If necessary, allocate an entry in the paging file for this page - */ - if (SwapEntry == 0) - { - SwapEntry = MmAllocSwapPage(); - if (SwapEntry == 0) - { - MmShowOutOfSpaceMessagePagingFile(); - MmLockAddressSpace(AddressSpace); - /* - * For private pages restore the old mappings. - */ - if (Context.Private) - { - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - } - else - { - ULONG_PTR OldEntry; - - MmLockSectionSegment(Context.Segment); - - /* - * For non-private pages if the page wasn't direct mapped then - * set it back into the section segment entry so we don't loose - * our copy. Otherwise it will be handled by the cache manager. - */ - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - // If we got here, the previous entry should have been a wait - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - OldEntry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset); - ASSERT(OldEntry == 0 || OldEntry == MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - } - MmUnlockAddressSpace(AddressSpace); - MiSetPageEvent(NULL, NULL); - return(STATUS_PAGEFILE_QUOTA); - } - } - - /* - * Write the page to the pagefile - */ - Status = MmWriteToSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", - Status); - /* - * As above: undo our actions. - * FIXME: Also free the swap page. - */ - MmLockAddressSpace(AddressSpace); - if (Context.Private) - { - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - } - else - { - MmLockSectionSegment(Context.Segment); - Status = MmCreateVirtualMapping(Process, - Address, - MmProtectToValue[MemoryArea->VadNode.u.VadFlags.Protection], - &Page, - 1); - MmSetDirtyPage(Process, Address); - MmInsertRmap(Page, - Process, - Address); - Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - } - MmUnlockAddressSpace(AddressSpace); - MiSetPageEvent(NULL, NULL); - return(STATUS_UNSUCCESSFUL); - } - - /* - * Otherwise we have succeeded. - */ - DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MmSetSavedSwapEntryPage(Page, 0); - if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmLockSectionSegment(Context.Segment); - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); - MmUnlockSectionSegment(Context.Segment); - } - else - { - MmReleasePageMemoryConsumer(MC_USER, Page); - } - - if (Context.Private) - { - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Context.Segment); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - MmUnlockAddressSpace(AddressSpace); - if (!NT_SUCCESS(Status)) - { - DPRINT1("Status %x Creating page file mapping for %p:%p\n", Status, Process, Address); - KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); - } - } - else - { - MmLockAddressSpace(AddressSpace); - MmLockSectionSegment(Context.Segment); - Entry = MAKE_SWAP_SSE(SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); - MmUnlockSectionSegment(Context.Segment); - MmUnlockAddressSpace(AddressSpace); - } - - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); -} - -NTSTATUS -NTAPI -MmWritePageSectionView(PMMSUPPORT AddressSpace, - PMEMORY_AREA MemoryArea, - PVOID Address, - ULONG PageEntry) -{ - LARGE_INTEGER Offset; - PMM_SECTION_SEGMENT Segment; - PFN_NUMBER Page; - SWAPENTRY SwapEntry; - ULONG_PTR Entry; - BOOLEAN Private; - NTSTATUS Status; - PFILE_OBJECT FileObject; -#ifndef NEWCC - PROS_SHARED_CACHE_MAP SharedCacheMap = NULL; -#endif - BOOLEAN DirectMapped; - BOOLEAN IsImageSection; - PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - - Address = (PVOID)PAGE_ROUND_DOWN(Address); - - Offset.QuadPart = (ULONG_PTR)Address - MA_GetStartingAddress(MemoryArea) - + MemoryArea->SectionData.ViewOffset.QuadPart; - - /* - * Get the segment and section. - */ - Segment = MemoryArea->SectionData.Segment; - IsImageSection = MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap; - - FileObject = Segment->FileObject; - DirectMapped = FALSE; - if (FileObject != NULL && - !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) - { -#ifndef NEWCC - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; -#endif - - /* - * If the file system is letting us go directly to the cache and the - * memory area was mapped at an offset in the file which is page aligned - * then note this is a direct mapped page. - */ - if (((Offset.QuadPart + Segment->Image.FileOffset) % PAGE_SIZE) == 0 && - (Offset.QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) - { - DirectMapped = TRUE; - } - } - - /* - * Get the section segment entry and the physical address. - */ - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - if (!MmIsPagePresent(Process, Address)) - { - DPRINT1("Trying to page out not-present page at (%p,0x%p).\n", - Process ? Process->UniqueProcessId : 0, Address); - KeBugCheck(MEMORY_MANAGEMENT); - } - Page = MmGetPfnForProcess(Process, Address); - SwapEntry = MmGetSavedSwapEntryPage(Page); - - /* - * Check for a private (COWed) page. - */ - if (IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) - { - Private = TRUE; - } - else - { - Private = FALSE; - } - - /* - * Speculatively set all mappings of the page to clean. - */ - MmSetCleanAllRmaps(Page); - - /* - * If this page was direct mapped from the cache then the cache manager - * will take care of writing it back to disk. - */ - if (DirectMapped && !Private) - { - //LARGE_INTEGER SOffset; - ASSERT(SwapEntry == 0); - //SOffset.QuadPart = Offset.QuadPart + Segment->Image.FileOffset; -#ifndef NEWCC - CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart); -#endif - MmLockSectionSegment(Segment); - MmSetPageEntrySectionSegment(Segment, &Offset, PageEntry); - MmUnlockSectionSegment(Segment); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); - } - - /* - * If necessary, allocate an entry in the paging file for this page - */ - if (SwapEntry == 0) - { - SwapEntry = MmAllocSwapPage(); - if (SwapEntry == 0) - { - MmSetDirtyAllRmaps(Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_PAGEFILE_QUOTA); - } - MmSetSavedSwapEntryPage(Page, SwapEntry); - } - - /* - * Write the page to the pagefile - */ - Status = MmWriteToSwapPage(SwapEntry, Page); - if (!NT_SUCCESS(Status)) - { - DPRINT1("MM: Failed to write to swap page (Status was 0x%.8X)\n", - Status); - MmSetDirtyAllRmaps(Page); - MiSetPageEvent(NULL, NULL); - return(STATUS_UNSUCCESSFUL); - } - - /* - * Otherwise we have succeeded. - */ - DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); - MiSetPageEvent(NULL, NULL); - return(STATUS_SUCCESS); -} - NTSTATUS NTAPI MmProtectSectionView(PMMSUPPORT AddressSpace, @@ -2560,48 +1959,6 @@ MmQuerySectionView(PMEMORY_AREA MemoryArea, return(STATUS_SUCCESS); } -VOID -NTAPI -MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) -{ - ULONG Length; - LARGE_INTEGER Offset; - ULONG_PTR Entry; - SWAPENTRY SavedSwapEntry; - PFN_NUMBER Page; - - Page = 0; - - MmLockSectionSegment(Segment); - - Length = PAGE_ROUND_UP(Segment->Length.QuadPart); - for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) - { - Entry = MmGetPageEntrySectionSegment(Segment, &Offset); - if (Entry) - { - MmSetPageEntrySectionSegment(Segment, &Offset, 0); - if (IS_SWAP_FROM_SSE(Entry)) - { - MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); - } - else - { - Page = PFN_FROM_SSE(Entry); - SavedSwapEntry = MmGetSavedSwapEntryPage(Page); - if (SavedSwapEntry != 0) - { - MmSetSavedSwapEntryPage(Page, 0); - MmFreeSwapPage(SavedSwapEntry); - } - MmReleasePageMemoryConsumer(MC_USER, Page); - } - } - } - - MmUnlockSectionSegment(Segment); -} - VOID NTAPI MmpDeleteSection(PVOID ObjectBody) { @@ -2617,10 +1974,7 @@ MmpDeleteSection(PVOID ObjectBody) DPRINT("MmpDeleteSection(ObjectBody %p)\n", ObjectBody); if (Section->u.Flags.Image) { - ULONG i; - ULONG NrSegments; - ULONG RefCount; - PMM_SECTION_SEGMENT SectionSegments; + PMM_IMAGE_SECTION_OBJECT ImageSectionObject = (PMM_IMAGE_SECTION_OBJECT)Section->Segment; /* * NOTE: Section->ImageSection can be NULL for short time @@ -2631,25 +1985,9 @@ MmpDeleteSection(PVOID ObjectBody) if (Section->Segment == NULL) return; - SectionSegments = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->Segments; - NrSegments = ((PMM_IMAGE_SECTION_OBJECT)Section->Segment)->NrSegments; - - for (i = 0; i < NrSegments; i++) - { - if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmLockSectionSegment(&SectionSegments[i]); - } - RefCount = InterlockedDecrementUL(&SectionSegments[i].ReferenceCount); - if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) - { - MmUnlockSectionSegment(&SectionSegments[i]); - if (RefCount == 0) - { - MmpFreePageFileSegment(&SectionSegments[i]); - } - } - } + /* We just dereference the first segment */ + ASSERT(ImageSectionObject->RefCount > 0); + MmDereferenceSegment(ImageSectionObject->Segments); } #ifdef NEWCC else if (Section->Segment && Section->Segment->Flags & MM_DATAFILE_SEGMENT) @@ -2672,25 +2010,17 @@ MmpDeleteSection(PVOID ObjectBody) #endif else { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + /* * NOTE: Section->Segment can be NULL for short time * during the section creating. */ - if (Section->Segment == NULL) + if (Segment == NULL) return; - (void)InterlockedDecrementUL(&((PMM_SECTION_SEGMENT)Section->Segment)->ReferenceCount); - } - - if (Section->Segment) - { - PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; - if (Segment->FileObject != NULL) - { - #ifndef NEWCC - CcRosDereferenceCache(Segment->FileObject); - #endif - } + Segment->SectionCount--; + MmDereferenceSegment(Segment); } } @@ -2763,13 +2093,17 @@ MmCreatePhysicalMemorySection(VOID) } RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); PhysSection->Segment = (PSEGMENT)Segment; - Segment->ReferenceCount = 1; + Segment->RefCount = 1; + + Segment->ReferenceCount = &Segment->RefCount; + Segment->Flags = &Segment->SegFlags; + ExInitializeFastMutex(&Segment->Lock); Segment->Image.FileOffset = 0; Segment->Protection = PAGE_EXECUTE_READWRITE; Segment->RawLength = SectionSize; Segment->Length = SectionSize; - Segment->Flags = 0; + Segment->SegFlags = 0; Segment->WriteCopy = FALSE; Segment->Image.VirtualAddress = 0; Segment->Image.Characteristics = 0; @@ -2823,6 +2157,7 @@ MmInitSectionImplementation(VOID) return(STATUS_SUCCESS); } +static NTSTATUS NTAPI MmCreateDataFileSection(PSECTION *SectionObject, @@ -2831,7 +2166,8 @@ MmCreateDataFileSection(PSECTION *SectionObject, PLARGE_INTEGER UMaximumSize, ULONG SectionPageProtection, ULONG AllocationAttributes, - PFILE_OBJECT FileObject) + PFILE_OBJECT FileObject, + BOOLEAN GotFileHandle) /* * Create a section backed by a data file */ @@ -2840,8 +2176,7 @@ MmCreateDataFileSection(PSECTION *SectionObject, NTSTATUS Status; LARGE_INTEGER MaximumSize; PMM_SECTION_SEGMENT Segment; - FILE_STANDARD_INFORMATION FileInfo; - ULONG Length; + KIRQL OldIrql; /* * Create the section @@ -2869,63 +2204,63 @@ MmCreateDataFileSection(PSECTION *SectionObject, Section->u.Flags.filler = 1; Section->InitialPageProtection = SectionPageProtection; Section->u.Flags.File = 1; + if (AllocationAttributes & SEC_NO_CHANGE) Section->u.Flags.NoChange = 1; - /* - * FIXME: This is propably not entirely correct. We can't look into - * the standard FCB header because it might not be initialized yet - * (as in case of the EXT2FS driver by Manoj Paul Joseph where the - * standard file information is filled on first request). - */ - Status = IoQueryFileInformation(FileObject, - FileStandardInformation, - sizeof(FILE_STANDARD_INFORMATION), - &FileInfo, - &Length); - if (!NT_SUCCESS(Status)) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return Status; - } - - /* - * FIXME: Revise this once a locking order for file size changes is - * decided - */ - if ((UMaximumSize != NULL) && (UMaximumSize->QuadPart != 0)) + if (!GotFileHandle) { + ASSERT(UMaximumSize != NULL); + ASSERT(UMaximumSize->QuadPart != 0); MaximumSize = *UMaximumSize; } else { - MaximumSize = FileInfo.EndOfFile; - /* Mapping zero-sized files isn't allowed. */ - if (MaximumSize.QuadPart == 0) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return STATUS_MAPPED_FILE_SIZE_ZERO; - } - } - - if (MaximumSize.QuadPart > FileInfo.EndOfFile.QuadPart) - { - Status = IoSetInformation(FileObject, - FileEndOfFileInformation, - sizeof(LARGE_INTEGER), - &MaximumSize); + LARGE_INTEGER FileSize; + Status = FsRtlGetFileSize(FileObject, &FileSize); if (!NT_SUCCESS(Status)) { ObDereferenceObject(Section); ObDereferenceObject(FileObject); - return(STATUS_SECTION_NOT_EXTENDED); + return Status; + } + + /* + * FIXME: Revise this once a locking order for file size changes is + * decided + */ + if ((UMaximumSize != NULL) && (UMaximumSize->QuadPart != 0)) + { + MaximumSize = *UMaximumSize; + } + else + { + MaximumSize = FileSize; + /* Mapping zero-sized files isn't allowed. */ + if (MaximumSize.QuadPart == 0) + { + ObDereferenceObject(Section); + ObDereferenceObject(FileObject); + return STATUS_MAPPED_FILE_SIZE_ZERO; + } + } + + if (MaximumSize.QuadPart > FileSize.QuadPart) + { + Status = IoSetInformation(FileObject, + FileEndOfFileInformation, + sizeof(LARGE_INTEGER), + &MaximumSize); + if (!NT_SUCCESS(Status)) + { + ObDereferenceObject(Section); + ObDereferenceObject(FileObject); + return(STATUS_SECTION_NOT_EXTENDED); + } } } - if (FileObject->SectionObjectPointer == NULL || - FileObject->SectionObjectPointer->SharedCacheMap == NULL) + if (FileObject->SectionObjectPointer == NULL) { ObDereferenceObject(Section); ObDereferenceObject(FileObject); @@ -2943,34 +2278,61 @@ MmCreateDataFileSection(PSECTION *SectionObject, return(Status); } + /* Lock the PFN lock while messing with Section Object pointers */ + OldIrql = MiAcquirePfnLock(); + Segment = FileObject->SectionObjectPointer->DataSectionObject; + + while (Segment && (Segment->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + LARGE_INTEGER ShortTime = {{-10 * 100 * 1000, -1}}; + + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + OldIrql = MiAcquirePfnLock(); + Segment = FileObject->SectionObjectPointer->DataSectionObject; + } + /* * If this file hasn't been mapped as a data file before then allocate a * section segment to describe the data file mapping */ - if (FileObject->SectionObjectPointer->DataSectionObject == NULL) + if (Segment == NULL) { Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), TAG_MM_SECTION_SEGMENT); if (Segment == NULL) { //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); + MiReleasePfnLock(OldIrql); ObDereferenceObject(Section); ObDereferenceObject(FileObject); return(STATUS_NO_MEMORY); } + + /* We are creating it */ + RtlZeroMemory(Segment, sizeof(*Segment)); + Segment->SegFlags = MM_DATAFILE_SEGMENT | MM_SEGMENT_INCREATE; + Segment->RefCount = 1; + + FileObject->SectionObjectPointer->DataSectionObject = Segment; + + /* We're safe to release the lock now */ + MiReleasePfnLock(OldIrql); + Section->Segment = (PSEGMENT)Segment; - Segment->ReferenceCount = 1; + + /* Self-referencing segment */ + Segment->Flags = &Segment->SegFlags; + Segment->ReferenceCount = &Segment->RefCount; + + Segment->SectionCount = 1; + ExInitializeFastMutex(&Segment->Lock); Segment->FileObject = FileObject; - /* - * Set the lock before assigning the segment to the file object - */ - ExAcquireFastMutex(&Segment->Lock); - FileObject->SectionObjectPointer->DataSectionObject = (PVOID)Segment; Segment->Image.FileOffset = 0; Segment->Protection = SectionPageProtection; - Segment->Flags = MM_DATAFILE_SEGMENT; + Segment->Image.Characteristics = 0; Segment->WriteCopy = (SectionPageProtection & (PAGE_WRITECOPY | PAGE_EXECUTE_WRITECOPY)); if (AllocationAttributes & SEC_RESERVE) @@ -2983,20 +2345,21 @@ MmCreateDataFileSection(PSECTION *SectionObject, Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } Segment->Image.VirtualAddress = 0; - Segment->Locked = TRUE; MiInitializeSectionPageTable(Segment); + + /* We're good to use it now */ + OldIrql = MiAcquirePfnLock(); + Segment->SegFlags &= ~MM_SEGMENT_INCREATE; + MiReleasePfnLock(OldIrql); } else { - /* - * If the file is already mapped as a data file then we may need - * to extend it - */ - Segment = - (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> - DataSectionObject; Section->Segment = (PSEGMENT)Segment; - (void)InterlockedIncrementUL(&Segment->ReferenceCount); + Segment->RefCount++; + InterlockedIncrementUL(&Segment->SectionCount); + + MiReleasePfnLock(OldIrql); + MmLockSectionSegment(Segment); if (MaximumSize.QuadPart > Segment->RawLength.QuadPart && @@ -3006,15 +2369,13 @@ MmCreateDataFileSection(PSECTION *SectionObject, Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } - /* We let the segment reference the file object */ + MmUnlockSectionSegment(Segment); + + /* The segment already has a reference to a file object. Don't bother keeping one.*/ ObDereferenceObject(FileObject); - FileObject = Segment->FileObject; } - MmUnlockSectionSegment(Segment); Section->SizeOfSection = MaximumSize; -#ifndef NEWCC - CcRosReferenceCache(FileObject); -#endif + //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; return(STATUS_SUCCESS); @@ -3541,7 +2902,6 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, */ for (i = 0; i < RTL_NUMBER_OF(ExeFmtpLoaders); ++ i) { - RtlZeroMemory(ImageSectionObject, sizeof(*ImageSectionObject)); Flags = 0; Status = ExeFmtpLoaders[i](FileHeader, @@ -3580,6 +2940,7 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, return Status; ASSERT(ImageSectionObject->Segments != NULL); + ASSERT(ImageSectionObject->RefCount > 0); /* * Some defaults @@ -3640,11 +3001,14 @@ ExeFmtpCreateImageSection(PFILE_OBJECT FileObject, for ( i = 0; i < ImageSectionObject->NrSegments; ++ i ) { ExInitializeFastMutex(&ImageSectionObject->Segments[i].Lock); - ImageSectionObject->Segments[i].ReferenceCount = 1; + ImageSectionObject->Segments[i].ReferenceCount = &ImageSectionObject->RefCount; + ImageSectionObject->Segments[i].Flags = &ImageSectionObject->SegFlags; MiInitializeSectionPageTable(&ImageSectionObject->Segments[i]); ImageSectionObject->Segments[i].FileObject = FileObject; } + ASSERT(ImageSectionObject->RefCount > 0); + ImageSectionObject->FileObject = FileObject; ASSERT(NT_SUCCESS(Status)); @@ -3662,20 +3026,18 @@ MmCreateImageSection(PSECTION *SectionObject, { PSECTION Section; NTSTATUS Status; - PMM_SECTION_SEGMENT SectionSegments; PMM_IMAGE_SECTION_OBJECT ImageSectionObject; - ULONG i; + KIRQL OldIrql; + if (FileObject == NULL) return STATUS_INVALID_FILE_FOR_SECTION; -#ifndef NEWCC - if (!CcIsFileCached(FileObject)) + if (FileObject->SectionObjectPointer == NULL) { DPRINT1("Denying section creation due to missing cache initialization\n"); return STATUS_INVALID_FILE_FOR_SECTION; } -#endif /* * Create the section @@ -3709,13 +3071,31 @@ MmCreateImageSection(PSECTION *SectionObject, if (AllocationAttributes & SEC_NO_CHANGE) Section->u.Flags.NoChange = 1; - if (FileObject->SectionObjectPointer->ImageSectionObject == NULL) + OldIrql = MiAcquirePfnLock(); + + /* Wait for it to be properly created or deleted */ + ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; + while(ImageSectionObject && (ImageSectionObject->SegFlags & (MM_SEGMENT_INDELETE | MM_SEGMENT_INCREATE))) + { + LARGE_INTEGER ShortTime; + + MiReleasePfnLock(OldIrql); + + ShortTime.QuadPart = - 10 * 100 * 1000; + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + + OldIrql = MiAcquirePfnLock(); + ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; + } + + if (ImageSectionObject == NULL) { NTSTATUS StatusExeFmt; - ImageSectionObject = ExAllocatePoolWithTag(PagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); + ImageSectionObject = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_IMAGE_SECTION_OBJECT), TAG_MM_SECTION_SEGMENT); if (ImageSectionObject == NULL) { + MiReleasePfnLock(OldIrql); ObDereferenceObject(FileObject); ObDereferenceObject(Section); return(STATUS_NO_MEMORY); @@ -3723,10 +3103,21 @@ MmCreateImageSection(PSECTION *SectionObject, RtlZeroMemory(ImageSectionObject, sizeof(MM_IMAGE_SECTION_OBJECT)); + ImageSectionObject->SegFlags = MM_SEGMENT_INCREATE; + ImageSectionObject->RefCount = 1; + FileObject->SectionObjectPointer->ImageSectionObject = ImageSectionObject; + + MiReleasePfnLock(OldIrql); + StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); if (!NT_SUCCESS(StatusExeFmt)) { + /* Unset */ + OldIrql = MiAcquirePfnLock(); + FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + if(ImageSectionObject->Segments != NULL) ExFreePool(ImageSectionObject->Segments); @@ -3747,6 +3138,7 @@ MmCreateImageSection(PSECTION *SectionObject, Section->Segment = (PSEGMENT)ImageSectionObject; ASSERT(ImageSectionObject->Segments); + ASSERT(ImageSectionObject->RefCount > 0); /* * Lock the file @@ -3754,6 +3146,11 @@ MmCreateImageSection(PSECTION *SectionObject, Status = MmspWaitForFileLock(FileObject); if (!NT_SUCCESS(Status)) { + /* Unset */ + OldIrql = MiAcquirePfnLock(); + FileObject->SectionObjectPointer->ImageSectionObject = NULL; + MiReleasePfnLock(OldIrql); + ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); ObDereferenceObject(Section); @@ -3761,66 +3158,30 @@ MmCreateImageSection(PSECTION *SectionObject, return(Status); } - if (NULL != InterlockedCompareExchangePointer(&FileObject->SectionObjectPointer->ImageSectionObject, - ImageSectionObject, NULL)) - { - /* - * An other thread has initialized the same image in the background - */ - ExFreePool(ImageSectionObject->Segments); - ExFreePool(ImageSectionObject); - ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; - Section->Segment = (PSEGMENT)ImageSectionObject; - SectionSegments = ImageSectionObject->Segments; - - for (i = 0; i < ImageSectionObject->NrSegments; i++) - { - (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); - } - - /* We let the Image Section Object hold the reference */ - ObDereferenceObject(FileObject); - FileObject = ImageSectionObject->FileObject; - } + OldIrql = MiAcquirePfnLock(); + ImageSectionObject->SegFlags &= ~MM_SEGMENT_INCREATE; + MiReleasePfnLock(OldIrql); Status = StatusExeFmt; } else { - /* - * Lock the file - */ - Status = MmspWaitForFileLock(FileObject); - if (Status != STATUS_SUCCESS) - { - ObDereferenceObject(Section); - ObDereferenceObject(FileObject); - return(Status); - } + /* Take one ref */ + ImageSectionObject->RefCount++; + + MiReleasePfnLock(OldIrql); - ImageSectionObject = FileObject->SectionObjectPointer->ImageSectionObject; Section->Segment = (PSEGMENT)ImageSectionObject; - SectionSegments = ImageSectionObject->Segments; - - /* - * Otherwise just reference all the section segments - */ - for (i = 0; i < ImageSectionObject->NrSegments; i++) - { - (void)InterlockedIncrementUL(&SectionSegments[i].ReferenceCount); - } /* We let the Image Section Object hold the reference */ ObDereferenceObject(FileObject); - FileObject = ImageSectionObject->FileObject; Status = STATUS_SUCCESS; } -#ifndef NEWCC - CcRosReferenceCache(FileObject); -#endif //KeSetEvent((PVOID)&FileObject->Lock, IO_NO_INCREMENT, FALSE); *SectionObject = Section; + ASSERT(ImageSectionObject->RefCount > 0); + return(Status); } @@ -3908,10 +3269,6 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, PFN_NUMBER Page, SWAPENTRY SwapEntry, BOOLEAN Dirty) { ULONG_PTR Entry; -#ifndef NEWCC - PFILE_OBJECT FileObject; - PROS_SHARED_CACHE_MAP SharedCacheMap; -#endif LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; PMM_SECTION_SEGMENT Segment; @@ -3942,18 +3299,12 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, } /* - * For a dirty, datafile, non-private page mark it as dirty in the - * cache manager. + * For a dirty, datafile, non-private page, there shoulkd be no swap entry */ - if (Segment->Flags & MM_DATAFILE_SEGMENT) + if (*Segment->Flags & MM_DATAFILE_SEGMENT) { if (Page == PFN_FROM_SSE(Entry) && Dirty) { -#ifndef NEWCC - FileObject = MemoryArea->SectionData.Segment->FileObject; - SharedCacheMap = FileObject->SectionObjectPointer->SharedCacheMap; - CcRosMarkDirtyFile(SharedCacheMap, Offset.QuadPart + Segment->Image.FileOffset); -#endif ASSERT(SwapEntry == 0); } } @@ -3970,6 +3321,8 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, if (IS_SWAP_FROM_SSE(Entry) || Page != PFN_FROM_SSE(Entry)) { + ASSERT(Process != NULL); + /* * Just dereference private pages */ @@ -3984,7 +3337,8 @@ MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, } else { - MmDeleteRmap(Page, Process, Address); + if (Process) + MmDeleteRmap(Page, Process, Address); MmUnsharePageEntrySectionSegment(MemoryArea, Segment, &Offset, Dirty, FALSE, NULL); } } @@ -4467,6 +3821,8 @@ MmMapViewOfSection(IN PVOID SectionObject, SectionSegments = ImageSectionObject->Segments; NrSegments = ImageSectionObject->NrSegments; + ASSERT(ImageSectionObject->RefCount > 0); + ImageBase = (ULONG_PTR)*BaseAddress; if (ImageBase == 0) { @@ -4550,6 +3906,8 @@ MmMapViewOfSection(IN PVOID SectionObject, PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; LONGLONG ViewOffset; + ASSERT(Segment->RefCount > 0); + /* check for write access */ if ((Protect & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE)) && !(Section->InitialPageProtection & (PAGE_READWRITE|PAGE_EXECUTE_READWRITE))) @@ -4633,68 +3991,58 @@ BOOLEAN NTAPI MmCanFileBeTruncated (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, IN PLARGE_INTEGER NewFileSize) { + KIRQL OldIrql = MiAcquirePfnLock(); + BOOLEAN Ret; + PMM_SECTION_SEGMENT Segment; + +CheckSectionPointer: /* Check whether an ImageSectionObject exists */ if (SectionObjectPointer->ImageSectionObject != NULL) { DPRINT1("ERROR: File can't be truncated because it has an image section\n"); + MiReleasePfnLock(OldIrql); + return FALSE; } - if (SectionObjectPointer->DataSectionObject != NULL) + Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer->DataSectionObject; + /* Wait for it to be created/deleted properly */ + while (Segment && (Segment->SegFlags & (MM_SEGMENT_INCREATE | MM_SEGMENT_INDELETE))) { - PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER ShortTime; - Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer-> - DataSectionObject; + ShortTime.QuadPart = -10 * 100 * 1000; - if (Segment->ReferenceCount != 0) + /* Bad luck. Wait a bit for the operation to finish */ + MiReleasePfnLock(OldIrql); + KeDelayExecutionThread(KernelMode, FALSE, &ShortTime); + OldIrql = MiAcquirePfnLock(); + goto CheckSectionPointer; + } + + if (Segment) + { + if ((Segment->SectionCount == 1) && (SectionObjectPointer->SharedCacheMap != NULL)) { -#ifdef NEWCC - CC_FILE_SIZES FileSizes; - CcpLock(); - if (SectionObjectPointer->SharedCacheMap && (Segment->ReferenceCount > CcpCountCacheSections((PNOCC_CACHE_MAP)SectionObjectPointer->SharedCacheMap))) - { - CcpUnlock(); - /* Check size of file */ - if (SectionObjectPointer->SharedCacheMap) - { - if (!CcGetFileSizes(Segment->FileObject, &FileSizes)) - { - return FALSE; - } - - if (NewFileSize->QuadPart <= FileSizes.FileSize.QuadPart) - { - return FALSE; - } - } - } - else - CcpUnlock(); -#else - /* Check size of file */ - if (SectionObjectPointer->SharedCacheMap) - { - PROS_SHARED_CACHE_MAP SharedCacheMap = SectionObjectPointer->SharedCacheMap; - if (NewFileSize->QuadPart <= SharedCacheMap->FileSize.QuadPart) - { - return FALSE; - } - } -#endif + /* If the cache is the only one holding a reference to the segment, then it's fine to resize */ + Ret = TRUE; } else { - /* Something must gone wrong - * how can we have a Section but no - * reference? */ - DPRINT("ERROR: DataSectionObject without reference!\n"); + /* We can't shrink, but we can extend */ + Ret = NewFileSize->QuadPart > Segment->RawLength.QuadPart; } } + else + { + Ret = TRUE; + } + + MiReleasePfnLock(OldIrql); DPRINT("FIXME: didn't check for outstanding write probes\n"); - return TRUE; + return Ret; } @@ -4797,8 +4145,7 @@ MmMapViewInSystemSpaceEx ( MmLockAddressSpace(AddressSpace); - - if ((*ViewSize == 0) || ((SectionOffset->QuadPart + *ViewSize) > Section->SizeOfSection.QuadPart)) + if (*ViewSize == 0) { *ViewSize = MIN((Section->SizeOfSection.QuadPart - SectionOffset->QuadPart), SIZE_T_MAX); } @@ -4812,7 +4159,7 @@ MmMapViewInSystemSpaceEx ( *ViewSize, PAGE_READWRITE, SectionOffset->QuadPart, - 0); + SEC_RESERVE); MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); @@ -4946,13 +4293,8 @@ MmCreateSection (OUT PVOID * Section, return STATUS_INVALID_PARAMETER_6; } - /* Did the caller pass an object? */ - if (FileObject) - { - /* Reference the object directly */ - ObReferenceObject(FileObject); - } - else + /* Did the caller pass a handle? */ + if (FileHandle) { /* Reference the file handle to get the object */ Status = ObReferenceObjectByHandle(FileHandle, @@ -4967,6 +4309,11 @@ MmCreateSection (OUT PVOID * Section, return Status; } } + else + { + /* Reference the object directly */ + ObReferenceObject(FileObject); + } } else { @@ -4974,63 +4321,6 @@ MmCreateSection (OUT PVOID * Section, if (AllocationAttributes & SEC_IMAGE) return STATUS_INVALID_FILE_FOR_SECTION; } -#ifndef NEWCC // A hack for initializing caching. - // This is needed only in the old case. - if (FileHandle) - { - IO_STATUS_BLOCK Iosb; - NTSTATUS Status; - CHAR Buffer; - LARGE_INTEGER ByteOffset; - ByteOffset.QuadPart = 0; - Status = ZwReadFile(FileHandle, - NULL, - NULL, - NULL, - &Iosb, - &Buffer, - sizeof(Buffer), - &ByteOffset, - NULL); - if (!NT_SUCCESS(Status) && Status != STATUS_END_OF_FILE) - { - DPRINT1("CC failure: %lx\n", Status); - if (FileObject) - ObDereferenceObject(FileObject); - return Status; - } - // Caching is initialized... - - // Hack of the hack: actually, it might not be initialized if FSD init on effective right and if file is null-size - // In such case, force cache by initiating a write IRP - if (Status == STATUS_END_OF_FILE && !(AllocationAttributes & SEC_IMAGE) && FileObject != NULL && - (FileObject->SectionObjectPointer == NULL || FileObject->SectionObjectPointer->SharedCacheMap == NULL)) - { - Buffer = 0xdb; - Status = ZwWriteFile(FileHandle, - NULL, - NULL, - NULL, - &Iosb, - &Buffer, - sizeof(Buffer), - &ByteOffset, - NULL); - if (NT_SUCCESS(Status)) - { - LARGE_INTEGER Zero; - Zero.QuadPart = 0LL; - - Status = IoSetInformation(FileObject, - FileEndOfFileInformation, - sizeof(LARGE_INTEGER), - &Zero); - ASSERT(NT_SUCCESS(Status)); - } - } - } -#endif - if (AllocationAttributes & SEC_IMAGE) { Status = MmCreateImageSection(SectionObject, @@ -5042,7 +4332,7 @@ MmCreateSection (OUT PVOID * Section, FileObject); } #ifndef NEWCC - else if (FileHandle != NULL) + else if (FileObject != NULL) { Status = MmCreateDataFileSection(SectionObject, DesiredAccess, @@ -5050,7 +4340,8 @@ MmCreateSection (OUT PVOID * Section, MaximumSize, SectionPageProtection, AllocationAttributes, - FileObject); + FileObject, + FileHandle != NULL); } #else else if (FileHandle != NULL || FileObject != NULL) @@ -5066,7 +4357,7 @@ MmCreateSection (OUT PVOID * Section, #endif else { - /* All cases should be handled above, and the Physical Memorw section was created at initialization phase */ + /* All cases should be handled above */ ASSERT(FALSE); Status = STATUS_INVALID_PARAMETER; if (FileObject) @@ -5087,13 +4378,14 @@ MmArePagesResident( BOOLEAN Ret = TRUE; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); - MmLockAddressSpace(&Process->Vm); + MmLockAddressSpace(AddressSpace); - MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, Address); + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL) { - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); return FALSE; } @@ -5123,7 +4415,7 @@ MmArePagesResident( MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); return Ret; } @@ -5137,14 +4429,15 @@ MmMakePagesResident( PMEMORY_AREA MemoryArea; PMM_SECTION_SEGMENT Segment; LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); - MmLockAddressSpace(&Process->Vm); + MmLockAddressSpace(AddressSpace); - MemoryArea = MmLocateMemoryAreaByAddress(&Process->Vm, Address); + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); if (MemoryArea == NULL) { - MmUnlockAddressSpace(&Process->Vm); - return FALSE; + MmUnlockAddressSpace(AddressSpace); + return STATUS_NOT_MAPPED_VIEW; } /* Only supported in old Mm for now */ @@ -5160,6 +4453,8 @@ MmMakePagesResident( RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + MemoryArea->SectionData.ViewOffset.QuadPart; + DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart); + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) { ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); @@ -5168,9 +4463,9 @@ MmMakePagesResident( while (MM_IS_WAIT_PTE(Entry)) { MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); MiWaitForPageEvent(NULL, NULL); - MmLockAddressSpace(&Process->Vm); + MmLockAddressSpace(AddressSpace); MmLockSectionSegment(Segment); Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); } @@ -5189,7 +4484,7 @@ MmMakePagesResident( */ MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); /* FIXME: Read the whole range at once instead of one page at a time */ Status = MiReadPage(MemoryArea, SegmentOffset.QuadPart, &Page); @@ -5203,16 +4498,279 @@ MmMakePagesResident( return Status; } - MmLockAddressSpace(&Process->Vm); + MmLockAddressSpace(AddressSpace); MmLockSectionSegment(Segment); - MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 1)); + + /* We set it with 0 ref count, nobody maps this page yet. */ + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, MAKE_SSE(Page << PAGE_SHIFT, 0)); + MiSetPageEvent(Process, Address); } SegmentOffset.QuadPart += PAGE_SIZE; } MmUnlockSectionSegment(Segment); - MmUnlockAddressSpace(&Process->Vm); + MmUnlockAddressSpace(AddressSpace); + return STATUS_SUCCESS; +} + +NTSTATUS +NTAPI +MmRosFlushVirtualMemory( + _In_ PEPROCESS Process, + _Inout_ PVOID* Address, + _Inout_ PSIZE_T Length, + _Out_ PIO_STATUS_BLOCK Iosb) +{ + PMEMORY_AREA MemoryArea; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); + PVOID CurrentAddress; + + PAGED_CODE(); + + MmLockAddressSpace(AddressSpace); + + DPRINT("Flushing Process %p at %p --> 0x%x", Process, *Address, *Length); + + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, *Address); + if ((MemoryArea == NULL) || (MemoryArea->Type != MEMORY_AREA_SECTION_VIEW) || + (MemoryArea->VadNode.u.VadFlags.VadType == VadImageMap)) + { + MmUnlockAddressSpace(AddressSpace); + return STATUS_NOT_MAPPED_VIEW; + } + + Segment = MemoryArea->SectionData.Segment; + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(*Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)*Address + *Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + + CurrentAddress = *Address; + + MmUnlockAddressSpace(AddressSpace); + + MmLockSectionSegment(Segment); + + Iosb->Information = 0; + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + MiWaitForPageEvent(NULL, NULL); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + } + + /* We are called from Cc, this can't be backed by the page files */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* At this point, there may be a valid page there */ + if (Entry != 0) + { + /* This will write the page to disk, if needed */ + MmCheckDirtySegment(Segment, &SegmentOffset, MmIsDirtyPage(Process, CurrentAddress), FALSE); + Iosb->Information += PAGE_SIZE; + } + SegmentOffset.QuadPart += PAGE_SIZE; + CurrentAddress = (PVOID)((ULONG_PTR)CurrentAddress + PAGE_SIZE); + } + + MmUnlockSectionSegment(Segment); + + return STATUS_SUCCESS; +} + +_Requires_exclusive_lock_held_(Segment->Lock) +BOOLEAN +NTAPI +MmCheckDirtySegment( + PMM_SECTION_SEGMENT Segment, + PLARGE_INTEGER Offset, + BOOLEAN ForceDirty, + BOOLEAN PageOut) +{ + ULONG_PTR Entry; + NTSTATUS Status; + PFN_NUMBER Page; + + ASSERT(Segment->Locked); + + DPRINT("Checking segment for file %wZ at offset 0x%I64X.\n", &Segment->FileObject->FileName, Offset->QuadPart); + + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + if (Entry == 0) + return FALSE; + + Page = PFN_FROM_SSE(Entry); + if ((IS_DIRTY_SSE(Entry)) || ForceDirty) + { + BOOLEAN DirtyAgain; + + /* We got a dirty entry. Is this segment copy on write */ + ASSERT(!Segment->WriteCopy); + ASSERT(Segment->SegFlags & MM_DATAFILE_SEGMENT); + + /* Insert the cleaned entry back. Keep one ref to the page so nobody pages it out again behind us */ + MmSetPageEntrySectionSegment(Segment, Offset, + MAKE_SSE(Page << PAGE_SHIFT, SHARE_COUNT_FROM_SSE(Entry) + 1)); + + /* Tell the other users that we are clean again */ + MmSetCleanAllRmaps(Page); + + MmUnlockSectionSegment(Segment); + + /* Go ahead and write the page */ + Status = MiWritePage(Segment, Offset->QuadPart, Page); + + MmLockSectionSegment(Segment); + + /* Get the entry again */ + Entry = MmGetPageEntrySectionSegment(Segment, Offset); + ASSERT(PFN_FROM_SSE(Entry) == Page); + + if (!NT_SUCCESS(Status)) + { + /* Damn, this failed. Consider this page as still dirty */ + DPRINT1("MiWritePage FAILED: Status 0x%08x!\n", Status); + DirtyAgain = TRUE; + } + else + { + /* Check if someone dirtified this page while we were not looking */ + DirtyAgain = IS_DIRTY_SSE(Entry) || MmIsDirtyPageRmap(Page); + } + + /* Drop the reference we got */ + Entry = MAKE_SSE(Page << PAGE_SHIFT, SHARE_COUNT_FROM_SSE(Entry) - 1); + if (DirtyAgain) Entry = DIRTY_SSE(Entry); + MmSetPageEntrySectionSegment(Segment, Offset, Entry); + } + + /* Were this page hanging there just for the sake of being present ? */ + if (!IS_DIRTY_SSE(Entry) && (SHARE_COUNT_FROM_SSE(Entry) == 0) && PageOut) + { + /* Yes. Release it */ + MmSetPageEntrySectionSegment(Segment, Offset, 0); + MmReleasePageMemoryConsumer(MC_USER, Page); + /* Tell the caller we released the page */ + return TRUE; + } + + return FALSE; +} + +NTSTATUS +NTAPI +MmMakePagesDirty( + _In_ PEPROCESS Process, + _In_ PVOID Address, + _In_ ULONG Length) +{ + PMEMORY_AREA MemoryArea; + PMM_SECTION_SEGMENT Segment; + LARGE_INTEGER SegmentOffset, RangeEnd; + PMMSUPPORT AddressSpace = Process ? &Process->Vm : MmGetKernelAddressSpace(); + + MmLockAddressSpace(AddressSpace); + + MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, Address); + if (MemoryArea == NULL) + { + MmUnlockAddressSpace(AddressSpace); + return STATUS_NOT_MAPPED_VIEW; + } + + /* Only supported in old Mm for now */ + ASSERT(MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + /* For file mappings */ + ASSERT(MemoryArea->VadNode.u.VadFlags.VadType != VadImageMap); + + Segment = MemoryArea->SectionData.Segment; + MmLockSectionSegment(Segment); + + SegmentOffset.QuadPart = PAGE_ROUND_DOWN(Address) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + RangeEnd.QuadPart = PAGE_ROUND_UP((ULONG_PTR)Address + Length) - MA_GetStartingAddress(MemoryArea) + + MemoryArea->SectionData.ViewOffset.QuadPart; + + DPRINT("MmMakePagesResident: Segment %p, 0x%I64x -> 0x%I64x\n", Segment, SegmentOffset.QuadPart, RangeEnd.QuadPart); + + while (SegmentOffset.QuadPart < RangeEnd.QuadPart) + { + ULONG_PTR Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + + /* Let any pending read proceed */ + while (MM_IS_WAIT_PTE(Entry)) + { + MmUnlockSectionSegment(Segment); + MmUnlockAddressSpace(AddressSpace); + MiWaitForPageEvent(NULL, NULL); + MmLockAddressSpace(AddressSpace); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &SegmentOffset); + } + + /* We are called from Cc, this can't be backed by the page files */ + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + + /* If there is no page there, there is nothing to make dirty */ + if (Entry != 0) + { + /* Dirtify the entry */ + MmSetPageEntrySectionSegment(Segment, &SegmentOffset, DIRTY_SSE(Entry)); + } + + SegmentOffset.QuadPart += PAGE_SIZE; + } + + MmUnlockSectionSegment(Segment); + + MmUnlockAddressSpace(AddressSpace); + return STATUS_SUCCESS; +} + +NTSTATUS +NTAPI +MmExtendSection( + _In_ PVOID _Section, + _Inout_ PLARGE_INTEGER NewSize) +{ + PSECTION Section = _Section; + + /* It makes no sense to extend an image mapping */ + if (Section->u.Flags.Image) + return STATUS_SECTION_NOT_EXTENDED; + + /* Nor is it possible to extend a page file mapping */ + if (!Section->u.Flags.File) + return STATUS_SECTION_NOT_EXTENDED; + + if (!MiIsRosSectionObject(Section)) + return STATUS_NOT_IMPLEMENTED; + + /* We just extend the sizes. Shrinking is a no-op ? */ + if (NewSize->QuadPart > Section->SizeOfSection.QuadPart) + { + PMM_SECTION_SEGMENT Segment = (PMM_SECTION_SEGMENT)Section->Segment; + Section->SizeOfSection = *NewSize; + + MmLockSectionSegment(Segment); + if (Segment->RawLength.QuadPart < NewSize->QuadPart) + { + Segment->RawLength = *NewSize; + Segment->Length.QuadPart = (NewSize->QuadPart + PAGE_SIZE - 1) & ~((LONGLONG)PAGE_SIZE); + } + MmUnlockSectionSegment(Segment); + } + return STATUS_SUCCESS; } diff --git a/ntoskrnl/po/power.c b/ntoskrnl/po/power.c index 9490da817ce..7d2f4d7114a 100644 --- a/ntoskrnl/po/power.c +++ b/ntoskrnl/po/power.c @@ -4,7 +4,7 @@ * FILE: ntoskrnl/po/power.c * PURPOSE: Power Manager * PROGRAMMERS: Casper S. Hornstrup (chorns@users.sourceforge.net) - * Hervé Poussineau (hpoussin@reactos.com) + * Herv� Poussineau (hpoussin@reactos.com) */ /* INCLUDES ******************************************************************/ @@ -1075,7 +1075,7 @@ NtSetSystemPowerState(IN POWER_ACTION SystemAction, #ifndef NEWCC /* Flush dirty cache pages */ /* XXX: Is that still mandatory? As now we'll wait on lazy writer to complete? */ - CcRosFlushDirtyPages(-1, &Dummy, FALSE, FALSE); //HACK: We really should wait here! + CcRosFlushDirtyPages(-1, &Dummy, TRUE, FALSE); //HACK: We really should wait here! #else Dummy = 0; #endif