From 13ef11ff23c1ad1c9a93c78822f03b375ba861ae Mon Sep 17 00:00:00 2001 From: Alex Ionescu Date: Sun, 2 Jul 2006 16:20:10 +0000 Subject: [PATCH] - Start of heavy work on iofunc.c: - Created our new friends: IopCleanupFailedIrp, IopAbortInterruptedIrp, IopPerformSynchronousRequest, IopLockFileObject, IopUnlockFileObject, IopQueueIrpToThread, IopUpdateOperationCount. - What does this mean: We actually queue IRPs to their thread! We actually respect I/O transfers being interrupted/alerted! We actually respect I/O operation counts! We actually LOCK FILE OBJECTS instead of randomly using them! We now support Deferred (read: MUCH faster) I/O completion. - First function blessed: IopDeviceFsIoControl. - Also simplified access rights check and fixedup some formatting. svn path=/trunk/; revision=22769 --- reactos/ntoskrnl/include/internal/io.h | 29 ++- reactos/ntoskrnl/include/internal/io_x.h | 78 +++++++ reactos/ntoskrnl/include/internal/ob.h | 8 +- reactos/ntoskrnl/io/iomgr/iofunc.c | 253 ++++++++++++++++++----- reactos/ntoskrnl/io/iomgr/iomgr.c | 1 + reactos/ntoskrnl/io/iomgr/irp.c | 34 +-- 6 files changed, 331 insertions(+), 72 deletions(-) create mode 100644 reactos/ntoskrnl/include/internal/io_x.h diff --git a/reactos/ntoskrnl/include/internal/io.h b/reactos/ntoskrnl/include/internal/io.h index 34ddaacf65b..2e386e8558b 100644 --- a/reactos/ntoskrnl/include/internal/io.h +++ b/reactos/ntoskrnl/include/internal/io.h @@ -5,7 +5,6 @@ * PURPOSE: Internal header for the I/O Manager * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) */ -//#include "io_x.h" #include "ntdddisk.h" // @@ -193,6 +192,16 @@ typedef enum _IOP_DEVICE_LIST_OPERATION IopAdd } IOP_DEVICE_LIST_OPERATION, *PIOP_DEVICE_LIST_OPERATION; +// +// Transfer statistics +// +typedef enum _IOP_TRANSFER_TYPE +{ + IopReadTransfer, + IopWriteTransfer, + IopOtherTransfer +} IOP_TRANSFER_TYPE, *PIOP_TRANSFER_TYPE; + // // Special version of the IRP Overlay used to optimize I/O completion // by not using up a separate structure. @@ -673,6 +682,19 @@ IoInitCancelHandling( VOID ); +// +// I/O Completion +// +VOID +NTAPI +IopCompleteRequest( + IN PKAPC Apc, + IN PKNORMAL_ROUTINE* NormalRoutine, + IN PVOID* NormalContext, + IN PVOID* SystemArgument1, + IN PVOID* SystemArgument2 +); + // // Error Logging Routines // @@ -949,3 +971,8 @@ extern POBJECT_TYPE IoCompletionType; extern PDEVICE_NODE IopRootDeviceNode; extern ULONG IopTraceLevel; extern NPAGED_LOOKASIDE_LIST IopMdlLookasideList; + +// +// Inlined Functions +// +#include "io_x.h" diff --git a/reactos/ntoskrnl/include/internal/io_x.h b/reactos/ntoskrnl/include/internal/io_x.h new file mode 100644 index 00000000000..40f4c49b93c --- /dev/null +++ b/reactos/ntoskrnl/include/internal/io_x.h @@ -0,0 +1,78 @@ +/* +* PROJECT: ReactOS Kernel +* LICENSE: GPL - See COPYING in the top level directory +* FILE: ntoskrnl/include/io_x.h +* PURPOSE: Internal Inlined Functions for the I/O Manager +* PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) +*/ + +VOID +static __inline +IopLockFileObject(IN PFILE_OBJECT FileObject) +{ + /* Lock the FO and check for contention */ + if (InterlockedExchange(&FileObject->Busy, TRUE)) + { + /* FIXME: Implement contention case */ + KEBUGCHECK(0); + } +} + +VOID +static __inline +IopUnlockFileObject(IN PFILE_OBJECT FileObject) +{ + /* Unlock the FO and wake any waiters up */ + InterlockedExchange(&FileObject->Busy, FALSE); + if (FileObject->Waiters) KeSetEvent(&FileObject->Lock, 0, FALSE); +} + +VOID +static __inline +IopQueueIrpToThread(IN PIRP Irp) +{ + KIRQL OldIrql; + + /* Raise to APC Level */ + KeRaiseIrql(APC_LEVEL, &OldIrql); + + /* Insert it into the list */ + InsertHeadList(&Irp->Tail.Overlay.Thread->IrpList, &Irp->ThreadListEntry); + + /* Lower irql */ + KeLowerIrql(OldIrql); +} + +VOID +static __inline +IopUpdateOperationCount(IN IOP_TRANSFER_TYPE Type) +{ + PLARGE_INTEGER CountToChange; + + /* Make sure I/O operations are being counted */ + if (IoCountOperations) + { + if (Type == IopReadTransfer) + { + /* Increase read count */ + IoReadOperationCount++; + CountToChange = &PsGetCurrentProcess()->ReadOperationCount; + } + else if (Type == IopWriteTransfer) + { + /* Increase write count */ + IoWriteOperationCount++; + CountToChange = &PsGetCurrentProcess()->ReadOperationCount; + } + else + { + /* Increase other count */ + IoOtherOperationCount++; + CountToChange = &PsGetCurrentProcess()->ReadOperationCount; + } + + /* Increase the process-wide count */ + ExInterlockedAddLargeStatistic(CountToChange, 1); + } +} + diff --git a/reactos/ntoskrnl/include/internal/ob.h b/reactos/ntoskrnl/include/internal/ob.h index 65f97ca293e..275d14f6560 100644 --- a/reactos/ntoskrnl/include/internal/ob.h +++ b/reactos/ntoskrnl/include/internal/ob.h @@ -5,8 +5,6 @@ * PURPOSE: Internal header for the Object Manager * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) */ -#include "ob_x.h" - // // Define this if you want debugging support // @@ -323,3 +321,9 @@ extern PHANDLE_TABLE ObpKernelHandleTable; extern WORK_QUEUE_ITEM ObpReaperWorkItem; extern volatile PVOID ObpReaperList; extern NPAGED_LOOKASIDE_LIST ObpNmLookasideList, ObpCiLookasideList; +extern BOOLEAN IoCountOperations; + +// +// Inlined Functions +// +#include "ob_x.h" diff --git a/reactos/ntoskrnl/io/iomgr/iofunc.c b/reactos/ntoskrnl/io/iomgr/iofunc.c index 94576ddb66d..fb409b6d017 100644 --- a/reactos/ntoskrnl/io/iomgr/iofunc.c +++ b/reactos/ntoskrnl/io/iomgr/iofunc.c @@ -23,14 +23,145 @@ /* PRIVATE FUNCTIONS *********************************************************/ +/* DON'T inline this: it's a failure case */ NTSTATUS NTAPI -IopQueryDirectoryFileCompletion(IN PDEVICE_OBJECT DeviceObject, - IN PIRP Irp, - IN PVOID Context) +IopCleanupFailedIrp(IN PFILE_OBJECT FileObject, + IN PKEVENT EventObject) { - ExFreePool(Context); - return STATUS_SUCCESS; + PAGED_CODE(); + + /* Dereference the event */ + if (EventObject) ObDereferenceObject(EventObject); + + /* If this was a file opened for synch I/O, then unlock it */ + if (FileObject->Flags & FO_SYNCHRONOUS_IO) IopUnlockFileObject(FileObject); + + /* Now dereference it and return */ + ObDereferenceObject(FileObject); + return STATUS_INSUFFICIENT_RESOURCES; +} + +/* DON'T inline this: it's a failure case */ +VOID +NTAPI +IopAbortInterruptedIrp(IN PKEVENT EventObject, + IN PIRP Irp) +{ + KIRQL OldIrql; + BOOLEAN CancelResult; + LARGE_INTEGER Wait; + PAGED_CODE(); + + /* Raise IRQL to APC */ + KeRaiseIrql(APC_LEVEL, &OldIrql); + + /* Check if nobody completed it yet */ + if (!KeReadStateEvent(EventObject)) + { + /* First, cancel it */ + CancelResult = IoCancelIrp(Irp); + KeLowerIrql(OldIrql); + + /* Check if we cancelled it */ + if (CancelResult) + { + /* Wait for the IRP to be cancelled */ + Wait.QuadPart = -100000; + while (!KeReadStateEvent(EventObject)) + { + /* Delay indefintely */ + KeDelayExecutionThread(KernelMode, FALSE, &Wait); + } + } + else + { + /* No cancellation done, so wait for the I/O system to kill it */ + KeWaitForSingleObject(EventObject, + Executive, + KernelMode, + FALSE, + NULL); + } + } + else + { + /* We got preempted, so give up */ + KeLowerIrql(OldIrql); + } +} + +NTSTATUS +NTAPI +IopPerformSynchronousRequest(IN PDEVICE_OBJECT DeviceObject, + IN PIRP Irp, + IN PFILE_OBJECT FileObject, + IN BOOLEAN Deferred, + IN KPROCESSOR_MODE PreviousMode, + IN BOOLEAN SynchIo, + IN IOP_TRANSFER_TYPE TransferType) +{ + NTSTATUS Status; + PKNORMAL_ROUTINE NormalRoutine; + PVOID NormalContext; + KIRQL OldIrql; + PAGED_CODE(); + + /* Queue the IRP */ + IopQueueIrpToThread(Irp); + + /* Update operation counts */ + IopUpdateOperationCount(TransferType); + + /* Call the driver */ + Status = IoCallDriver(DeviceObject, Irp); + + /* Check if we're optimizing this case */ + if (Deferred) + { + /* We are! Check if the IRP wasn't completed */ + if (Status != STATUS_PENDING) + { + /* Complete it ourselves */ + ASSERT(!Irp->PendingReturned); + KeRaiseIrql(APC_LEVEL, &OldIrql); + IopCompleteRequest(&Irp->Tail.Apc, + &NormalRoutine, + &NormalContext, + (PVOID*)&FileObject, + &NormalContext); + KeLowerIrql(OldIrql); + } + } + + /* Check if this was synch I/O */ + if (SynchIo) + { + /* Make sure the IRP was completed, but returned pending */ + if (Status == STATUS_PENDING) + { + /* Wait for the IRP */ + Status = KeWaitForSingleObject(&FileObject->Event, + Executive, + PreviousMode, + (FileObject->Flags & FO_ALERTABLE_IO), + NULL); + if ((Status == STATUS_ALERTED) || (Status == STATUS_USER_APC)) + { + /* Abort the request */ + IopAbortInterruptedIrp(&FileObject->Event, Irp); + } + + /* Set the final status */ + Status = FileObject->FinalStatus; + } + + /* Release the file lock */ + IopUnlockFileObject(FileObject); + } + + /* Return status */ + return Status; } NTSTATUS @@ -45,7 +176,7 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle, IN ULONG InputBufferLength OPTIONAL, OUT PVOID OutputBuffer, IN ULONG OutputBufferLength OPTIONAL, - BOOLEAN IsDevIoCtl) + IN BOOLEAN IsDevIoCtl) { NTSTATUS Status = STATUS_SUCCESS; PFILE_OBJECT FileObject; @@ -53,54 +184,65 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle, PIRP Irp; PIO_STACK_LOCATION StackPtr; PKEVENT EventObject = NULL; - BOOLEAN LocalEvent = FALSE; + BOOLEAN LockedForSynch = FALSE; ULONG AccessType; OBJECT_HANDLE_INFORMATION HandleInformation; + ACCESS_MASK DesiredAccess; KPROCESSOR_MODE PreviousMode = ExGetPreviousMode(); + /* Get the access type */ AccessType = IO_METHOD_FROM_CTL_CODE(IoControlCode); + /* Check if we came from user mode */ if (PreviousMode != KernelMode) { _SEH_TRY { + /* Probe the status block */ ProbeForWrite(IoStatusBlock, sizeof(IO_STATUS_BLOCK), sizeof(ULONG)); - /* probe the input and output buffers if needed */ + /* Check if this is buffered I/O */ if (AccessType == METHOD_BUFFERED) { - if (OutputBuffer != NULL) + /* Check if we have an output buffer */ + if (OutputBuffer) { + /* Probe the output buffer */ ProbeForWrite(OutputBuffer, OutputBufferLength, 1); } else { - /* make sure the caller can't fake this as we depend on this */ + /* Make sure the caller can't fake this as we depend on this */ OutputBufferLength = 0; } } + /* Check if we we have an input buffer I/O */ if (AccessType != METHOD_NEITHER) { - if (InputBuffer != NULL) + /* Check if we have an input buffer */ + if (InputBuffer) { + /* Probe the input buffer */ ProbeForRead(InputBuffer, InputBufferLength, 1); } else { - /* make sure the caller can't fake this as we depend on this */ + /* Make sure the caller can't fake this as we depend on this */ InputBufferLength = 0; } } } _SEH_HANDLE { + /* Get the exception code */ Status = _SEH_GetExceptionCode(); } _SEH_END; + /* Fail if we got an access violation */ if (!NT_SUCCESS(Status)) return Status; } @@ -113,15 +255,18 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle, &HandleInformation); if (!NT_SUCCESS(Status)) return Status; - /* Check for sufficient access rights */ + /* Check if we from user mode */ if (PreviousMode != KernelMode) { - ACCESS_MASK DesiredAccess = (ACCESS_MASK)((IoControlCode >> 14) & 3); - if (DesiredAccess != FILE_ANY_ACCESS && - !RtlAreAllAccessesGranted(HandleInformation.GrantedAccess, - (ACCESS_MASK)((IoControlCode >> 14) & 3))) + /* Get the access mask */ + DesiredAccess = (ACCESS_MASK)((IoControlCode >> 14) & 3); + + /* Check if we can open it */ + if ((DesiredAccess != FILE_ANY_ACCESS) && + (HandleInformation.GrantedAccess & DesiredAccess) != DesiredAccess) { - ObDereferenceObject (FileObject); + /* Dereference the file object and fail */ + ObDereferenceObject(FileObject); return STATUS_ACCESS_DENIED; } } @@ -138,7 +283,8 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle, NULL); if (!NT_SUCCESS(Status)) { - ObDereferenceObject (FileObject); + /* Dereference the file object and fail */ + ObDereferenceObject(FileObject); return Status; } @@ -146,27 +292,30 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle, KeClearEvent(EventObject); } + /* Check if this is a file that was opened for Synch I/O */ + if (FileObject->Flags & FO_SYNCHRONOUS_IO) + { + /* Lock it */ + IopLockFileObject(FileObject); + + /* Remember to unlock later */ + LockedForSynch = TRUE; + } + /* Check if this is a direct open or not */ if (FileObject->Flags & FO_DIRECT_DEVICE_OPEN) { + /* It's a direct open, get the attached device */ DeviceObject = IoGetAttachedDevice(FileObject->DeviceObject); } else { + /* Otherwise get the related device */ DeviceObject = IoGetRelatedDeviceObject(FileObject); } - /* Check if we should use Sync IO or not */ - if (FileObject->Flags & FO_SYNCHRONOUS_IO) - { - /* Use File Object event */ - KeClearEvent(&FileObject->Event); - } - else - { - /* Use local event */ - LocalEvent = TRUE; - } + /* Clear the event */ + KeClearEvent(&FileObject->Event); /* Build the IRP */ Irp = IoBuildDeviceIoControlRequest(IoControlCode, @@ -178,14 +327,10 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle, FALSE, EventObject, IoStatusBlock); - if (!Irp) - { - if (EventObject) ObDereferenceObject(EventObject); - ObDereferenceObject(FileObject); - return STATUS_INSUFFICIENT_RESOURCES; - } + if (!Irp) return IopCleanupFailedIrp(FileObject, Event); /* Set some extra settings */ + Irp->Tail.Overlay.AuxiliaryBuffer = (PVOID) NULL; Irp->Tail.Overlay.OriginalFileObject = FileObject; Irp->RequestorMode = PreviousMode; Irp->Overlay.AsynchronousParameters.UserApcRoutine = UserApcRoutine; @@ -196,23 +341,27 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle, IRP_MJ_DEVICE_CONTROL : IRP_MJ_FILE_SYSTEM_CONTROL; - /* Call the Driver */ - Status = IoCallDriver(DeviceObject, Irp); - if (Status == STATUS_PENDING) - { - if (!LocalEvent) - { - KeWaitForSingleObject(&FileObject->Event, - Executive, - PreviousMode, - FileObject->Flags & FO_ALERTABLE_IO, - NULL); - Status = FileObject->FinalStatus; - } - } + /* Use deferred completion for FS I/O */ + Irp->Flags |= (!IsDevIoCtl) ? IRP_DEFER_IO_COMPLETION : 0; - /* Return the Status */ - return Status; + /* Perform the call */ + return IopPerformSynchronousRequest(DeviceObject, + Irp, + FileObject, + !IsDevIoCtl, + PreviousMode, + LockedForSynch, + IopOtherTransfer); +} + +NTSTATUS +NTAPI +IopQueryDirectoryFileCompletion(IN PDEVICE_OBJECT DeviceObject, + IN PIRP Irp, + IN PVOID Context) +{ + ExFreePool(Context); + return STATUS_SUCCESS; } /* PUBLIC FUNCTIONS **********************************************************/ diff --git a/reactos/ntoskrnl/io/iomgr/iomgr.c b/reactos/ntoskrnl/io/iomgr/iomgr.c index 8686d1bcb70..3e768d8a3ce 100644 --- a/reactos/ntoskrnl/io/iomgr/iomgr.c +++ b/reactos/ntoskrnl/io/iomgr/iomgr.c @@ -20,6 +20,7 @@ ULONG IopTraceLevel = IO_IRP_DEBUG; POBJECT_TYPE IoDeviceObjectType = NULL; POBJECT_TYPE IoFileObjectType = NULL; extern POBJECT_TYPE IoControllerObjectType; +BOOLEAN IoCountOperations; ULONG IoReadOperationCount = 0; LARGE_INTEGER IoReadTransferCount = {{0, 0}}; ULONG IoWriteOperationCount = 0; diff --git a/reactos/ntoskrnl/io/iomgr/irp.c b/reactos/ntoskrnl/io/iomgr/irp.c index 4e10cb5d1cd..a77c9c6aebc 100644 --- a/reactos/ntoskrnl/io/iomgr/irp.c +++ b/reactos/ntoskrnl/io/iomgr/irp.c @@ -849,13 +849,13 @@ IoCancelIrp(IN PIRP Irp) { KIRQL OldIrql; PDRIVER_CANCEL CancelRoutine; - - /* Acquire the cancel lock and cancel the IRP */ IOTRACE(IO_IRP_DEBUG, "%s - Canceling IRP %p\n", __FUNCTION__, Irp); + ASSERT(Irp->Type == IO_TYPE_IRP); + /* Acquire the cancel lock and cancel the IRP */ IoAcquireCancelSpinLock(&OldIrql); Irp->Cancel = TRUE; @@ -892,10 +892,11 @@ VOID NTAPI IoCancelThreadIo(IN PETHREAD Thread) { - PIRP Irp; KIRQL OldIrql; ULONG Retries = 3000; LARGE_INTEGER Interval; + //PLIST_ENTRY ListHead, NextEntry; + //PIRP Irp; IOTRACE(IO_IRP_DEBUG, "%s - Canceling IRPs for Thread %p\n", __FUNCTION__, @@ -905,11 +906,21 @@ IoCancelThreadIo(IN PETHREAD Thread) OldIrql = KfRaiseIrql(APC_LEVEL); /* Start by cancelling all the IRPs in the current thread queue. */ - LIST_FOR_EACH(Irp, &Thread->IrpList, IRP, ThreadListEntry) +#if 0 + ListHead = &Thread->IrpList; + NextEntry = ListHead->Flink; + while (ListHead != NextEntry) { + /* Get the IRP */ + Irp = CONTAINING_RECORD(NextEntry, IRP, ThreadListEntry); + /* Cancel it */ IoCancelIrp(Irp); + + /* Move to the next entry */ + NextEntry = NextEntry->Flink; } +#endif /* Wait 100 milliseconds */ Interval.QuadPart = -1000000; @@ -1462,24 +1473,13 @@ VOID NTAPI IoQueueThreadIrp(IN PIRP Irp) { - KIRQL OldIrql; IOTRACE(IO_IRP_DEBUG, "%s - Queueing IRP %p\n", __FUNCTION__, Irp); - /* Raise to APC */ - OldIrql = KfRaiseIrql(APC_LEVEL); - - /* - * Synchronous irp's are queued to requestor thread. If they are not - * completed when the thread exits, they are canceled (cleaned up). - * - Gunnar - */ - InsertTailList(&Irp->Tail.Overlay.Thread->IrpList, &Irp->ThreadListEntry); - - /* Lower back */ - KfLowerIrql(OldIrql); + /* Use our inlined routine */ + IopQueueIrpToThread(Irp); } /*