- Start of heavy work on iofunc.c:

- Created our new friends: IopCleanupFailedIrp, IopAbortInterruptedIrp, IopPerformSynchronousRequest, IopLockFileObject, IopUnlockFileObject, IopQueueIrpToThread, IopUpdateOperationCount.
  - What does this mean: We actually queue IRPs to their thread! We actually respect I/O transfers being interrupted/alerted! We actually respect I/O operation counts! We actually LOCK FILE OBJECTS instead of randomly using them! We now support Deferred (read: MUCH faster) I/O completion.
- First function blessed: IopDeviceFsIoControl.
- Also simplified access rights check and fixedup some formatting.

svn path=/trunk/; revision=22769

reactos/ntoskrnl/include/internal/io.h

@@ -5,7 +5,6 @@
 * PURPOSE:         Internal header for the I/O Manager
 * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
 */
-//#include "io_x.h"
 #include "ntdddisk.h"
 
 //
@@ -193,6 +192,16 @@ typedef enum _IOP_DEVICE_LIST_OPERATION
     IopAdd
 } IOP_DEVICE_LIST_OPERATION, *PIOP_DEVICE_LIST_OPERATION;
 
+//
+// Transfer statistics
+//
+typedef enum _IOP_TRANSFER_TYPE
+{
+    IopReadTransfer,
+    IopWriteTransfer,
+    IopOtherTransfer
+} IOP_TRANSFER_TYPE, *PIOP_TRANSFER_TYPE;
+
 //
 // Special version of the IRP Overlay used to optimize I/O completion
 // by not using up a separate structure.
@@ -673,6 +682,19 @@ IoInitCancelHandling(
     VOID
 );
 
+//
+// I/O Completion
+//
+VOID
+NTAPI
+IopCompleteRequest(
+    IN PKAPC Apc,
+    IN PKNORMAL_ROUTINE* NormalRoutine,
+    IN PVOID* NormalContext,
+    IN PVOID* SystemArgument1,
+    IN PVOID* SystemArgument2
+);
+
 //
 // Error Logging Routines
 //
@@ -949,3 +971,8 @@ extern POBJECT_TYPE IoCompletionType;
 extern PDEVICE_NODE IopRootDeviceNode;
 extern ULONG IopTraceLevel;
 extern NPAGED_LOOKASIDE_LIST IopMdlLookasideList;
+
+//
+// Inlined Functions
+//
+#include "io_x.h"

reactos/ntoskrnl/include/internal/io_x.h

@@ -0,0 +1,78 @@
+/*
+* PROJECT:         ReactOS Kernel
+* LICENSE:         GPL - See COPYING in the top level directory
+* FILE:            ntoskrnl/include/io_x.h
+* PURPOSE:         Internal Inlined Functions for the I/O Manager
+* PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
+*/
+
+VOID
+static __inline
+IopLockFileObject(IN PFILE_OBJECT FileObject)
+{
+    /* Lock the FO and check for contention */
+    if (InterlockedExchange(&FileObject->Busy, TRUE))
+    {
+        /* FIXME: Implement contention case */
+        KEBUGCHECK(0);
+    }
+}
+
+VOID
+static __inline
+IopUnlockFileObject(IN PFILE_OBJECT FileObject)
+{
+    /* Unlock the FO and wake any waiters up */
+    InterlockedExchange(&FileObject->Busy, FALSE);
+    if (FileObject->Waiters) KeSetEvent(&FileObject->Lock, 0, FALSE);
+}
+
+VOID
+static __inline
+IopQueueIrpToThread(IN PIRP Irp)
+{
+    KIRQL OldIrql;
+
+    /* Raise to APC Level */
+    KeRaiseIrql(APC_LEVEL, &OldIrql);
+
+    /* Insert it into the list */
+    InsertHeadList(&Irp->Tail.Overlay.Thread->IrpList, &Irp->ThreadListEntry);
+
+    /* Lower irql */
+    KeLowerIrql(OldIrql);
+}
+
+VOID
+static __inline
+IopUpdateOperationCount(IN IOP_TRANSFER_TYPE Type)
+{
+    PLARGE_INTEGER CountToChange;
+
+    /* Make sure I/O operations are being counted */
+    if (IoCountOperations)
+    {
+        if (Type == IopReadTransfer)
+        {
+            /* Increase read count */
+            IoReadOperationCount++;
+            CountToChange = &PsGetCurrentProcess()->ReadOperationCount;
+        }
+        else if (Type == IopWriteTransfer)
+        {
+            /* Increase write count */
+            IoWriteOperationCount++;
+            CountToChange = &PsGetCurrentProcess()->ReadOperationCount;
+        }
+        else
+        {
+            /* Increase other count */
+            IoOtherOperationCount++;
+            CountToChange = &PsGetCurrentProcess()->ReadOperationCount;
+        }
+
+        /* Increase the process-wide count */
+        ExInterlockedAddLargeStatistic(CountToChange, 1);
+    }
+}
+

reactos/ntoskrnl/include/internal/ob.h

@@ -5,8 +5,6 @@
 * PURPOSE:         Internal header for the Object Manager
 * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
 */
-#include "ob_x.h"
-
 //
 // Define this if you want debugging support
 //
@@ -323,3 +321,9 @@ extern PHANDLE_TABLE ObpKernelHandleTable;
 extern WORK_QUEUE_ITEM ObpReaperWorkItem;
 extern volatile PVOID ObpReaperList;
 extern NPAGED_LOOKASIDE_LIST ObpNmLookasideList, ObpCiLookasideList;
+extern BOOLEAN IoCountOperations;
+
+//
+// Inlined Functions
+//
+#include "ob_x.h"

reactos/ntoskrnl/io/iomgr/iofunc.c

@@ -23,14 +23,145 @@
 
 /* PRIVATE FUNCTIONS *********************************************************/
 
+/* DON'T inline this: it's a failure case */
 NTSTATUS
 NTAPI
-IopQueryDirectoryFileCompletion(IN PDEVICE_OBJECT DeviceObject,
+IopCleanupFailedIrp(IN PFILE_OBJECT FileObject,
-                                IN PIRP Irp,
+                    IN PKEVENT EventObject)
-                                IN PVOID Context)
 {
-    ExFreePool(Context);
+    PAGED_CODE();
-    return STATUS_SUCCESS;
+
+    /* Dereference the event */
+    if (EventObject) ObDereferenceObject(EventObject);
+
+    /* If this was a file opened for synch I/O, then unlock it */
+    if (FileObject->Flags & FO_SYNCHRONOUS_IO) IopUnlockFileObject(FileObject);
+
+    /* Now dereference it and return */
+    ObDereferenceObject(FileObject);
+    return STATUS_INSUFFICIENT_RESOURCES;
+}
+
+/* DON'T inline this: it's a failure case */
+VOID
+NTAPI
+IopAbortInterruptedIrp(IN PKEVENT EventObject,
+                       IN PIRP Irp)
+{
+    KIRQL OldIrql;
+    BOOLEAN CancelResult;
+    LARGE_INTEGER Wait;
+    PAGED_CODE();
+
+    /* Raise IRQL to APC */
+    KeRaiseIrql(APC_LEVEL, &OldIrql);
+
+    /* Check if nobody completed it yet */
+    if (!KeReadStateEvent(EventObject))
+    {
+        /* First, cancel it */
+        CancelResult = IoCancelIrp(Irp);
+        KeLowerIrql(OldIrql);
+
+        /* Check if we cancelled it */
+        if (CancelResult)
+        {
+            /* Wait for the IRP to be cancelled */
+            Wait.QuadPart = -100000;
+            while (!KeReadStateEvent(EventObject))
+            {
+                /* Delay indefintely */
+                KeDelayExecutionThread(KernelMode, FALSE, &Wait);
+            }
+        }
+        else
+        {
+            /* No cancellation done, so wait for the I/O system to kill it */
+            KeWaitForSingleObject(EventObject,
+                                  Executive,
+                                  KernelMode,
+                                  FALSE,
+                                  NULL);
+        }
+    }
+    else
+    {
+        /* We got preempted, so give up */
+        KeLowerIrql(OldIrql);
+    }
+}
+
+NTSTATUS
+NTAPI
+IopPerformSynchronousRequest(IN PDEVICE_OBJECT DeviceObject,
+                             IN PIRP Irp,
+                             IN PFILE_OBJECT FileObject,
+                             IN BOOLEAN Deferred,
+                             IN KPROCESSOR_MODE PreviousMode,
+                             IN BOOLEAN SynchIo,
+                             IN IOP_TRANSFER_TYPE TransferType)
+{
+    NTSTATUS Status;
+    PKNORMAL_ROUTINE NormalRoutine;
+    PVOID NormalContext;
+    KIRQL OldIrql;
+    PAGED_CODE();
+
+    /* Queue the IRP */
+    IopQueueIrpToThread(Irp);
+
+    /* Update operation counts */
+    IopUpdateOperationCount(TransferType);
+
+    /* Call the driver */
+    Status = IoCallDriver(DeviceObject, Irp);
+
+    /* Check if we're optimizing this case */
+    if (Deferred)
+    {
+        /* We are! Check if the IRP wasn't completed */
+        if (Status != STATUS_PENDING)
+        {
+            /* Complete it ourselves */
+            ASSERT(!Irp->PendingReturned);
+            KeRaiseIrql(APC_LEVEL, &OldIrql);
+            IopCompleteRequest(&Irp->Tail.Apc,
+                               &NormalRoutine,
+                               &NormalContext,
+                               (PVOID*)&FileObject,
+                               &NormalContext);
+            KeLowerIrql(OldIrql);
+        }
+    }
+
+    /* Check if this was synch I/O */
+    if (SynchIo)
+    {
+        /* Make sure the IRP was completed, but returned pending */
+        if (Status == STATUS_PENDING)
+        {
+            /* Wait for the IRP */
+            Status = KeWaitForSingleObject(&FileObject->Event,
+                                           Executive,
+                                           PreviousMode,
+                                           (FileObject->Flags & FO_ALERTABLE_IO),
+                                           NULL);
+            if ((Status == STATUS_ALERTED) || (Status == STATUS_USER_APC))
+            {
+                /* Abort the request */
+                IopAbortInterruptedIrp(&FileObject->Event, Irp);
+            }
+
+            /* Set the final status */
+            Status = FileObject->FinalStatus;
+        }
+
+        /* Release the file lock */
+        IopUnlockFileObject(FileObject);
+    }
+
+    /* Return status */
+    return Status;
 }
 
 NTSTATUS
@@ -45,7 +176,7 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle,
                      IN ULONG InputBufferLength OPTIONAL,
                      OUT PVOID OutputBuffer,
                      IN ULONG OutputBufferLength OPTIONAL,
-                     BOOLEAN IsDevIoCtl)
+                     IN BOOLEAN IsDevIoCtl)
 {
     NTSTATUS Status = STATUS_SUCCESS;
     PFILE_OBJECT FileObject;
@@ -53,54 +184,65 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle,
     PIRP Irp;
     PIO_STACK_LOCATION StackPtr;
     PKEVENT EventObject = NULL;
-    BOOLEAN LocalEvent = FALSE;
+    BOOLEAN LockedForSynch = FALSE;
     ULONG AccessType;
     OBJECT_HANDLE_INFORMATION HandleInformation;
+    ACCESS_MASK DesiredAccess;
     KPROCESSOR_MODE PreviousMode = ExGetPreviousMode();
 
+    /* Get the access type */
     AccessType = IO_METHOD_FROM_CTL_CODE(IoControlCode);
 
+    /* Check if we came from user mode */
     if (PreviousMode != KernelMode)
     {
         _SEH_TRY
         {
+            /* Probe the status block */
             ProbeForWrite(IoStatusBlock,
                           sizeof(IO_STATUS_BLOCK),
                           sizeof(ULONG));
 
-            /* probe the input and output buffers if needed */
+            /* Check if this is buffered I/O */
             if (AccessType == METHOD_BUFFERED)
             {
-                if (OutputBuffer != NULL)
+                /* Check if we have an output buffer */
+                if (OutputBuffer)
                 {
+                    /* Probe the output buffer */
                     ProbeForWrite(OutputBuffer, OutputBufferLength, 1);
                 }
                 else
                 {
-                    /* make sure the caller can't fake this as we depend on this */
+                    /* Make sure the caller can't fake this as we depend on this */
                     OutputBufferLength = 0;
                 }
             }
 
+            /* Check if we we have an input buffer I/O */
             if (AccessType != METHOD_NEITHER)
             {
-                if (InputBuffer != NULL)
+                /* Check if we have an input buffer */
+                if (InputBuffer)
                 {
+                    /* Probe the input buffer */
                     ProbeForRead(InputBuffer, InputBufferLength, 1);
                 }
                 else
                 {
-                    /* make sure the caller can't fake this as we depend on this */
+                    /* Make sure the caller can't fake this as we depend on this */
                     InputBufferLength = 0;
                 }
             }
         }
         _SEH_HANDLE
         {
+            /* Get the exception code */
             Status = _SEH_GetExceptionCode();
         }
         _SEH_END;
 
+        /* Fail if we got an access violation */
         if (!NT_SUCCESS(Status)) return Status;
     }
 
@@ -113,15 +255,18 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle,
                                        &HandleInformation);
     if (!NT_SUCCESS(Status)) return Status;
 
-    /* Check for sufficient access rights */
+    /* Check if we from user mode */
     if (PreviousMode != KernelMode)
     {
-        ACCESS_MASK DesiredAccess = (ACCESS_MASK)((IoControlCode >> 14) & 3);
+        /* Get the access mask */
-        if (DesiredAccess != FILE_ANY_ACCESS &&
+        DesiredAccess = (ACCESS_MASK)((IoControlCode >> 14) & 3);
-            !RtlAreAllAccessesGranted(HandleInformation.GrantedAccess,
+
-                                      (ACCESS_MASK)((IoControlCode >> 14) & 3)))
+        /* Check if we can open it */
+        if ((DesiredAccess != FILE_ANY_ACCESS) &&
+            (HandleInformation.GrantedAccess & DesiredAccess) != DesiredAccess)
         {
-            ObDereferenceObject (FileObject);
+            /* Dereference the file object and fail */
+            ObDereferenceObject(FileObject);
             return STATUS_ACCESS_DENIED;
         }
     }
@@ -138,7 +283,8 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle,
                                            NULL);
         if (!NT_SUCCESS(Status))
         {
-            ObDereferenceObject (FileObject);
+            /* Dereference the file object and fail */
+            ObDereferenceObject(FileObject);
             return Status;
         }
 
@@ -146,27 +292,30 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle,
         KeClearEvent(EventObject);
     }
 
+    /* Check if this is a file that was opened for Synch I/O */
+    if (FileObject->Flags & FO_SYNCHRONOUS_IO)
+    {
+        /* Lock it */
+        IopLockFileObject(FileObject);
+
+        /* Remember to unlock later */
+        LockedForSynch = TRUE;
+    }
+
     /* Check if this is a direct open or not */
     if (FileObject->Flags & FO_DIRECT_DEVICE_OPEN)
     {
+        /* It's a direct open, get the attached device */
         DeviceObject = IoGetAttachedDevice(FileObject->DeviceObject);
     }
     else
     {
+        /* Otherwise get the related device */
         DeviceObject = IoGetRelatedDeviceObject(FileObject);
     }
 
-    /* Check if we should use Sync IO or not */
+    /* Clear the event */
-    if (FileObject->Flags & FO_SYNCHRONOUS_IO)
+    KeClearEvent(&FileObject->Event);
-    {
-        /* Use File Object event */
-        KeClearEvent(&FileObject->Event);
-    }
-    else
-    {
-        /* Use local event */
-        LocalEvent = TRUE;
-    }
 
     /* Build the IRP */
     Irp = IoBuildDeviceIoControlRequest(IoControlCode,
@@ -178,14 +327,10 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle,
                                         FALSE,
                                         EventObject,
                                         IoStatusBlock);
-    if (!Irp)
+    if (!Irp) return IopCleanupFailedIrp(FileObject, Event);
-    {
-        if (EventObject) ObDereferenceObject(EventObject);
-        ObDereferenceObject(FileObject);
-        return STATUS_INSUFFICIENT_RESOURCES;
-    }
 
     /* Set some extra settings */
+    Irp->Tail.Overlay.AuxiliaryBuffer = (PVOID) NULL;
     Irp->Tail.Overlay.OriginalFileObject = FileObject;
     Irp->RequestorMode = PreviousMode;
     Irp->Overlay.AsynchronousParameters.UserApcRoutine = UserApcRoutine;
@@ -196,23 +341,27 @@ IopDeviceFsIoControl(IN HANDLE DeviceHandle,
                               IRP_MJ_DEVICE_CONTROL :
                               IRP_MJ_FILE_SYSTEM_CONTROL;
 
-    /* Call the Driver */
+    /* Use deferred completion for FS I/O */
-    Status = IoCallDriver(DeviceObject, Irp);
+    Irp->Flags |= (!IsDevIoCtl) ? IRP_DEFER_IO_COMPLETION : 0;
-    if (Status == STATUS_PENDING)
-    {
-        if (!LocalEvent)
-        {
-            KeWaitForSingleObject(&FileObject->Event,
-                                  Executive,
-                                  PreviousMode,
-                                  FileObject->Flags & FO_ALERTABLE_IO,
-                                  NULL);
-            Status = FileObject->FinalStatus;
-        }
-    }
 
-    /* Return the Status */
+    /* Perform the call */
-    return Status;
+    return IopPerformSynchronousRequest(DeviceObject,
+                                        Irp,
+                                        FileObject,
+                                        !IsDevIoCtl,
+                                        PreviousMode,
+                                        LockedForSynch,
+                                        IopOtherTransfer);
+}
+
+NTSTATUS
+NTAPI
+IopQueryDirectoryFileCompletion(IN PDEVICE_OBJECT DeviceObject,
+                                IN PIRP Irp,
+                                IN PVOID Context)
+{
+    ExFreePool(Context);
+    return STATUS_SUCCESS;
 }
 
 /* PUBLIC FUNCTIONS **********************************************************/

reactos/ntoskrnl/io/iomgr/iomgr.c

@@ -20,6 +20,7 @@ ULONG IopTraceLevel = IO_IRP_DEBUG;
 POBJECT_TYPE IoDeviceObjectType = NULL;
 POBJECT_TYPE IoFileObjectType = NULL;
 extern POBJECT_TYPE IoControllerObjectType;
+BOOLEAN IoCountOperations;
 ULONG IoReadOperationCount = 0;
 LARGE_INTEGER IoReadTransferCount = {{0, 0}};
 ULONG IoWriteOperationCount = 0;

reactos/ntoskrnl/io/iomgr/irp.c

@@ -849,13 +849,13 @@ IoCancelIrp(IN PIRP Irp)
 {
     KIRQL OldIrql;
     PDRIVER_CANCEL CancelRoutine;
-
-    /* Acquire the cancel lock and cancel the IRP */
     IOTRACE(IO_IRP_DEBUG,
             "%s - Canceling IRP %p\n",
             __FUNCTION__,
             Irp);
+    ASSERT(Irp->Type == IO_TYPE_IRP);
 
+    /* Acquire the cancel lock and cancel the IRP */
     IoAcquireCancelSpinLock(&OldIrql);
     Irp->Cancel = TRUE;
 
@@ -892,10 +892,11 @@ VOID
 NTAPI
 IoCancelThreadIo(IN PETHREAD Thread)
 {
-    PIRP Irp;
     KIRQL OldIrql;
     ULONG Retries = 3000;
     LARGE_INTEGER Interval;
+    //PLIST_ENTRY ListHead, NextEntry;
+    //PIRP Irp;
     IOTRACE(IO_IRP_DEBUG,
             "%s - Canceling IRPs for Thread %p\n",
             __FUNCTION__,
@@ -905,11 +906,21 @@ IoCancelThreadIo(IN PETHREAD Thread)
     OldIrql = KfRaiseIrql(APC_LEVEL);
 
     /* Start by cancelling all the IRPs in the current thread queue. */
-    LIST_FOR_EACH(Irp, &Thread->IrpList, IRP, ThreadListEntry)
+#if 0
+    ListHead = &Thread->IrpList;
+    NextEntry = ListHead->Flink;
+    while (ListHead != NextEntry)
     {
+        /* Get the IRP */
+        Irp = CONTAINING_RECORD(NextEntry, IRP, ThreadListEntry);
+
         /* Cancel it */
         IoCancelIrp(Irp);
+
+        /* Move to the next entry */
+        NextEntry = NextEntry->Flink;
     }
+#endif
 
      /* Wait 100 milliseconds */
     Interval.QuadPart = -1000000;
@@ -1462,24 +1473,13 @@ VOID
 NTAPI
 IoQueueThreadIrp(IN PIRP Irp)
 {
-    KIRQL OldIrql;
     IOTRACE(IO_IRP_DEBUG,
             "%s - Queueing IRP %p\n",
             __FUNCTION__,
             Irp);
 
-    /* Raise to APC */
+    /* Use our inlined routine */
-    OldIrql = KfRaiseIrql(APC_LEVEL);
+    IopQueueIrpToThread(Irp);
-
-   /*
-    * Synchronous irp's are queued to requestor thread. If they are not
-    * completed when the thread exits, they are canceled (cleaned up).
-    * - Gunnar
-    */
-    InsertTailList(&Irp->Tail.Overlay.Thread->IrpList, &Irp->ThreadListEntry);
-
-    /* Lower back */
-    KfLowerIrql(OldIrql);
 }
 
 /*