Fix IRP structure (add missing member) and optimize io completion by allowing the io completion packet to be piggybacked on the IRP. Also make io completion use the cpu lookaside lists now as well as the paged pool as a last-chance attempt

svn path=/trunk/; revision=15256

reactos/include/ddk/iotypes.h

@@ -794,12 +794,17 @@ typedef struct _IRP
 	     };
 	     struct _ETHREAD* Thread;
 	     PCHAR AuxiliaryBuffer;
-	     LIST_ENTRY ListEntry;
+         struct {
-	     struct _IO_STACK_LOCATION* CurrentStackLocation;
+	         LIST_ENTRY ListEntry;
+             union {
+	             struct _IO_STACK_LOCATION* CurrentStackLocation;
+                 ULONG PacketType;
+             };
+         };
 	     PFILE_OBJECT OriginalFileObject;
 	  } Overlay;
 	KAPC Apc;
-	ULONG CompletionKey;
+	PVOID CompletionKey;
      } Tail;
 } IRP, *PIRP;
 

reactos/ntoskrnl/include/internal/io.h

@@ -43,13 +43,25 @@
 
 struct _DEVICE_OBJECT_POWER_EXTENSION;
 
-typedef struct _IO_COMPLETION_PACKET{
+/* This is like the IRP Overlay so we can optimize its insertion */
+typedef struct _IO_COMPLETION_PACKET
+{
+   struct {
+       LIST_ENTRY ListEntry;
+       union {
+           struct _IO_STACK_LOCATION *CurrentStackLocation;
+           ULONG PacketType;
+       };
+   };
    PVOID             Key;
    PVOID             Context;
    IO_STATUS_BLOCK   IoStatus;
-   LIST_ENTRY        ListEntry;
 } IO_COMPLETION_PACKET, *PIO_COMPLETION_PACKET;
 
+/* Packet Types */
+#define IrpCompletionPacket     0x1
+#define IrpMiniCompletionPacket 0x2
+
 typedef struct _DEVOBJ_EXTENSION {
    CSHORT Type;
    USHORT Size;

reactos/ntoskrnl/io/iocomp.c

@@ -35,6 +35,40 @@ static const INFORMATION_CLASS_INFO ExIoCompletionInfoClass[] = {
 
 /* FUNCTIONS *****************************************************************/
 
+VOID
+STDCALL
+IopFreeIoCompletionPacket(PIO_COMPLETION_PACKET Packet)
+{
+    PKPRCB Prcb = KeGetCurrentPrcb();
+    PNPAGED_LOOKASIDE_LIST List;
+    
+    /* Use the P List */
+    List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].P;
+    List->L.TotalFrees++;
+        
+    /* Check if the Free was within the Depth or not */
+    if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
+    {
+        /* Let the balancer know */
+        List->L.FreeMisses++;
+            
+        /* Use the L List */
+        List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].L;
+        List->L.TotalFrees++;
+
+        /* Check if the Free was within the Depth or not */
+        if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth)
+        {            
+            /* All lists failed, use the pool */
+            List->L.FreeMisses++;
+            ExFreePool(Packet);
+        }
+    }
+        
+    /* The free was within dhe Depth */
+    InterlockedPushEntrySList(&List->L.ListHead, (PSINGLE_LIST_ENTRY)Packet);
+}
+
 VOID
 STDCALL
 IopDeleteIoCompletion(PVOID ObjectBody)
@@ -42,6 +76,7 @@ IopDeleteIoCompletion(PVOID ObjectBody)
     PKQUEUE Queue = ObjectBody;
     PLIST_ENTRY FirstEntry;
     PLIST_ENTRY CurrentEntry;
+    PIRP Irp;
     PIO_COMPLETION_PACKET Packet;
 
     DPRINT("IopDeleteIoCompletion()\n");
@@ -61,8 +96,18 @@ IopDeleteIoCompletion(PVOID ObjectBody)
             /* Go to next Entry */
             CurrentEntry = CurrentEntry->Flink;
 
-            /* Free it */
+            /* Check if it's part of an IRP, or a separate packet */
-            ExFreeToNPagedLookasideList(&IoCompletionPacketLookaside, Packet);
+            if (Packet->PacketType == IrpCompletionPacket)
+            {
+                /* Get the IRP and free it */
+                Irp = CONTAINING_RECORD(Packet, IRP, Tail.Overlay.ListEntry);
+                IoFreeIrp(Irp);
+            }
+            else
+            {
+                /* Use common routine */
+                IopFreeIoCompletionPacket(Packet);
+            }
         } while (FirstEntry != CurrentEntry);
     }
 }
@@ -80,20 +125,58 @@ IoSetIoCompletion(IN PVOID IoCompletion,
                   IN BOOLEAN Quota)
 {
     PKQUEUE Queue = (PKQUEUE)IoCompletion;
+    PNPAGED_LOOKASIDE_LIST List;
+    PKPRCB Prcb = KeGetCurrentPrcb();
     PIO_COMPLETION_PACKET Packet;
 
-    /* Allocate the Packet */
+    /* Get the P List */
-    Packet = ExAllocateFromNPagedLookasideList(&IoCompletionPacketLookaside);
+    List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].P;
-    if (NULL == Packet) return STATUS_NO_MEMORY;
+    
+    /* Try to allocate the Packet */
+    List->L.TotalAllocates++;
+    Packet = (PVOID)InterlockedPopEntrySList(&List->L.ListHead);
+    
+    /* Check if that failed, use the L list if it did */
+    if (!Packet)
+    {
+        /* Let the balancer know */
+        List->L.AllocateMisses++;
+        
+        /* Get L List */
+        List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].L;
+    
+        /* Try to allocate the Packet */
+        List->L.TotalAllocates++;
+        Packet = (PVOID)InterlockedPopEntrySList(&List->L.ListHead);
+    }
+    
+    /* Still failed, use pool */
+    if (!Packet)
+    {
+        /* Let the balancer know */
+        List->L.AllocateMisses++;
+        
+        /* Allocate from Nonpaged Pool */
+        Packet = ExAllocatePoolWithTag(NonPagedPool, sizeof(*Packet), IOC_TAG);
+    }
+    
+    /* Make sure we have one by now... */
+    if (Packet)
+    {
+        /* Set up the Packet */
+        Packet->PacketType = IrpMiniCompletionPacket;
+        Packet->Key = KeyContext;
+        Packet->Context = ApcContext;
+        Packet->IoStatus.Status = IoStatus;
+        Packet->IoStatus.Information = IoStatusInformation;
 
-    /* Set up the Packet */
+        /* Insert the Queue */
-    Packet->Key = KeyContext;
+        KeInsertQueue(Queue, &Packet->ListEntry);
-    Packet->Context = ApcContext;
+    }
-    Packet->IoStatus.Status = IoStatus;
+    else
-    Packet->IoStatus.Information = IoStatusInformation;
+    {
-
+        return STATUS_INSUFFICIENT_RESOURCES;
-    /* Insert the Queue */
+    }
-    KeInsertQueue(Queue, &Packet->ListEntry);
 
     /* Return Success */
     return STATUS_SUCCESS;
@@ -410,10 +493,27 @@ NtRemoveIoCompletion(IN  HANDLE IoCompletionHandle,
 
             _SEH_TRY {
 
-                /* Return it */
+                /* Check if this is piggybacked on an IRP */
-                *CompletionKey = Packet->Key;
+                if (Packet->PacketType == IrpCompletionPacket)
-                *CompletionContext = Packet->Context;
+                {
-                *IoStatusBlock = Packet->IoStatus;
+                    /* Get the IRP */
+                    PIRP Irp = NULL;
+                    Irp = CONTAINING_RECORD(ListEntry, IRP, Tail.Overlay.ListEntry);
+                    
+                    /* Return values to user */
+                    *CompletionKey = Irp->Tail.CompletionKey;
+                    *CompletionContext = Irp->Overlay.AsynchronousParameters.UserApcContext;
+                    *IoStatusBlock = Packet->IoStatus;
+                    IoFreeIrp(Irp);
+                }
+                else
+                {
+                    /* This is a user-mode generated or API generated mini-packet */
+                    *CompletionKey = Packet->Key;
+                    *CompletionContext = Packet->Context;
+                    *IoStatusBlock = Packet->IoStatus;
+                    IopFreeIoCompletionPacket(Packet);
+                }
 
             } _SEH_HANDLE {
 

reactos/ntoskrnl/io/irp.c

@@ -187,12 +187,10 @@ IopCompleteRequest(PKAPC Apc,
         else if (FileObject && FileObject->CompletionContext)
         {
             /* Call the IO Completion Port if we have one, instead */
-            IoSetIoCompletion(FileObject->CompletionContext->Port,
+            Irp->Tail.CompletionKey = FileObject->CompletionContext->Key;
-                              FileObject->CompletionContext->Key,
+            Irp->Tail.Overlay.PacketType = IrpCompletionPacket;
-                              Irp->Overlay.AsynchronousParameters.UserApcContext,
+            KeInsertQueue(FileObject->CompletionContext->Port,
-                              Irp->IoStatus.Status,
+                          &Irp->Tail.Overlay.ListEntry);
-                              Irp->IoStatus.Information,
-                              FALSE);
             Irp = NULL;
         }
     }
@@ -369,7 +367,7 @@ IoAllocateIrp(CCHAR StackSize,
     PIRP Irp = NULL;
     USHORT Size = IoSizeOfIrp(StackSize);
     PKPRCB Prcb;
-    ULONG Flags = 0;
+    UCHAR Flags = 0;
     PNPAGED_LOOKASIDE_LIST List;
     PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;