diff --git a/reactos/include/ddk/iotypes.h b/reactos/include/ddk/iotypes.h index 05789cdd311..5cb0b048738 100644 --- a/reactos/include/ddk/iotypes.h +++ b/reactos/include/ddk/iotypes.h @@ -794,12 +794,17 @@ typedef struct _IRP }; struct _ETHREAD* Thread; PCHAR AuxiliaryBuffer; - LIST_ENTRY ListEntry; - struct _IO_STACK_LOCATION* CurrentStackLocation; + struct { + LIST_ENTRY ListEntry; + union { + struct _IO_STACK_LOCATION* CurrentStackLocation; + ULONG PacketType; + }; + }; PFILE_OBJECT OriginalFileObject; } Overlay; KAPC Apc; - ULONG CompletionKey; + PVOID CompletionKey; } Tail; } IRP, *PIRP; diff --git a/reactos/ntoskrnl/include/internal/io.h b/reactos/ntoskrnl/include/internal/io.h index a0e5fb24b83..e313a04279e 100644 --- a/reactos/ntoskrnl/include/internal/io.h +++ b/reactos/ntoskrnl/include/internal/io.h @@ -43,13 +43,25 @@ struct _DEVICE_OBJECT_POWER_EXTENSION; -typedef struct _IO_COMPLETION_PACKET{ +/* This is like the IRP Overlay so we can optimize its insertion */ +typedef struct _IO_COMPLETION_PACKET +{ + struct { + LIST_ENTRY ListEntry; + union { + struct _IO_STACK_LOCATION *CurrentStackLocation; + ULONG PacketType; + }; + }; PVOID Key; PVOID Context; IO_STATUS_BLOCK IoStatus; - LIST_ENTRY ListEntry; } IO_COMPLETION_PACKET, *PIO_COMPLETION_PACKET; +/* Packet Types */ +#define IrpCompletionPacket 0x1 +#define IrpMiniCompletionPacket 0x2 + typedef struct _DEVOBJ_EXTENSION { CSHORT Type; USHORT Size; diff --git a/reactos/ntoskrnl/io/iocomp.c b/reactos/ntoskrnl/io/iocomp.c index 3b956357646..298ace65982 100644 --- a/reactos/ntoskrnl/io/iocomp.c +++ b/reactos/ntoskrnl/io/iocomp.c @@ -35,6 +35,40 @@ static const INFORMATION_CLASS_INFO ExIoCompletionInfoClass[] = { /* FUNCTIONS *****************************************************************/ +VOID +STDCALL +IopFreeIoCompletionPacket(PIO_COMPLETION_PACKET Packet) +{ + PKPRCB Prcb = KeGetCurrentPrcb(); + PNPAGED_LOOKASIDE_LIST List; + + /* Use the P List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].P; + List->L.TotalFrees++; + + /* Check if the Free was within the Depth or not */ + if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth) + { + /* Let the balancer know */ + List->L.FreeMisses++; + + /* Use the L List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].L; + List->L.TotalFrees++; + + /* Check if the Free was within the Depth or not */ + if (ExQueryDepthSList(&List->L.ListHead) >= List->L.Depth) + { + /* All lists failed, use the pool */ + List->L.FreeMisses++; + ExFreePool(Packet); + } + } + + /* The free was within dhe Depth */ + InterlockedPushEntrySList(&List->L.ListHead, (PSINGLE_LIST_ENTRY)Packet); +} + VOID STDCALL IopDeleteIoCompletion(PVOID ObjectBody) @@ -42,6 +76,7 @@ IopDeleteIoCompletion(PVOID ObjectBody) PKQUEUE Queue = ObjectBody; PLIST_ENTRY FirstEntry; PLIST_ENTRY CurrentEntry; + PIRP Irp; PIO_COMPLETION_PACKET Packet; DPRINT("IopDeleteIoCompletion()\n"); @@ -61,8 +96,18 @@ IopDeleteIoCompletion(PVOID ObjectBody) /* Go to next Entry */ CurrentEntry = CurrentEntry->Flink; - /* Free it */ - ExFreeToNPagedLookasideList(&IoCompletionPacketLookaside, Packet); + /* Check if it's part of an IRP, or a separate packet */ + if (Packet->PacketType == IrpCompletionPacket) + { + /* Get the IRP and free it */ + Irp = CONTAINING_RECORD(Packet, IRP, Tail.Overlay.ListEntry); + IoFreeIrp(Irp); + } + else + { + /* Use common routine */ + IopFreeIoCompletionPacket(Packet); + } } while (FirstEntry != CurrentEntry); } } @@ -80,20 +125,58 @@ IoSetIoCompletion(IN PVOID IoCompletion, IN BOOLEAN Quota) { PKQUEUE Queue = (PKQUEUE)IoCompletion; + PNPAGED_LOOKASIDE_LIST List; + PKPRCB Prcb = KeGetCurrentPrcb(); PIO_COMPLETION_PACKET Packet; - /* Allocate the Packet */ - Packet = ExAllocateFromNPagedLookasideList(&IoCompletionPacketLookaside); - if (NULL == Packet) return STATUS_NO_MEMORY; + /* Get the P List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].P; + + /* Try to allocate the Packet */ + List->L.TotalAllocates++; + Packet = (PVOID)InterlockedPopEntrySList(&List->L.ListHead); + + /* Check if that failed, use the L list if it did */ + if (!Packet) + { + /* Let the balancer know */ + List->L.AllocateMisses++; + + /* Get L List */ + List = (PNPAGED_LOOKASIDE_LIST)Prcb->PPLookasideList[LookasideCompletionList].L; + + /* Try to allocate the Packet */ + List->L.TotalAllocates++; + Packet = (PVOID)InterlockedPopEntrySList(&List->L.ListHead); + } + + /* Still failed, use pool */ + if (!Packet) + { + /* Let the balancer know */ + List->L.AllocateMisses++; + + /* Allocate from Nonpaged Pool */ + Packet = ExAllocatePoolWithTag(NonPagedPool, sizeof(*Packet), IOC_TAG); + } + + /* Make sure we have one by now... */ + if (Packet) + { + /* Set up the Packet */ + Packet->PacketType = IrpMiniCompletionPacket; + Packet->Key = KeyContext; + Packet->Context = ApcContext; + Packet->IoStatus.Status = IoStatus; + Packet->IoStatus.Information = IoStatusInformation; - /* Set up the Packet */ - Packet->Key = KeyContext; - Packet->Context = ApcContext; - Packet->IoStatus.Status = IoStatus; - Packet->IoStatus.Information = IoStatusInformation; - - /* Insert the Queue */ - KeInsertQueue(Queue, &Packet->ListEntry); + /* Insert the Queue */ + KeInsertQueue(Queue, &Packet->ListEntry); + } + else + { + return STATUS_INSUFFICIENT_RESOURCES; + } /* Return Success */ return STATUS_SUCCESS; @@ -410,10 +493,27 @@ NtRemoveIoCompletion(IN HANDLE IoCompletionHandle, _SEH_TRY { - /* Return it */ - *CompletionKey = Packet->Key; - *CompletionContext = Packet->Context; - *IoStatusBlock = Packet->IoStatus; + /* Check if this is piggybacked on an IRP */ + if (Packet->PacketType == IrpCompletionPacket) + { + /* Get the IRP */ + PIRP Irp = NULL; + Irp = CONTAINING_RECORD(ListEntry, IRP, Tail.Overlay.ListEntry); + + /* Return values to user */ + *CompletionKey = Irp->Tail.CompletionKey; + *CompletionContext = Irp->Overlay.AsynchronousParameters.UserApcContext; + *IoStatusBlock = Packet->IoStatus; + IoFreeIrp(Irp); + } + else + { + /* This is a user-mode generated or API generated mini-packet */ + *CompletionKey = Packet->Key; + *CompletionContext = Packet->Context; + *IoStatusBlock = Packet->IoStatus; + IopFreeIoCompletionPacket(Packet); + } } _SEH_HANDLE { diff --git a/reactos/ntoskrnl/io/irp.c b/reactos/ntoskrnl/io/irp.c index 4909d32e1a5..e7647c25d1e 100644 --- a/reactos/ntoskrnl/io/irp.c +++ b/reactos/ntoskrnl/io/irp.c @@ -187,12 +187,10 @@ IopCompleteRequest(PKAPC Apc, else if (FileObject && FileObject->CompletionContext) { /* Call the IO Completion Port if we have one, instead */ - IoSetIoCompletion(FileObject->CompletionContext->Port, - FileObject->CompletionContext->Key, - Irp->Overlay.AsynchronousParameters.UserApcContext, - Irp->IoStatus.Status, - Irp->IoStatus.Information, - FALSE); + Irp->Tail.CompletionKey = FileObject->CompletionContext->Key; + Irp->Tail.Overlay.PacketType = IrpCompletionPacket; + KeInsertQueue(FileObject->CompletionContext->Port, + &Irp->Tail.Overlay.ListEntry); Irp = NULL; } } @@ -369,7 +367,7 @@ IoAllocateIrp(CCHAR StackSize, PIRP Irp = NULL; USHORT Size = IoSizeOfIrp(StackSize); PKPRCB Prcb; - ULONG Flags = 0; + UCHAR Flags = 0; PNPAGED_LOOKASIDE_LIST List; PP_NPAGED_LOOKASIDE_NUMBER ListType = LookasideSmallIrpList;