- Implement KiUnlinkThread to unlink a thread from its wait blocks, handle queue and timer activation/removal.

- Don't play with priorities in KiAbortWaitThread anymore, since we'll soon support Win2003 delayed "adjust increment" functionality, so that the code is not repeated many times.
- Rename KiAbortWaitThread to KiUnwaitThread, make it use KiUnlinkThread and delayed adjustment.
- Implement KxUnwaitThread and KxUnwaitThreadForEvent, optimized versions of KiWaitTest that can be used in special circumstances (notification events, thread termination, process signalling).
- Optimize KeSetEvent by handling signaled notification events without acquiring the dispatcher lock, and by using new inlined routines described above.
- Reimplement KeSetEventBoostPriority properly to actually do boosting.
- Fixup KeRundownQueue with a more typical/proper LIST_ENTRY loop.
- Let me know if you see regressions...

svn path=/trunk/; revision=24102

reactos/ntoskrnl/include/internal/ke.h

@@ -455,6 +455,13 @@ KiDispatcherObjectWake(
     KPRIORITY increment
 );
 
+VOID
+FASTCALL
+KiUnlinkThread(
+    IN PKTHREAD Thread,
+    IN NTSTATUS WaitStatus
+);
+
 VOID
 NTAPI
 KeExpireTimers(
@@ -474,7 +481,7 @@ KiTestAlert(VOID);
 
 VOID
 FASTCALL
-KiAbortWaitThread(
+KiUnwaitThread(
     IN PKTHREAD Thread,
     IN NTSTATUS WaitStatus,
     IN KPRIORITY Increment

reactos/ntoskrnl/include/internal/ke_x.h

@@ -292,6 +292,98 @@ KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime,
     }                                                                       \
 }
 
+//
+// Unwaits a Thread
+//
+FORCEINLINE
+VOID
+KxUnwaitThread(IN DISPATCHER_HEADER *Object,
+               IN KPRIORITY Increment)
+{
+    PLIST_ENTRY WaitEntry, WaitList;
+    PKWAIT_BLOCK CurrentWaitBlock;
+    PKTHREAD WaitThread;
+    ULONG WaitKey;
+
+    /* Loop the Wait Entries */
+    WaitList = &Object->WaitListHead;
+    WaitEntry = WaitList->Flink;
+    do
+    {
+        /* Get the current wait block */
+        CurrentWaitBlock = CONTAINING_RECORD(WaitEntry,
+                                             KWAIT_BLOCK,
+                                             WaitListEntry);
+
+        /* Get the waiting thread */
+        WaitThread = CurrentWaitBlock->Thread;
+
+        /* Check the current Wait Mode */
+        if (CurrentWaitBlock->WaitType == WaitAny)
+        {
+            /* Use the actual wait key */
+            WaitKey = CurrentWaitBlock->WaitKey;
+        }
+        else
+        {
+            /* Otherwise, use STATUS_KERNEL_APC */
+            WaitKey = STATUS_KERNEL_APC;
+        }
+
+        /* Unwait the thread */
+        KiUnwaitThread(WaitThread, WaitKey, Increment);
+
+        /* Next entry */
+        WaitEntry = WaitList->Flink;
+    } while (WaitEntry != WaitList);
+}
+
+//
+// Unwaits a Thread waiting on an event
+//
+FORCEINLINE
+VOID
+KxUnwaitThreadForEvent(IN PKEVENT Event,
+                       IN KPRIORITY Increment)
+{
+    PLIST_ENTRY WaitEntry, WaitList;
+    PKWAIT_BLOCK CurrentWaitBlock;
+    PKTHREAD WaitThread;
+
+    /* Loop the Wait Entries */
+    WaitList = &Event->Header.WaitListHead;
+    WaitEntry = WaitList->Flink;
+    do
+    {
+        /* Get the current wait block */
+        CurrentWaitBlock = CONTAINING_RECORD(WaitEntry,
+                                             KWAIT_BLOCK,
+                                             WaitListEntry);
+
+        /* Get the waiting thread */
+        WaitThread = CurrentWaitBlock->Thread;
+
+        /* Check the current Wait Mode */
+        if (CurrentWaitBlock->WaitType == WaitAny)
+        {
+            /* Un-signal it */
+            Event->Header.SignalState = 0;
+
+            /* Un-signal the event and unwait the thread */
+            KiUnwaitThread(WaitThread, CurrentWaitBlock->WaitKey, Increment);
+            break;
+        }
+        else
+        {
+            /* Unwait the thread with STATUS_KERNEL_APC */
+            KiUnwaitThread(WaitThread, STATUS_KERNEL_APC, Increment);
+        }
+
+        /* Next entry */
+        WaitEntry = WaitList->Flink;
+    } while (WaitEntry != WaitList);
+}
+
 //
 // Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
 //

reactos/ntoskrnl/ke/apc.c

@@ -239,7 +239,7 @@ KiInsertQueueApc(IN PKAPC Apc,
 
                         /* Wake up the thread */
 Unwait:
-                        KiAbortWaitThread(Thread, Status, PriorityBoost);
+                        KiUnwaitThread(Thread, Status, PriorityBoost);
                     }
                     else
                     {
@@ -978,3 +978,4 @@ KeAreAllApcsDisabled(VOID)
             (KeGetCurrentIrql() >= APC_LEVEL)) ? TRUE : FALSE;
 }
 
+

reactos/ntoskrnl/ke/event.c

@@ -166,19 +166,29 @@ KeSetEvent(IN PKEVENT Event,
     ASSERT_EVENT(Event);
     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 
+    /*
+     * Check if this is an signaled notification event without an upcoming wait.
+     * In this case, we can immediately return TRUE, without locking.
+     */
+    if ((Event->Header.Type == NotificationEvent) &&
+        (Event->Header.SignalState == 1) &&
+        !(Wait))
+    {
+        /* Return the signal state (TRUE/Signalled) */
+        return TRUE;
+    }
+
     /* Lock the Dispathcer Database */
     OldIrql = KiAcquireDispatcherLock();
 
     /* Save the Previous State */
     PreviousState = Event->Header.SignalState;
 
-    /* Check if we have stuff in the Wait Queue */
+    /* Set the Event to Signaled */
-    if (IsListEmpty(&Event->Header.WaitListHead))
+    Event->Header.SignalState = 1;
-    {
+
-        /* Set the Event to Signaled */
+    /* Check if the event just became signaled now, and it has waiters */
-        Event->Header.SignalState = 1;
+    if (!(PreviousState) && !(IsListEmpty(&Event->Header.WaitListHead)))
-    }
-    else
     {
         /* Get the Wait Block */
         WaitBlock = CONTAINING_RECORD(Event->Header.WaitListHead.Flink,
@@ -186,22 +196,15 @@ KeSetEvent(IN PKEVENT Event,
                                       WaitListEntry);
 
         /* Check the type of event */
-        if ((Event->Header.Type == NotificationEvent) || (WaitBlock->WaitType == WaitAll))
+        if (Event->Header.Type == NotificationEvent)
         {
-            /* Check if it wasn't signaled */
+            /* Unwait the thread */
-            if (!PreviousState)
+            KxUnwaitThread(&Event->Header, Increment);
-            {
-                /* We must do a full wait satisfaction */
-                Event->Header.SignalState = 1;
-                KiWaitTest(&Event->Header, Increment);
-            }
         }
         else
         {
-            /* We can satisfy wait simply by waking the thread */
+            /* Otherwise unwait the thread and unsignal the event */
-            KiAbortWaitThread(WaitBlock->Thread,
+            KxUnwaitThreadForEvent(Event, Increment);
-                              WaitBlock->WaitKey,
-                              Increment);
         }
     }
 
@@ -229,38 +232,60 @@ KeSetEvent(IN PKEVENT Event,
 VOID
 NTAPI
 KeSetEventBoostPriority(IN PKEVENT Event,
-                        IN PKTHREAD *Thread OPTIONAL)
+                        IN PKTHREAD *WaitingThread OPTIONAL)
 {
-    PKTHREAD WaitingThread;
     KIRQL OldIrql;
+    PKWAIT_BLOCK WaitBlock;
+    PKTHREAD Thread = KeGetCurrentThread(), WaitThread;
     ASSERT_EVENT(Event);
     ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
 
-    //
-    // FIXME: This is half-broken, there's no boosting!
-    //
-
     /* Acquire Dispatcher Database Lock */
     OldIrql = KiAcquireDispatcherLock();
 
-    /* If our wait list is empty, then signal the event and return */
+    /* Check if the list is empty */
     if (IsListEmpty(&Event->Header.WaitListHead))
     {
+        /* Set the Event to Signaled */
         Event->Header.SignalState = 1;
+
+        /* Return */
+        KiReleaseDispatcherLock(OldIrql);
+        return;
+    }
+
+    /* Get the Wait Block */
+    WaitBlock = CONTAINING_RECORD(Event->Header.WaitListHead.Flink,
+                                  KWAIT_BLOCK,
+                                  WaitListEntry);
+
+    /* Check if this is a WaitAll */
+    if (WaitBlock->WaitType == WaitAll)
+    {
+        /* Set the Event to Signaled */
+        Event->Header.SignalState = 1;
+
+        /* Unwait the thread and unsignal the event */
+        KxUnwaitThreadForEvent(Event, EVENT_INCREMENT);
     }
     else
     {
-        /* Get the waiting thread */
+        /* Return waiting thread to caller */
-        WaitingThread = CONTAINING_RECORD(Event->Header.WaitListHead.Flink,
+        WaitThread = WaitBlock->Thread;
-                                          KWAIT_BLOCK,
+        if (WaitingThread) *WaitingThread = WaitThread;
-                                          WaitListEntry)->Thread;
 
-        /* Return it to caller if requested */
+        /* Calculate new priority */
-        if (Thread) *Thread = WaitingThread;
+        Thread->Priority = KiComputeNewPriority(Thread);
 
-        /* Reset the Quantum and Unwait the Thread */
+        /* Unlink the waiting thread */
-        WaitingThread->Quantum = WaitingThread->QuantumReset;
+        KiUnlinkThread(WaitThread, STATUS_WAIT_0);
-        KiAbortWaitThread(WaitingThread, STATUS_SUCCESS, EVENT_INCREMENT);
+
+        /* Request priority boosting */
+        WaitThread->AdjustIncrement = Thread->Priority;
+        WaitThread->AdjustReason = 2;
+
+        /* Ready the thread */
+        KiReadyThread(WaitThread);
     }
 
     /* Release the Dispatcher Database Lock */

reactos/ntoskrnl/ke/gate.c

@@ -142,7 +142,7 @@ KeSignalGateBoostPriority(IN PKGATE Gate)
         if (WaitThread->Queue) WaitThread->Queue->CurrentCount++;
 
         /* FIXME: This isn't really correct!!! */
-        KiAbortWaitThread(WaitThread, WaitStatus, EVENT_INCREMENT);
+        KiUnwaitThread(WaitThread, WaitStatus, EVENT_INCREMENT);
     }
 
     /* Exit the dispatcher */

reactos/ntoskrnl/ke/process.c

@@ -208,8 +208,8 @@ KeSetProcess(IN PKPROCESS Process,
     if (!(OldState) &&
         !(IsListEmpty(&Process->Header.WaitListHead)))
     {
-        /* Satisfy waits */
+        /* Unwait the threads */
-        KiWaitTest((PVOID)Process, Increment);
+        KxUnwaitThread(&Process->Header, Increment);
     }
 
     /* Release Dispatcher Database */

reactos/ntoskrnl/ke/queue.c

@@ -57,7 +57,7 @@ KiWakeQueue(IN PKQUEUE Queue)
                                           KWAIT_BLOCK,
                                           WaitListEntry);
             Thread = WaitBlock->Thread;
-            KiAbortWaitThread(Thread, (NTSTATUS)QueueEntry, IO_NO_INCREMENT);
+            KiUnwaitThread(Thread, (NTSTATUS)QueueEntry, IO_NO_INCREMENT);
         }
     }
 }
@@ -401,7 +401,7 @@ KeRemoveQueue(IN PKQUEUE Queue,
                     if (!KiInsertTimer(Timer, *Timeout))
                     {
                         /* FIXME */
-                        DPRINT1("If you see thie message contact Alex ASAP\n");
+                        DPRINT1("If you see this message contact Alex ASAP\n");
                         KEBUGCHECK(0);
                     }
 
@@ -469,7 +469,7 @@ PLIST_ENTRY
 NTAPI
 KeRundownQueue(IN PKQUEUE Queue)
 {
-    PLIST_ENTRY EnumEntry;
+    PLIST_ENTRY ListHead, NextEntry;
     PLIST_ENTRY FirstEntry = NULL;
     PKTHREAD Thread;
     KIRQL OldIrql;
@@ -488,16 +488,21 @@ KeRundownQueue(IN PKQUEUE Queue)
     }
 
     /* Unlink threads and clear their Thread->Queue */
-    while (!IsListEmpty(&Queue->ThreadListHead))
+    ListHead = &Queue->ThreadListHead;
+    NextEntry = ListHead->Flink;
+    while (ListHead != NextEntry)
     {
-        /* Get the Entry and Remove it */
-        EnumEntry = RemoveHeadList(&Queue->ThreadListHead);
-
         /* Get the Entry's Thread */
-        Thread = CONTAINING_RECORD(EnumEntry, KTHREAD, QueueListEntry);
+        Thread = CONTAINING_RECORD(NextEntry, KTHREAD, QueueListEntry);
 
         /* Kill its Queue */
         Thread->Queue = NULL;
+
+        /* Remove this entry */
+        RemoveEntryList(NextEntry);
+
+        /* Get the next entry */
+        NextEntry = NextEntry->Flink;
     }
 
     /* Release the lock and return */

reactos/ntoskrnl/ke/thrdobj.c

@@ -100,7 +100,7 @@ KeAlertResumeThread(IN PKTHREAD Thread)
         if ((Thread->State == Waiting) && (Thread->Alertable))
         {
             /* Abort the wait */
-            KiAbortWaitThread(Thread, STATUS_ALERTED, THREAD_ALERT_INCREMENT);
+            KiUnwaitThread(Thread, STATUS_ALERTED, THREAD_ALERT_INCREMENT);
         }
         else
         {
@@ -157,7 +157,7 @@ KeAlertThread(IN PKTHREAD Thread,
             (Thread->Alertable))
         {
             /* Abort the wait to alert the thread */
-            KiAbortWaitThread(Thread, STATUS_ALERTED, THREAD_ALERT_INCREMENT);
+            KiUnwaitThread(Thread, STATUS_ALERTED, THREAD_ALERT_INCREMENT);
         }
         else
         {
@@ -1311,10 +1311,10 @@ KeTerminateThread(IN KPRIORITY Increment)
 
     /* Signal the thread */
     Thread->DispatcherHeader.SignalState = TRUE;
-    if (IsListEmpty(&Thread->DispatcherHeader.WaitListHead) != TRUE)
+    if (!IsListEmpty(&Thread->DispatcherHeader.WaitListHead))
     {
-        /* Satisfy waits */
+        /* Unwait the threads */
-        KiWaitTest((PVOID)Thread, Increment);
+        KxUnwaitThread(&Thread->DispatcherHeader, Increment);
     }
 
     /* Remove the thread from the list */

reactos/ntoskrnl/ke/wait.c

@@ -13,10 +13,6 @@
 #define NDEBUG
 #include <internal/debug.h>
 
-/* GLOBALS ******************************************************************/
-
-KSPIN_LOCK DispatcherDatabaseLock;
-
 /* PRIVATE FUNCTIONS *********************************************************/
 
 VOID
@@ -110,7 +106,7 @@ KiWaitTest(IN PVOID ObjectPointer,
         }
 
         /* All waits satisfied, unwait the thread */
-        KiAbortWaitThread(WaitThread, CurrentWaitBlock->WaitKey, Increment);
+        KiUnwaitThread(WaitThread, CurrentWaitBlock->WaitKey, Increment);
 
 SkipUnwait:
         /* Next entry */
@@ -118,16 +114,13 @@ SkipUnwait:
     }
 }
 
-/* Must be called with the dispatcher lock held */
 VOID
 FASTCALL
-KiAbortWaitThread(IN PKTHREAD Thread,
+KiUnlinkThread(IN PKTHREAD Thread,
-                  IN NTSTATUS WaitStatus,
+               IN NTSTATUS WaitStatus)
-                  IN KPRIORITY Increment)
 {
     PKWAIT_BLOCK WaitBlock;
     PKTIMER Timer;
-    LONG NewPriority;
 
     /* Update wait status */
     Thread->WaitStatus |= WaitStatus;
@@ -158,64 +151,22 @@ KiAbortWaitThread(IN PKTHREAD Thread,
 
     /* Increment the Queue's active threads */
     if (Thread->Queue) Thread->Queue->CurrentCount++;
+}
 
-    /* Check if this is a non-RT thread */
+/* Must be called with the dispatcher lock held */
-    if (Thread->Priority < LOW_REALTIME_PRIORITY)
+VOID
-    {
+FASTCALL
-        /* Check if boosting is enabled and we can boost */
+KiUnwaitThread(IN PKTHREAD Thread,
-        if (!(Thread->DisableBoost) && !(Thread->PriorityDecrement))
+               IN NTSTATUS WaitStatus,
-        {
+               IN KPRIORITY Increment)
-            /* We can boost, so calculate the new priority */
+{
-            NewPriority = Thread->BasePriority + Increment;
+    /* Unlink the thread */
-            if (NewPriority > Thread->Priority)
+    KiUnlinkThread(Thread, WaitStatus);
-            {
-                /* Make sure the new priority wouldn't push the thread to RT */
-                if (NewPriority >= LOW_REALTIME_PRIORITY)
-                {
-                    /* Set it just before the RT zone */
-                    Thread->Priority = LOW_REALTIME_PRIORITY - 1;
-                }
-                else
-                {
-                    /* Otherwise, set our calculated priority */
-                    Thread->Priority = NewPriority;
-                }
-            }
-        }
 
-        /* Check if this is a high-priority thread */
+    /* Tell the scheduler do to the increment when it readies the thread */
-        if (Thread->BasePriority >= 14)
+    ASSERT(Increment >= 0);
-        {
+    Thread->AdjustIncrement = (SCHAR)Increment;
-            /* It is, simply reset the quantum */
+    Thread->AdjustReason = 1;
-            Thread->Quantum = Thread->QuantumReset;
-        }
-        else
-        {
-            /* Otherwise, decrease quantum */
-            Thread->Quantum--;
-            if (Thread->Quantum <= 0)
-            {
-                /* We've went below 0, reset it */
-                Thread->Quantum = Thread->QuantumReset;
-
-                /* Apply per-quantum priority decrement */
-                Thread->Priority -= (Thread->PriorityDecrement + 1);
-                if (Thread->Priority < Thread->BasePriority)
-                {
-                    /* We've went too low, reset it */
-                    Thread->Priority = Thread->BasePriority;
-                }
-
-                /* Delete per-quantum decrement */
-                Thread->PriorityDecrement = 0;
-            }
-        }
-    }
-    else
-    {
-        /* For real time threads, just reset the quantum */
-        Thread->Quantum = Thread->QuantumReset;
-    }
 
     /* Reschedule the Thread */
     KiReadyThread(Thread);
@@ -794,7 +745,7 @@ KeWaitForMultipleObjects(IN ULONG Count,
                 WaitBlock->Object = CurrentObject;
                 WaitBlock->Thread = CurrentThread;
                 WaitBlock->WaitKey = (USHORT)WaitIndex;
-                WaitBlock->WaitType = (USHORT)WaitType;
+                WaitBlock->WaitType = (UCHAR)WaitType;
                 WaitBlock->NextWaitBlock = WaitBlock + 1;
 
                 /* Move to the next Wait Block */