- Merge in my latest Dispatcher changes for KeWaitForMultipleObject, which are basically the same kind of changes as have been done for KeDelay/KeWaitForSingle. (Optimizations and readability improvements).

- Also fixed a previous bug where we didn't respect alertable waits anymore for KeWaitForMultiple...
- Remove krnlfun entry, the rest of dispatcher changes are tied to the new timer implementation for later.

svn path=/trunk/; revision=24138

reactos/ntoskrnl/KrnlFun.c

@@ -33,7 +33,6 @@
 //      * Add DR macro/save and VM macro/save.
 //  - FEATURES:
 //      * New optimized table-based tick-hashed timer implementation.
-//      * New optimized dispatching logic/branching.
 //      * New Thread Scheduler based on 2003.
 //      * Proper Idle/Initial Thread setup and 2nd stage boot.
 //      * Implement KiCallbackReturn, KiGetTickCount, KiRaiseAssertion.

reactos/ntoskrnl/include/internal/ke_x.h

@@ -331,6 +331,40 @@ KxDelayThreadWait(IN PKTHREAD Thread,
     return Swappable;
 }
 
+FORCEINLINE
+BOOLEAN
+KxMultiThreadWait(IN PKTHREAD Thread,
+                  IN PKWAIT_BLOCK WaitBlock,
+                  IN BOOLEAN Alertable,
+                  IN KWAIT_REASON WaitReason,
+                  IN KPROCESSOR_MODE WaitMode)
+{
+    BOOLEAN Swappable;
+    PKTIMER ThreadTimer = &Thread->Timer;
+
+    /* Set default wait status */
+    Thread->WaitStatus = STATUS_WAIT_0;
+
+    /* Link wait block array to the thread */
+    Thread->WaitBlockList = WaitBlock;
+
+    /* Initialize the timer list */
+    InitializeListHead(&ThreadTimer->Header.WaitListHead);
+
+    /* Set wait settings */
+    Thread->Alertable = Alertable;
+    Thread->WaitMode = WaitMode;
+    Thread->WaitReason = WaitReason;
+
+    /* Check if we can swap the thread's stack */
+    Thread->WaitListEntry.Flink = NULL;
+    Swappable = KiCheckThreadStackSwap(Thread, WaitMode);
+
+    /* Set the wait time */
+    Thread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
+    return Swappable;
+}
+
 FORCEINLINE
 BOOLEAN
 KxSingleThreadWait(IN PKTHREAD Thread,

reactos/ntoskrnl/ke/wait.c

@@ -556,7 +556,7 @@ KeWaitForMultipleObjects(IN ULONG Count,
     PKWAIT_BLOCK WaitBlock;
     PKWAIT_BLOCK TimerWaitBlock;
     PKTIMER ThreadTimer;
-    PKTHREAD CurrentThread = KeGetCurrentThread();
+    PKTHREAD Thread = KeGetCurrentThread();
     ULONG AllObjectsSignaled;
     ULONG WaitIndex;
     NTSTATUS WaitStatus = STATUS_SUCCESS;
@@ -564,21 +564,6 @@ KeWaitForMultipleObjects(IN ULONG Count,
     PLARGE_INTEGER OriginalDueTime = Timeout;
     LARGE_INTEGER DueTime, NewDueTime;
 
-    /* Set the Current Thread */
-    CurrentThread = KeGetCurrentThread();
-
-    /* Check if the lock is already held */
-    if (CurrentThread->WaitNext)
-    {
-        /* Lock is held, disable Wait Next */
-        CurrentThread->WaitNext = FALSE;
-    }
-    else
-    {
-        /* Lock not held, acquire it */
-        CurrentThread->WaitIrql = KiAcquireDispatcherLock();
-    }
-
     /* Make sure the Wait Count is valid */
     if (!WaitBlockArray)
     {
@@ -590,7 +575,7 @@ KeWaitForMultipleObjects(IN ULONG Count,
         }
 
         /* Use the Thread's Wait Block */
-        WaitBlockArray = &CurrentThread->WaitBlock[0];
+        WaitBlockArray = &Thread->WaitBlock[0];
     }
     else
     {
@@ -602,24 +587,40 @@ KeWaitForMultipleObjects(IN ULONG Count,
         }
     }
 
-    /* Start the actual Loop */
+    /* Sanity check */
-    do
+    ASSERT(Count != 0);
+
+    /* Check if the lock is already held */
+    if (Thread->WaitNext)
     {
-        /* Check if a kernel APC is pending and we're below APC_LEVEL */
+        /* Lock is held, disable Wait Next */
-        if ((CurrentThread->ApcState.KernelApcPending) &&
+        Thread->WaitNext = FALSE;
-            !(CurrentThread->SpecialApcDisable) &&
-            (CurrentThread->WaitIrql < APC_LEVEL))
-        {
-            /* Unlock the dispatcher */
-            KiReleaseDispatcherLock(CurrentThread->WaitIrql);
     }
     else
     {
-            /* Append wait block to the KTHREAD wait block list */
+        /* Lock not held, acquire it */
-            CurrentThread->WaitBlockList = WaitBlock = WaitBlockArray;
+StartWait:
+        Thread->WaitIrql = KiAcquireDispatcherLock();
+    }
 
-            /* Set default wait status */
+    /* Prepare for the wait */
-            CurrentThread->WaitStatus = STATUS_WAIT_0;
+    Swappable = KxMultiThreadWait(Thread,
+                                  WaitBlockArray,
+                                  Alertable,
+                                  WaitReason,
+                                  WaitMode);
+
+    /* Check if a kernel APC is pending and we're below APC_LEVEL */
+    if ((Thread->ApcState.KernelApcPending) && !(Thread->SpecialApcDisable) &&
+        (Thread->WaitIrql < APC_LEVEL))
+    {
+        /* Unlock the dispatcher */
+        KiReleaseDispatcherLock(Thread->WaitIrql);
+        goto StartWait;
+    }
+
+    /* Append wait block to the KTHREAD wait block list */
+    WaitBlock = WaitBlockArray;
 
     /* Check if the wait is (already) satisfied */
     AllObjectsSignaled = TRUE;
@@ -629,6 +630,7 @@ KeWaitForMultipleObjects(IN ULONG Count,
     {
         /* Get the Current Object */
         CurrentObject = (PKMUTANT)Object[WaitIndex];
+        ASSERT(CurrentObject->Header.Type != QueueObject);
 
         /* Check the type of wait */
         if (WaitType == WaitAny)
@@ -638,24 +640,21 @@ KeWaitForMultipleObjects(IN ULONG Count,
             {
                 /* Check if it's signaled */
                 if ((CurrentObject->Header.SignalState > 0) ||
-                            (CurrentThread == CurrentObject->OwnerThread))
+                    (Thread == CurrentObject->OwnerThread))
                 {
                     /* This is a Wait Any, so unwait this and exit */
                     if (CurrentObject->Header.SignalState !=
                         (LONG)MINLONG)
                     {
                         /* Normal signal state, unwait it and return */
-                                KiSatisfyMutantWait(CurrentObject,
+                        KiSatisfyMutantWait(CurrentObject, Thread);
-                                                    CurrentThread);
+                        WaitStatus = Thread->WaitStatus | WaitIndex;
-                                WaitStatus = CurrentThread->WaitStatus |
-                                             WaitIndex;
                         goto DontWait;
                     }
                     else
                     {
                         /* Raise an exception (see wasm.ru) */
-                                KiReleaseDispatcherLock(CurrentThread->
+                        KiReleaseDispatcherLock(Thread->WaitIrql);
-                                                                WaitIrql);
                         ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
                     }
                 }
@@ -663,7 +662,7 @@ KeWaitForMultipleObjects(IN ULONG Count,
             else if (CurrentObject->Header.SignalState > 0)
             {
                 /* Another signaled object, unwait and return */
-                        KiSatisfyNonMutantWait(CurrentObject, CurrentThread);
+                KiSatisfyNonMutantWait(CurrentObject, Thread);
                 WaitStatus = WaitIndex;
                 goto DontWait;
             }
@@ -674,16 +673,15 @@ KeWaitForMultipleObjects(IN ULONG Count,
             if (CurrentObject->Header.Type == MutantObject)
             {
                 /* Check if it has an invalid count */
-                        if ((CurrentThread == CurrentObject->OwnerThread) &&
+                if ((Thread == CurrentObject->OwnerThread) &&
                     (CurrentObject->Header.SignalState == MINLONG))
                 {
                     /* Raise an exception */
-                            KiReleaseDispatcherLock(CurrentThread->
+                    KiReleaseDispatcherLock(Thread->WaitIrql);
-                                                            WaitIrql);
                     ExRaiseStatus(STATUS_MUTANT_LIMIT_EXCEEDED);
                 }
                 else if ((CurrentObject->Header.SignalState <= 0) &&
-                                 (CurrentThread != CurrentObject->OwnerThread))
+                         (Thread != CurrentObject->OwnerThread))
                 {
                     /* We don't own it, can't satisfy the wait */
                     AllObjectsSignaled = FALSE;
@@ -697,38 +695,34 @@ KeWaitForMultipleObjects(IN ULONG Count,
         }
 
         /* Set up a Wait Block for this Object */
+        WaitBlock = &WaitBlockArray[WaitIndex];
         WaitBlock->Object = CurrentObject;
-                WaitBlock->Thread = CurrentThread;
+        WaitBlock->Thread = Thread;
         WaitBlock->WaitKey = (USHORT)WaitIndex;
         WaitBlock->WaitType = (UCHAR)WaitType;
-                WaitBlock->NextWaitBlock = WaitBlock + 1;
+        WaitBlock->NextWaitBlock = &WaitBlockArray[WaitIndex + 1];
-
-                /* Move to the next Wait Block */
-                WaitBlock = WaitBlock->NextWaitBlock;
     }
 
-            /* Return to the Root Wait Block */
-            WaitBlock--;
-            WaitBlock->NextWaitBlock = WaitBlockArray;
-
     /* Check if this is a Wait All and all the objects are signaled */
     if ((WaitType == WaitAll) && (AllObjectsSignaled))
     {
         /* Return to the Root Wait Block */
-                WaitBlock = CurrentThread->WaitBlockList;
+        WaitBlock->NextWaitBlock = &WaitBlockArray[0];
 
         /* Satisfy their Waits and return to the caller */
         KiWaitSatisfyAll(WaitBlock);
-                WaitStatus = CurrentThread->WaitStatus;
+        WaitStatus = Thread->WaitStatus;
         goto DontWait;
     }
 
     /* Make sure we can satisfy the Alertable request */
-            WaitStatus = KiCheckAlertability(CurrentThread, Alertable, WaitMode);
+    WaitStatus = KiCheckAlertability(Thread, Alertable, WaitMode);
-
+    if (WaitStatus != STATUS_WAIT_0)
-            /* Check if we can swap the thread's stack */
+    {
-            CurrentThread->WaitListEntry.Flink = NULL;
+        /* Unlock the dispatcher and return */
-            Swappable = KiCheckThreadStackSwap(CurrentThread, WaitMode);
+        KiReleaseDispatcherLock(Thread->WaitIrql);
+        return WaitStatus;
+    }
 
     /* Enable the Timeout Timer if there was any specified */
     if (Timeout)
@@ -741,20 +735,15 @@ KeWaitForMultipleObjects(IN ULONG Count,
             goto DontWait;
         }
 
-                /* Point to Timer Wait Block and Thread Timer */
+        /* Link timer wait block */
-                TimerWaitBlock = &CurrentThread->WaitBlock[TIMER_WAIT_BLOCK];
+        TimerWaitBlock = &Thread->WaitBlock[TIMER_WAIT_BLOCK];
-                ThreadTimer = &CurrentThread->Timer;
-
-                /* Connect the Timer Wait Block */
         WaitBlock->NextWaitBlock = TimerWaitBlock;
 
-                /* Set up the Timer Wait Block */
+        /* Use this the timer block for linking below */
-                TimerWaitBlock->NextWaitBlock = WaitBlockArray;
+        WaitBlock = TimerWaitBlock;
-
-                /* Initialize the list head */
-                InitializeListHead(&ThreadTimer->Header.WaitListHead);
 
         /* Insert the Timer into the Timer Lists and enable it */
+        ThreadTimer = &Thread->Timer;
         if (!KiInsertTimer(ThreadTimer, *Timeout))
         {
             /* Return a timeout if we couldn't insert the timer */
@@ -766,8 +755,11 @@ KeWaitForMultipleObjects(IN ULONG Count,
         DueTime.QuadPart = ThreadTimer->DueTime.QuadPart;
     }
 
+    /* Link to the Root Wait Block */
+    WaitBlock->NextWaitBlock = &WaitBlockArray[0];
+
     /* Insert into Object's Wait List*/
-            WaitBlock = CurrentThread->WaitBlockList;
+    WaitBlock = &WaitBlockArray[0];
     do
     {
         /* Get the Current Object */
@@ -782,23 +774,22 @@ KeWaitForMultipleObjects(IN ULONG Count,
     } while (WaitBlock != WaitBlockArray);
 
     /* Handle Kernel Queues */
-            if (CurrentThread->Queue) KiActivateWaiterQueue(CurrentThread->Queue);
+    if (Thread->Queue) KiActivateWaiterQueue(Thread->Queue);
 
     /* Setup the wait information */
-            CurrentThread->Alertable = Alertable;
+    Thread->State = Waiting;
-            CurrentThread->WaitMode = WaitMode;
-            CurrentThread->WaitReason = WaitReason;
-            CurrentThread->WaitTime = ((PLARGE_INTEGER)&KeTickCount)->LowPart;
-            CurrentThread->State = Waiting;
 
-            /* Find a new thread to run */
+    /* Add the thread to the wait list */
-            KiAddThreadToWaitList(CurrentThread, Swappable);
+    KiAddThreadToWaitList(Thread, Swappable);
-            WaitStatus = KiSwapThread(CurrentThread, KeGetCurrentPrcb());
+
-            ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL);
+    /* Swap the thread */
+    ASSERT(Thread->WaitIrql <= DISPATCH_LEVEL);
+    KiSetThreadSwapBusy(Thread);
+    WaitStatus = KiSwapThread(Thread, KeGetCurrentPrcb());
 
     /* Check if we were executing an APC */
-            if (WaitStatus != STATUS_KERNEL_APC) return WaitStatus;
+    if (WaitStatus == STATUS_KERNEL_APC)
-
+    {
         /* Check if we had a timeout */
         if (Timeout)
         {
@@ -808,21 +799,19 @@ KeWaitForMultipleObjects(IN ULONG Count,
                                            &NewDueTime);
         }
 
-            /* Acquire again the lock */
+        /* Wait again */
-            CurrentThread->WaitIrql = KiAcquireDispatcherLock();
+        goto StartWait;
     }
-    } while (TRUE);
 
-    /* Release the Lock, we are done */
+    /* We are done */
-    KiReleaseDispatcherLock(CurrentThread->WaitIrql);
     return WaitStatus;
 
 DontWait:
     /* Adjust the Quantum */
-    KiAdjustQuantumThread(CurrentThread);
+    KiAdjustQuantumThread(Thread);
 
     /* Release & Return */
-    KiReleaseDispatcherLock(CurrentThread->WaitIrql);
+    KiReleaseDispatcherLock(Thread->WaitIrql);
     return WaitStatus;
 }