From 20425875ce380811ece9841a9fc83fa52ec2c96e Mon Sep 17 00:00:00 2001 From: Alex Ionescu Date: Thu, 14 Sep 2006 05:53:51 +0000 Subject: [PATCH] - Implement KiAcquireDispatcherObject, KiReleaseDispatcherObject, used on SMP for dispatcher objects which use the volatile long member in the dispatcher header (such as gates). - Implement KiTryThreadLock which tries to acquire the lock but exits if it can't, isntead of looping. - Re-factor KeWaitForGate to make it SMP-safe by acquiring various locks when required. - Fixed up KeSignalGateBoostPriority so it can also be SMP-safe, and also fix what exactly happens when the gate is signaled (directly make the other thread ready instead of doing a KiUnwaitThread on it). - Split KxAcquireSpinLock/KxReleaseSpinLock into SMP vs UP like the other routines. svn path=/trunk/; revision=24103 --- reactos/include/ndk/ketypes.h | 1 + reactos/ntoskrnl/include/internal/ke_x.h | 186 +++++++++++++++++------ reactos/ntoskrnl/ke/gate.c | 185 +++++++++++++++------- 3 files changed, 275 insertions(+), 97 deletions(-) diff --git a/reactos/include/ndk/ketypes.h b/reactos/include/ndk/ketypes.h index 07a1cabbdfd..4df317882be 100644 --- a/reactos/include/ndk/ketypes.h +++ b/reactos/include/ndk/ketypes.h @@ -43,6 +43,7 @@ Author: // Object Type Mask for Kernel Dispatcher Objects // #define KOBJECT_TYPE_MASK 0x7F +#define KOBJECT_LOCK_BIT 0x80 // // Dispatcher Priority increments diff --git a/reactos/ntoskrnl/include/internal/ke_x.h b/reactos/ntoskrnl/include/internal/ke_x.h index 9b3d8bffe6c..c70eb871396 100644 --- a/reactos/ntoskrnl/include/internal/ke_x.h +++ b/reactos/ntoskrnl/include/internal/ke_x.h @@ -384,45 +384,16 @@ KxUnwaitThreadForEvent(IN PKEVENT Event, } while (WaitEntry != WaitList); } +#ifndef _CONFIG_SMP // -// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL +// Spinlock Acquire at IRQL >= DISPATCH_LEVEL // FORCEINLINE VOID KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock) { -#ifdef CONFIG_SMP - for (;;) - { - /* Try to acquire it */ - if (InterlockedBitTestAndSet((PLONG)SpinLock, 0)) - { - /* Value changed... wait until it's locked */ - while (*(volatile KSPIN_LOCK *)SpinLock == 1) - { -#ifdef DBG - /* On debug builds, we use a much slower but useful routine */ - Kii386SpinOnSpinLock(SpinLock, 5); -#else - /* Otherwise, just yield and keep looping */ - YieldProcessor(); -#endif - } - } - else - { -#ifdef DBG - /* On debug builds, we OR in the KTHREAD */ - *SpinLock = KeGetCurrentThread() | 1; -#endif - /* All is well, break out */ - break; - } - } -#else /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ UNREFERENCED_PARAMETER(SpinLock); -#endif } // @@ -432,27 +403,30 @@ FORCEINLINE VOID KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock) { -#ifdef CONFIG_SMP -#ifdef DBG - /* Make sure that the threads match */ - if ((KeGetCurrentThread() | 1) != *SpinLock) - { - /* They don't, bugcheck */ - KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0); - } -#endif - /* Clear the lock */ - InterlockedAnd(SpinLock, 0); -#else /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ UNREFERENCED_PARAMETER(SpinLock); -#endif } // -// Thread Scheduling Routines +// This routine protects against multiple CPU acquires, it's meaningless on UP. // -#ifndef _CONFIG_SMP +VOID +FORCEINLINE +KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) +{ + UNREFERENCED_PARAMETER(Object); +} + +// +// This routine protects against multiple CPU acquires, it's meaningless on UP. +// +VOID +FORCEINLINE +KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) +{ + UNREFERENCED_PARAMETER(Object); +} + KIRQL FORCEINLINE KiAcquireDispatcherLock(VOID) @@ -560,6 +534,17 @@ KiReleaseThreadLock(IN PKTHREAD Thread) UNREFERENCED_PARAMETER(Thread); } +// +// This routine protects against multiple CPU acquires, it's meaningless on UP. +// +FORCEINLINE +BOOLEAN +KiTryThreadLock(IN PKTHREAD Thread) +{ + UNREFERENCED_PARAMETER(Thread); + return FALSE; +} + FORCEINLINE VOID KiCheckDeferredReadyList(IN PKPRCB Prcb) @@ -597,6 +582,98 @@ KiRequestApcInterrupt(IN BOOLEAN NeedApc, #else +// +// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL +// +FORCEINLINE +VOID +KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock) +{ + for (;;) + { + /* Try to acquire it */ + if (InterlockedBitTestAndSet((PLONG)SpinLock, 0)) + { + /* Value changed... wait until it's locked */ + while (*(volatile KSPIN_LOCK *)SpinLock == 1) + { +#ifdef DBG + /* On debug builds, we use a much slower but useful routine */ + Kii386SpinOnSpinLock(SpinLock, 5); +#else + /* Otherwise, just yield and keep looping */ + YieldProcessor(); +#endif + } + } + else + { +#ifdef DBG + /* On debug builds, we OR in the KTHREAD */ + *SpinLock = KeGetCurrentThread() | 1; +#endif + /* All is well, break out */ + break; + } + } +} + +// +// Spinlock Release at IRQL >= DISPATCH_LEVEL +// +FORCEINLINE +VOID +KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock) +{ +#ifdef DBG + /* Make sure that the threads match */ + if ((KeGetCurrentThread() | 1) != *SpinLock) + { + /* They don't, bugcheck */ + KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0); + } +#endif + /* Clear the lock */ + InterlockedAnd(SpinLock, 0); +} + +KIRQL +FORCEINLINE +KiAcquireDispatcherObject(IN DISPATCHER_HEADER* Object) +{ + LONG OldValue, NewValue; + + /* Make sure we're at a safe level to touch the lock */ + ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + + /* Start acquire loop */ + do + { + /* Loop until the other CPU releases it */ + while ((UCHAR)Object->Lock & KOBJECT_LOCK_BIT) + { + /* Let the CPU know that this is a loop */ + YieldProcessor(); + }; + + /* Try acquiring the lock now */ + NewValue = InterlockedCompareExchange(&Object->Lock, + OldValue | KOBJECT_LOCK_BIT, + OldValue); + } while (NewValue != OldValue); +} + +KIRQL +FORCEINLINE +KiReleaseDispatcherObject(IN DISPATCHER_HEADER* Object) +{ + /* Make sure we're at a safe level to touch the lock */ + ASSERT(KeGetCurrentIrql() >= DISPATCH_LEVEL); + + /* Release it */ + InterlockedAnd(&Object->Lock, ~KOBJECT_LOCK_BIT); +} + KIRQL FORCEINLINE KiAcquireDispatcherLock(VOID) @@ -753,6 +830,23 @@ KiReleaseThreadLock(IN PKTHREAD Thread) InterlockedAnd(&Thread->ThreadLock, 0); } +FORCEINLINE +BOOLEAN +KiTryThreadLock(IN PKTHREAD Thread) +{ + LONG Value; + + /* If the lock isn't acquired, return false */ + if (!Thread->ThreadLock) return FALSE; + + /* Otherwise, try to acquire it and check the result */ + Value = 1; + Value = InterlockedExchange(&Thread->ThreadLock, &Value); + + /* Return the lock state */ + return (Value == TRUE); +} + FORCEINLINE VOID KiCheckDeferredReadyList(IN PKPRCB Prcb) diff --git a/reactos/ntoskrnl/ke/gate.c b/reactos/ntoskrnl/ke/gate.c index f2cd3726d50..a8dab73d0b0 100644 --- a/reactos/ntoskrnl/ke/gate.c +++ b/reactos/ntoskrnl/ke/gate.c @@ -33,21 +33,23 @@ KeWaitForGate(IN PKGATE Gate, IN KPROCESSOR_MODE WaitMode) { KLOCK_QUEUE_HANDLE ApcLock; - PKTHREAD CurrentThread = KeGetCurrentThread(); + PKTHREAD Thread = KeGetCurrentThread(); PKWAIT_BLOCK GateWaitBlock; NTSTATUS Status; + PKQUEUE Queue; ASSERT_GATE(Gate); ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL); + ASSERT(FALSE); /* Start wait loop */ do { /* Acquire the APC lock */ - KiAcquireApcLock(CurrentThread, &ApcLock); + KiAcquireApcLock(Thread, &ApcLock); /* Check if a kernel APC is pending and we're below APC_LEVEL */ - if ((CurrentThread->ApcState.KernelApcPending) && - !(CurrentThread->SpecialApcDisable) && + if ((Thread->ApcState.KernelApcPending) && + !(Thread->SpecialApcDisable) && (ApcLock.OldIrql < APC_LEVEL)) { /* Release the lock, this will fire the APC */ @@ -55,41 +57,71 @@ KeWaitForGate(IN PKGATE Gate, } else { + /* Check if we have a queue and lock the dispatcher if so */ + Queue = Thread->Queue; + if (Queue) KiAcquireDispatcherLockAtDpcLevel(); + + /* Lock the thread */ + KiAcquireThreadLock(Thread); + + /* Lock the gate */ + KiAcquireDispatcherObject(&Gate->Header); + /* Check if it's already signaled */ if (Gate->Header.SignalState) { /* Unsignal it */ Gate->Header.SignalState = 0; + /* Release the gate and thread locks */ + KiReleaseDispatcherObject(&Gate->Header); + KiReleaseThreadLock(Thread); + /* Release the APC lock and return */ KiReleaseApcLock(&ApcLock); return; } /* Setup a Wait Block */ - GateWaitBlock = &CurrentThread->WaitBlock[0]; + GateWaitBlock = &Thread->WaitBlock[0]; GateWaitBlock->Object = (PVOID)Gate; - GateWaitBlock->Thread = CurrentThread; + GateWaitBlock->Thread = Thread; /* Set the Thread Wait Data */ - CurrentThread->WaitMode = WaitMode; - CurrentThread->WaitReason = WaitReason; - CurrentThread->WaitIrql = ApcLock.OldIrql; - CurrentThread->State = GateWait; - CurrentThread->GateObject = Gate; + Thread->WaitMode = WaitMode; + Thread->WaitReason = WaitReason; + Thread->WaitIrql = ApcLock.OldIrql; + Thread->State = GateWait; + Thread->GateObject = Gate; /* Insert into the Wait List */ InsertTailList(&Gate->Header.WaitListHead, &GateWaitBlock->WaitListEntry); - /* Handle Kernel Queues */ - if (CurrentThread->Queue) KiWakeQueue(CurrentThread->Queue); + /* Release the gate lock */ + KiReleaseDispatcherObject(&Gate->Header); + + /* Set swap busy */ + KiSetThreadSwapBusy(Thread); + + /* Release the thread lock */ + KiReleaseThreadLock(Thread); + + /* Check if we had a queue */ + if (Queue) + { + /* Wake it up */ + KiWakeQueue(Queue); + + /* Release the dispatcher lock */ + KiReleaseDispatcherLockFromDpcLevel(); + } /* Release the APC lock but stay at DPC level */ KiReleaseApcLockFromDpcLevel(&ApcLock); /* Find a new thread to run */ - Status = KiSwapThread(CurrentThread, KeGetCurrentPrcb()); + Status = KiSwapThread(Thread, KeGetCurrentPrcb()); /* Check if we were executing an APC */ if (Status != STATUS_KERNEL_APC) return; @@ -104,49 +136,100 @@ KeSignalGateBoostPriority(IN PKGATE Gate) PKTHREAD WaitThread; PKWAIT_BLOCK WaitBlock; KIRQL OldIrql; - NTSTATUS WaitStatus = STATUS_SUCCESS; ASSERT_GATE(Gate); ASSERT_IRQL_LESS_OR_EQUAL(DISPATCH_LEVEL); + ASSERT(FALSE); - /* Raise to synch level */ - OldIrql = KeRaiseIrqlToSynchLevel(); - - /* Make sure we're not already signaled or that the list is empty */ - if (Gate->Header.SignalState) + /* Start entry loop */ + for (;;) { - /* Lower IRQL and quit */ - KeLowerIrql(OldIrql); - return; + /* Raise to synch level */ + OldIrql = KeRaiseIrqlToSynchLevel(); + + /* Lock the gate */ + KiAcquireDispatcherObject(&Gate->Header); + + /* Make sure we're not already signaled or that the list is empty */ + if (Gate->Header.SignalState) + { + /* Lower IRQL and quit */ + KeLowerIrql(OldIrql); + return; + } + + /* Check if our wait list is empty */ + if (IsListEmpty(&Gate->Header.WaitListHead)) + { + /* It is, so signal the event */ + Gate->Header.SignalState = 1; + } + else + { + /* Get WaitBlock */ + WaitBlock = CONTAINING_RECORD(Gate->Header.WaitListHead.Flink, + KWAIT_BLOCK, + WaitListEntry); + + /* Get the Associated thread */ + WaitThread = WaitBlock->Thread; + + /* Check to see if the waiting thread is locked */ + if (!KiTryThreadLock(WaitThread)) + { + /* Unlock the gate */ + KiReleaseDispatcherObject(&Gate->Header); + + /* Lower IRQL and loop again */ + KeLowerIrql(OldIrql); + continue; + } + + /* Remove it */ + RemoveEntryList(&WaitBlock->WaitListEntry); + + /* Clear wait status */ + WaitThread->WaitStatus = 0; + + /* Set state and CPU */ + WaitThread->State = DeferredReady; + WaitThread->DeferredProcessor = KeGetCurrentPrcb()->Number; + + /* Release the gate lock */ + KiReleaseDispatcherObject(&Gate->Header); + + /* Release the thread lock */ + KiReleaseThreadLock(WaitThread); + + /* FIXME: Boosting */ + + /* Check if we have a queue */ + if (WaitThread->Queue) + { + /* Acquire the dispatcher lock */ + KiAcquireDispatcherLockAtDpcLevel(); + + /* Check if we still have one */ + if (WaitThread->Queue) + { + /* Increment active threads */ + WaitThread->Queue->CurrentCount++; + } + + /* Release lock */ + KiReleaseDispatcherLockFromDpcLevel(); + } + + /* Make the thread ready */ + KiReadyThread(WaitThread); + + /* Exit the dispatcher */ + KiExitDispatcher(OldIrql); + } } - /* Check if our wait list is empty */ - if (IsListEmpty(&Gate->Header.WaitListHead)) - { - /* It is, so signal the event */ - Gate->Header.SignalState = 1; - } - else - { - /* Get WaitBlock */ - WaitBlock = CONTAINING_RECORD(Gate->Header.WaitListHead.Flink, - KWAIT_BLOCK, - WaitListEntry); - - /* Remove it */ - RemoveEntryList(&WaitBlock->WaitListEntry); - - /* Get the Associated thread */ - WaitThread = WaitBlock->Thread; - - /* Increment the Queue's active threads */ - if (WaitThread->Queue) WaitThread->Queue->CurrentCount++; - - /* FIXME: This isn't really correct!!! */ - KiUnwaitThread(WaitThread, WaitStatus, EVENT_INCREMENT); - } - - /* Exit the dispatcher */ - KiExitDispatcher(OldIrql); + /* If we got here, then there's no rescheduling. */ + KiReleaseDispatcherObject(&Gate->Header); + KeLowerIrql(OldIrql); } /* EOF */