diff --git a/reactos/ntoskrnl/include/internal/ke_x.h b/reactos/ntoskrnl/include/internal/ke_x.h index 5a5634cc6af..c615c659563 100644 --- a/reactos/ntoskrnl/include/internal/ke_x.h +++ b/reactos/ntoskrnl/include/internal/ke_x.h @@ -292,6 +292,71 @@ KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime, } \ } +// +// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL +// +FORCEINLINE +VOID +KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock) +{ +#ifdef CONFIG_SMP + for (;;) + { + /* Try to acquire it */ + if (InterlockedBitTestAndSet((PLONG)SpinLock, 0)) + { + /* Value changed... wait until it's locked */ + while (*(volatile KSPIN_LOCK *)SpinLock == 1) + { +#ifdef DBG + /* On debug builds, we use a much slower but useful routine */ + Kii386SpinOnSpinLock(SpinLock, 5); +#else + /* Otherwise, just yield and keep looping */ + YieldProcessor(); +#endif + } + } + else + { +#ifdef DBG + /* On debug builds, we OR in the KTHREAD */ + *SpinLock = KeGetCurrentThread() | 1; +#endif + /* All is well, break out */ + break; + } + } +#else + /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ + UNREFERENCED_PARAMETER(SpinLock); +#endif +} + +// +// Spinlock Release at IRQL >= DISPATCH_LEVEL +// +FORCEINLINE +VOID +KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock) +{ +#ifdef CONFIG_SMP +#ifdef DBG + /* Make sure that the threads match */ + if ((KeGetCurrentThread() | 1) != *SpinLock) + { + /* They don't, bugcheck */ + KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0); + } +#endif + /* Clear the lock */ + InterlockedAnd(SpinLock, 0); +#else + /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ + UNREFERENCED_PARAMETER(SpinLock); +#endif +} + // // Thread Scheduling Routines // diff --git a/reactos/ntoskrnl/ke/spinlock.c b/reactos/ntoskrnl/ke/spinlock.c index 456576b00c1..b78f5f12579 100644 --- a/reactos/ntoskrnl/ke/spinlock.c +++ b/reactos/ntoskrnl/ke/spinlock.c @@ -1,197 +1,21 @@ /* - * COPYRIGHT: See COPYING in the top level directory * PROJECT: ReactOS Kernel + * LICENSE: GPL - See COPYING in the top level directory * FILE: ntoskrnl/ke/spinlock.c - * PURPOSE: Implements spinlocks - * PROGRAMMERS: Alex Ionescu (alex@relsoft.net) - * David Welch (welch@cwcom.net) + * PURPOSE: Spinlock and Queued Spinlock Support + * PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org) */ -/* INCLUDES ****************************************************************/ +/* INCLUDES ******************************************************************/ #include #define NDEBUG #include -#undef KefAcquireSpinLockAtDpcLevel -#undef KeAcquireSpinLockAtDpcLevel -#undef KefReleaseSpinLockFromDpcLevel -#undef KeReleaseSpinLockFromDpcLevel - #define LQ_WAIT 1 #define LQ_OWN 2 -/* FUNCTIONS ***************************************************************/ - -/* - * @implemented - * - * FUNCTION: Synchronizes the execution of a given routine with the ISR - * of a given interrupt object - * ARGUMENTS: - * Interrupt = Interrupt object to synchronize with - * SynchronizeRoutine = Routine to call whose execution is - * synchronized with the ISR - * SynchronizeContext = Parameter to pass to the synchronized routine - * RETURNS: TRUE if the operation succeeded - */ -BOOLEAN -STDCALL -KeSynchronizeExecution(PKINTERRUPT Interrupt, - PKSYNCHRONIZE_ROUTINE SynchronizeRoutine, - PVOID SynchronizeContext) -{ - KIRQL OldIrql; - BOOLEAN Status; - - /* Raise IRQL and acquire lock on MP */ - OldIrql = KeAcquireInterruptSpinLock(Interrupt); - - /* Call the routine */ - Status = SynchronizeRoutine(SynchronizeContext); - - /* Release lock and lower IRQL */ - KeReleaseInterruptSpinLock(Interrupt, OldIrql); - - /* Return routine status */ - return Status; -} - -/* - * @implemented - */ -KIRQL -STDCALL -KeAcquireInterruptSpinLock(IN PKINTERRUPT Interrupt) -{ - KIRQL OldIrql; - - /* Raise IRQL */ - KeRaiseIrql(Interrupt->SynchronizeIrql, &OldIrql); - - /* Acquire spinlock on MP */ - KiAcquireSpinLock(Interrupt->ActualLock); - return OldIrql; -} - -/* - * @implemented - * - * FUNCTION: Initalizes a spinlock - * ARGUMENTS: - * SpinLock = Caller supplied storage for the spinlock - */ -VOID -STDCALL -KeInitializeSpinLock(PKSPIN_LOCK SpinLock) -{ - *SpinLock = 0; -} - -/* - * @implemented - */ -VOID -FASTCALL -KefAcquireSpinLockAtDpcLevel(PKSPIN_LOCK SpinLock) -{ - ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); - KiAcquireSpinLock(SpinLock); -} - -/* - * @implemented - * - * FUNCTION: Acquires a spinlock when the caller is already running at - * dispatch level - * ARGUMENTS: - * SpinLock = Spinlock to acquire - */ -VOID -STDCALL -KeAcquireSpinLockAtDpcLevel (PKSPIN_LOCK SpinLock) -{ - KefAcquireSpinLockAtDpcLevel(SpinLock); -} - -/* - * @implemented - */ -VOID -FASTCALL -KefReleaseSpinLockFromDpcLevel(PKSPIN_LOCK SpinLock) -{ - ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); - KiReleaseSpinLock(SpinLock); -} - -/* - * @implemented - * - * FUNCTION: Releases a spinlock when the caller was running at dispatch - * level before acquiring it - * ARGUMENTS: - * SpinLock = Spinlock to release - */ -VOID -STDCALL -KeReleaseSpinLockFromDpcLevel (PKSPIN_LOCK SpinLock) -{ - KefReleaseSpinLockFromDpcLevel(SpinLock); -} - -/* - * @implemented - */ -VOID -FASTCALL -KiAcquireSpinLock(PKSPIN_LOCK SpinLock) -{ -#ifdef CONFIG_SMP - for (;;) - { - /* Try to acquire it */ - if (InterlockedBitTestAndSet((PLONG)SpinLock, 0)) - { - /* Value changed... wait until it's locked */ - while (*(volatile KSPIN_LOCK *)SpinLock == 1) YieldProcessor(); - } - else - { - /* All is well, break out */ - break; - } - } -#endif /* CONFIG_SMP */ -} - -/* - * @implemented - */ -VOID -STDCALL -KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt, - IN KIRQL OldIrql) -{ - /* Release lock on MP */ - KiReleaseSpinLock(Interrupt->ActualLock); - - /* Lower IRQL */ - KeLowerIrql(OldIrql); -} - -/* - * @implemented - */ -VOID -FASTCALL -KiReleaseSpinLock(PKSPIN_LOCK SpinLock) -{ -#ifdef CONFIG_SMP - /* Simply clear it */ - *SpinLock = 0; -#endif -} +/* PRIVATE FUNCTIONS *********************************************************/ VOID FASTCALL @@ -260,6 +84,87 @@ KeReleaseQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle) #endif } +/* PUBLIC FUNCTIONS **********************************************************/ + +/* + * @implemented + */ +VOID +NTAPI +KeInitializeSpinLock(IN PKSPIN_LOCK SpinLock) +{ + /* Clear it */ + *SpinLock = 0; +} + +/* + * @implemented + */ +#undef KeAcquireSpinLockAtDpcLevel +VOID +NTAPI +KeAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock) +{ + /* Do the inlined function */ + KxAcquireSpinLock(SpinLock); +} + +/* + * @implemented + */ +#undef KeReleaseSpinLockFromDpcLevel +VOID +NTAPI +KeReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock) +{ + /* Do the lined function */ + KxReleaseSpinLock(SpinLock); +} + +/* + * @implemented + */ +VOID +FASTCALL +KefAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock) +{ + /* Do the inlined function */ + KxAcquireSpinLock(SpinLock); +} + +/* + * @implemented + */ +VOID +FASTCALL +KefReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock) +{ + /* Do the lined function */ + KxReleaseSpinLock(SpinLock); +} + +/* + * @implemented + */ +VOID +FASTCALL +KiAcquireSpinLock(IN PKSPIN_LOCK SpinLock) +{ + /* Do the inlined function */ + KxAcquireSpinLock(SpinLock); +} + +/* + * @implemented + */ +VOID +FASTCALL +KiReleaseSpinLock(IN PKSPIN_LOCK SpinLock) +{ + /* Do the lined function */ + KxReleaseSpinLock(SpinLock); +} + /* * @implemented */ @@ -268,11 +173,13 @@ FASTCALL KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle) { +#ifdef CONFIG_SMP /* Set it up properly */ LockHandle->LockQueue.Next = NULL; LockHandle->LockQueue.Lock = SpinLock; KeAcquireQueuedSpinLockAtDpcLevel((PKLOCK_QUEUE_HANDLE) &LockHandle->LockQueue.Next); +#endif } /* @@ -282,9 +189,68 @@ VOID FASTCALL KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle) { +#ifdef CONFIG_SMP /* Call the internal function */ KeReleaseQueuedSpinLockFromDpcLevel((PKLOCK_QUEUE_HANDLE) &LockHandle->LockQueue.Next); +#endif +} + +/* + * @implemented + */ +KIRQL +NTAPI +KeAcquireInterruptSpinLock(IN PKINTERRUPT Interrupt) +{ + KIRQL OldIrql; + + /* Raise IRQL */ + KeRaiseIrql(Interrupt->SynchronizeIrql, &OldIrql); + + /* Acquire spinlock on MP */ + KefAcquireSpinLockAtDpcLevel(Interrupt->ActualLock); + return OldIrql; +} + +/* + * @implemented + */ +BOOLEAN +NTAPI +KeSynchronizeExecution(IN PKINTERRUPT Interrupt, + IN PKSYNCHRONIZE_ROUTINE SynchronizeRoutine, + IN PVOID SynchronizeContext) +{ + KIRQL OldIrql; + BOOLEAN Status; + + /* Raise IRQL and acquire lock on MP */ + OldIrql = KeAcquireInterruptSpinLock(Interrupt); + + /* Call the routine */ + Status = SynchronizeRoutine(SynchronizeContext); + + /* Release lock and lower IRQL */ + KeReleaseInterruptSpinLock(Interrupt, OldIrql); + + /* Return routine status */ + return Status; +} + +/* + * @implemented + */ +VOID +STDCALL +KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt, + IN KIRQL OldIrql) +{ + /* Release lock on MP */ + KefReleaseSpinLock(Interrupt->ActualLock); + + /* Lower IRQL */ + KeLowerIrql(OldIrql); } /* EOF */