diff --git a/reactos/ntoskrnl/ex/fmutex.c b/reactos/ntoskrnl/ex/fmutex.c index 1445feae69d..5f1743b8d7f 100644 --- a/reactos/ntoskrnl/ex/fmutex.c +++ b/reactos/ntoskrnl/ex/fmutex.c @@ -12,61 +12,14 @@ #define NDEBUG #include -VOID -FASTCALL -KiAcquireFastMutex( - IN PFAST_MUTEX FastMutex -); - -/* PRIVATE FUNCTIONS *********************************************************/ - -FORCEINLINE -VOID -ExiAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) -{ - PKTHREAD Thread = KeGetCurrentThread(); - - DPRINT("Sanity print: %d %d %p\n", - KeGetCurrentIrql(), Thread->CombinedApcDisable, Thread->Teb); - - /* Sanity check */ - ASSERT((KeGetCurrentIrql() == APC_LEVEL) || - (Thread->CombinedApcDisable != 0) || - (Thread->Teb == NULL) || - (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); - ASSERT(FastMutex->Owner != Thread); - - /* Decrease the count */ - if (InterlockedDecrement(&FastMutex->Count)) - { - /* Someone is still holding it, use slow path */ - KiAcquireFastMutex(FastMutex); - } - - /* Set the owner */ - FastMutex->Owner = Thread; -} - -FORCEINLINE -VOID -ExiReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) -{ - ASSERT((KeGetCurrentIrql() == APC_LEVEL) || - (KeGetCurrentThread()->CombinedApcDisable != 0) || - (KeGetCurrentThread()->Teb == NULL) || - (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); - ASSERT(FastMutex->Owner == KeGetCurrentThread()); - - /* Erase the owner */ - FastMutex->Owner = NULL; - - /* Increase the count */ - if (InterlockedIncrement(&FastMutex->Count) <= 0) - { - /* Someone was waiting for it, signal the waiter */ - KeSetEventBoostPriority(&FastMutex->Gate, NULL); - } -} +/* Undefine some macros we implement here */ +#undef ExEnterCriticalRegionAndAcquireFastMutexUnsafe +#undef ExReleaseFastMutexUnsafeAndLeaveCriticalRegion +#undef ExAcquireFastMutex +#undef ExReleaseFastMutex +#undef ExAcquireFastMutexUnsafe +#undef ExReleaseFastMutexUnsafe +#undef ExTryToAcquireFastMutex /* PUBLIC FUNCTIONS **********************************************************/ @@ -77,11 +30,8 @@ VOID FASTCALL ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) { - /* Enter the Critical Region */ - KeEnterCriticalRegion(); - - /* Acquire the mutex unsafely */ - ExiAcquireFastMutexUnsafe(FastMutex); + /* Call the inline */ + _ExEnterCriticalRegionAndAcquireFastMutexUnsafe(FastMutex); } /* @@ -91,11 +41,8 @@ VOID FASTCALL ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) { - /* Release the mutex unsafely */ - ExiReleaseFastMutexUnsafe(FastMutex); - - /* Leave the critical region */ - KeLeaveCriticalRegion(); + /* Call the inline */ + _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(FastMutex); } /* @@ -105,22 +52,8 @@ VOID FASTCALL ExAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) { - KIRQL OldIrql; - ASSERT(KeGetCurrentIrql() <= APC_LEVEL); - - /* Raise IRQL to APC */ - KeRaiseIrql(APC_LEVEL, &OldIrql); - - /* Decrease the count */ - if (InterlockedDecrement(&FastMutex->Count) != 0) - { - /* Someone is still holding it, use slow path */ - KiAcquireFastMutex(FastMutex); - } - - /* Set the owner and IRQL */ - FastMutex->Owner = KeGetCurrentThread(); - FastMutex->OldIrql = OldIrql; + /* Call the inline */ + _ExAcquireFastMutex(FastMutex); } /* @@ -130,22 +63,8 @@ VOID FASTCALL ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) { - KIRQL OldIrql; - ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); - - /* Erase the owner */ - FastMutex->Owner = NULL; - OldIrql = (KIRQL)FastMutex->OldIrql; - - /* Increase the count */ - if (InterlockedIncrement(&FastMutex->Count) <= 0) - { - /* Someone was waiting for it, signal the waiter */ - KeSetEventBoostPriority(&FastMutex->Gate, IO_NO_INCREMENT); - } - - /* Lower IRQL back */ - KeLowerIrql(OldIrql); + /* Call the inline */ + _ExReleaseFastMutex(FastMutex); } /* @@ -156,7 +75,7 @@ FASTCALL ExAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) { /* Acquire the mutex unsafely */ - ExiAcquireFastMutexUnsafe(FastMutex); + _ExAcquireFastMutexUnsafe(FastMutex); } /* @@ -167,7 +86,7 @@ FASTCALL ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) { /* Release the mutex unsafely */ - ExiReleaseFastMutexUnsafe(FastMutex); + _ExReleaseFastMutexUnsafe(FastMutex); } /* @@ -177,26 +96,8 @@ BOOLEAN FASTCALL ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) { - KIRQL OldIrql; - ASSERT(KeGetCurrentIrql() <= APC_LEVEL); - - /* Raise to APC_LEVEL */ - KeRaiseIrql(APC_LEVEL, &OldIrql); - - /* Check if we can quickly acquire it */ - if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) - { - /* We have, set us as owners */ - FastMutex->Owner = KeGetCurrentThread(); - FastMutex->OldIrql = OldIrql; - return TRUE; - } - else - { - /* Acquire attempt failed */ - KeLowerIrql(OldIrql); - return FALSE; - } + /* Call the inline */ + return _ExTryToAcquireFastMutex(FastMutex); } /* EOF */ diff --git a/reactos/ntoskrnl/include/internal/ex.h b/reactos/ntoskrnl/include/internal/ex.h index d7b7b4188e5..7051b3b1db5 100644 --- a/reactos/ntoskrnl/include/internal/ex.h +++ b/reactos/ntoskrnl/include/internal/ex.h @@ -994,6 +994,146 @@ ExReleasePushLock(PEX_PUSH_LOCK PushLock) } } +/* FAST MUTEX INLINES *********************************************************/ + +FORCEINLINE +VOID +_ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex) +{ + PKTHREAD Thread = KeGetCurrentThread(); + + /* Sanity check */ + ASSERT((KeGetCurrentIrql() == APC_LEVEL) || + (Thread->CombinedApcDisable != 0) || + (Thread->Teb == NULL) || + (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); + ASSERT(FastMutex->Owner != Thread); + + /* Decrease the count */ + if (InterlockedDecrement(&FastMutex->Count)) + { + /* Someone is still holding it, use slow path */ + KiAcquireFastMutex(FastMutex); + } + + /* Set the owner */ + FastMutex->Owner = Thread; +} + +FORCEINLINE +VOID +_ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) +{ + ASSERT((KeGetCurrentIrql() == APC_LEVEL) || + (KeGetCurrentThread()->CombinedApcDisable != 0) || + (KeGetCurrentThread()->Teb == NULL) || + (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); + ASSERT(FastMutex->Owner == KeGetCurrentThread()); + + /* Erase the owner */ + FastMutex->Owner = NULL; + + /* Increase the count */ + if (InterlockedIncrement(&FastMutex->Count) <= 0) + { + /* Someone was waiting for it, signal the waiter */ + KeSetEventBoostPriority(&FastMutex->Gate, NULL); + } +} + +FORCEINLINE +VOID +_ExAcquireFastMutex(IN PFAST_MUTEX FastMutex) +{ + KIRQL OldIrql; + ASSERT(KeGetCurrentIrql() <= APC_LEVEL); + + /* Raise IRQL to APC */ + KeRaiseIrql(APC_LEVEL, &OldIrql); + + /* Decrease the count */ + if (InterlockedDecrement(&FastMutex->Count)) + { + /* Someone is still holding it, use slow path */ + KiAcquireFastMutex(FastMutex); + } + + /* Set the owner and IRQL */ + FastMutex->Owner = KeGetCurrentThread(); + FastMutex->OldIrql = OldIrql; +} + +FORCEINLINE +VOID +_ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex) +{ + KIRQL OldIrql; + ASSERT(KeGetCurrentIrql() == APC_LEVEL); + + /* Erase the owner */ + FastMutex->Owner = NULL; + OldIrql = (KIRQL)FastMutex->OldIrql; + + /* Increase the count */ + if (InterlockedIncrement(&FastMutex->Count) <= 0) + { + /* Someone was waiting for it, signal the waiter */ + KeSetEventBoostPriority(&FastMutex->Gate, NULL); + } + + /* Lower IRQL back */ + KeLowerIrql(OldIrql); +} + +FORCEINLINE +BOOLEAN +_ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex) +{ + KIRQL OldIrql; + ASSERT(KeGetCurrentIrql() <= APC_LEVEL); + + /* Raise to APC_LEVEL */ + KeRaiseIrql(APC_LEVEL, &OldIrql); + + /* Check if we can quickly acquire it */ + if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1) + { + /* We have, set us as owners */ + FastMutex->Owner = KeGetCurrentThread(); + FastMutex->OldIrql = OldIrql; + return TRUE; + } + else + { + /* Acquire attempt failed */ + KeLowerIrql(OldIrql); + YieldProcessor(); + return FALSE; + } +} + +FORCEINLINE +VOID +_ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex) +{ + /* Enter the Critical Region */ + KeEnterCriticalRegion(); + + /* Acquire the mutex unsafely */ + _ExAcquireFastMutexUnsafe(FastMutex); +} + +FORCEINLINE +VOID +_ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex) +{ + /* Release the mutex unsafely */ + _ExReleaseFastMutexUnsafe(FastMutex); + + /* Leave the critical region */ + KeLeaveCriticalRegion(); +} + /* OTHER FUNCTIONS **********************************************************/ BOOLEAN diff --git a/reactos/ntoskrnl/include/internal/ke.h b/reactos/ntoskrnl/include/internal/ke.h index ca17c514122..bf05741b153 100644 --- a/reactos/ntoskrnl/include/internal/ke.h +++ b/reactos/ntoskrnl/include/internal/ke.h @@ -312,7 +312,15 @@ KiCompleteTimer( VOID FASTCALL -KiAcquireGuardedMutexContented(PKGUARDED_MUTEX GuardedMutex); +KiAcquireGuardedMutex( + IN OUT PKGUARDED_MUTEX GuardedMutex +); + +VOID +FASTCALL +KiAcquireFastMutex( + IN PFAST_MUTEX FastMutex +); /* gate.c **********************************************************************/ diff --git a/reactos/ntoskrnl/include/internal/ke_x.h b/reactos/ntoskrnl/include/internal/ke_x.h index dd81e3129e8..2564593daea 100644 --- a/reactos/ntoskrnl/include/internal/ke_x.h +++ b/reactos/ntoskrnl/include/internal/ke_x.h @@ -170,10 +170,6 @@ KeFlushProcessTb(VOID) } \ } -// -// TODO: Guarded Mutex Routines -// - // // Enters a Critical Region // @@ -1710,3 +1706,178 @@ KiComputeNewPriority(IN PKTHREAD Thread, /* Return the new priority */ return Priority; } + +// +// Guarded Mutex Routines +// +FORCEINLINE +VOID +_KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex) +{ + /* Setup the Initial Data */ + GuardedMutex->Count = GM_LOCK_BIT; + GuardedMutex->Owner = NULL; + GuardedMutex->Contention = 0; + + /* Initialize the Wait Gate */ + KeInitializeGate(&GuardedMutex->Gate); +} + +FORCEINLINE +VOID +_KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + PKTHREAD Thread = KeGetCurrentThread(); + + /* Sanity checks */ + ASSERT((KeGetCurrentIrql() == APC_LEVEL) || + (Thread->SpecialApcDisable < 0) || + (Thread->Teb == NULL) || + (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); + ASSERT(GuardedMutex->Owner != Thread); + + /* Remove the lock */ + if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) + { + /* The Guarded Mutex was already locked, enter contented case */ + KiAcquireGuardedMutex(GuardedMutex); + } + + /* Set the Owner */ + GuardedMutex->Owner = Thread; +} + +FORCEINLINE +VOID +_KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + LONG OldValue, NewValue; + + /* Sanity checks */ + ASSERT((KeGetCurrentIrql() == APC_LEVEL) || + (KeGetCurrentThread()->SpecialApcDisable < 0) || + (KeGetCurrentThread()->Teb == NULL) || + (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); + ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); + + /* Destroy the Owner */ + GuardedMutex->Owner = NULL; + + /* Add the Lock Bit */ + OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); + ASSERT((OldValue & GM_LOCK_BIT) == 0); + + /* Check if it was already locked, but not woken */ + if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) + { + /* Update the Oldvalue to what it should be now */ + OldValue += GM_LOCK_BIT; + + /* The mutex will be woken, minus one waiter */ + NewValue = OldValue + GM_LOCK_WAITER_WOKEN - + GM_LOCK_WAITER_INC; + + /* Remove the Woken bit */ + if (InterlockedCompareExchange(&GuardedMutex->Count, + NewValue, + OldValue) == OldValue) + { + /* Signal the Gate */ + KeSignalGateBoostPriority(&GuardedMutex->Gate); + } + } +} + +FORCEINLINE +VOID +_KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex) +{ + PKTHREAD Thread = KeGetCurrentThread(); + + /* Sanity checks */ + ASSERT(KeGetCurrentIrql() <= APC_LEVEL); + ASSERT(GuardedMutex->Owner != Thread); + + /* Disable Special APCs */ + KeEnterGuardedRegion(); + + /* Remove the lock */ + if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) + { + /* The Guarded Mutex was already locked, enter contented case */ + KiAcquireGuardedMutex(GuardedMutex); + } + + /* Set the Owner and Special APC Disable state */ + GuardedMutex->Owner = Thread; + GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; +} + +FORCEINLINE +VOID +_KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + LONG OldValue, NewValue; + + /* Sanity checks */ + ASSERT(KeGetCurrentIrql() <= APC_LEVEL); + ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); + ASSERT(KeGetCurrentThread()->SpecialApcDisable == + GuardedMutex->SpecialApcDisable); + + /* Destroy the Owner */ + GuardedMutex->Owner = NULL; + + /* Add the Lock Bit */ + OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); + ASSERT((OldValue & GM_LOCK_BIT) == 0); + + /* Check if it was already locked, but not woken */ + if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) + { + /* Update the Oldvalue to what it should be now */ + OldValue += GM_LOCK_BIT; + + /* The mutex will be woken, minus one waiter */ + NewValue = OldValue + GM_LOCK_WAITER_WOKEN - + GM_LOCK_WAITER_INC; + + /* Remove the Woken bit */ + if (InterlockedCompareExchange(&GuardedMutex->Count, + NewValue, + OldValue) == OldValue) + { + /* Signal the Gate */ + KeSignalGateBoostPriority(&GuardedMutex->Gate); + } + } + + /* Re-enable APCs */ + KeLeaveGuardedRegion(); +} + +FORCEINLINE +BOOLEAN +_KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + PKTHREAD Thread = KeGetCurrentThread(); + + /* Block APCs */ + KeEnterGuardedRegion(); + + /* Remove the lock */ + if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V)) + { + /* Re-enable APCs */ + KeLeaveGuardedRegion(); + YieldProcessor(); + + /* Return failure */ + return FALSE; + } + + /* Set the Owner and APC State */ + GuardedMutex->Owner = Thread; + GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; + return TRUE; +} diff --git a/reactos/ntoskrnl/include/internal/ntoskrnl.h b/reactos/ntoskrnl/include/internal/ntoskrnl.h index 8deb0616ec6..225e54c4453 100644 --- a/reactos/ntoskrnl/include/internal/ntoskrnl.h +++ b/reactos/ntoskrnl/include/internal/ntoskrnl.h @@ -42,6 +42,24 @@ #define InterlockedOr _InterlockedOr #define InterlockedAnd _InterlockedAnd +// +// Use inlined versions of fast/guarded mutex routines +// +#define ExEnterCriticalRegionAndAcquireFastMutexUnsafe _ExEnterCriticalRegionAndAcquireFastMutexUnsafe +#define ExReleaseFastMutexUnsafeAndLeaveCriticalRegion _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion +#define ExAcquireFastMutex _ExAcquireFastMutex +#define ExReleaseFastMutex _ExReleaseFastMutex +#define ExAcquireFastMutexUnsafe _ExAcquireFastMutexUnsafe +#define ExReleaseFastMutexUnsafe _ExReleaseFastMutexUnsafe +#define ExTryToAcquireFastMutex _ExTryToAcquireFastMutex + +#define KeInitializeGuardedMutex _KeInitializeGuardedMutex +#define KeAcquireGuardedMutex _KeAcquireGuardedMutex +#define KeReleaseGuardedMutex _KeReleaseGuardedMutex +#define KeAcquireGuardedMutexUnsafe _KeAcquireGuardedMutexUnsafe +#define KeReleaseGuardedMutexUnsafe _KeReleaseGuardedMutexUnsafe +#define KeTryToAcquireGuardedMutex _KeTryToAcquireGuardedMutex + #include "ke.h" #include "ob.h" #include "mm.h" diff --git a/reactos/ntoskrnl/ke/gmutex.c b/reactos/ntoskrnl/ke/gmutex.c index effd1445ea0..84cf01c8357 100644 --- a/reactos/ntoskrnl/ke/gmutex.c +++ b/reactos/ntoskrnl/ke/gmutex.c @@ -13,128 +13,13 @@ #define NDEBUG #include -/* PRIVATE FUNCTIONS *********************************************************/ - -VOID -FASTCALL -KiAcquireGuardedMutexContented(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - ULONG BitsToRemove, BitsToAdd; - LONG OldValue, NewValue; - - /* Increase the contention count */ - GuardedMutex->Contention++; - - /* Start by unlocking the Guarded Mutex */ - BitsToRemove = GM_LOCK_BIT; - BitsToAdd = GM_LOCK_WAITER_INC; - - /* Start change loop */ - for (;;) - { - /* Loop sanity checks */ - ASSERT((BitsToRemove == GM_LOCK_BIT) || - (BitsToRemove == (GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN))); - ASSERT((BitsToAdd == GM_LOCK_WAITER_INC) || - (BitsToAdd == GM_LOCK_WAITER_WOKEN)); - - /* Get the Count Bits */ - OldValue = GuardedMutex->Count; - - /* Start internal bit change loop */ - for (;;) - { - /* Check if the Guarded Mutex is locked */ - if (OldValue & GM_LOCK_BIT) - { - /* Sanity check */ - ASSERT((BitsToRemove == GM_LOCK_BIT) || - ((OldValue & GM_LOCK_WAITER_WOKEN) != 0)); - - /* Unlock it by removing the Lock Bit */ - NewValue = OldValue ^ BitsToRemove; - NewValue = InterlockedCompareExchange(&GuardedMutex->Count, - NewValue, - OldValue); - if (NewValue == OldValue) return; - } - else - { - /* The Guarded Mutex isn't locked, so simply set the bits */ - NewValue = OldValue + BitsToAdd; - NewValue = InterlockedCompareExchange(&GuardedMutex->Count, - NewValue, - OldValue); - if (NewValue == OldValue) break; - } - - /* Old value changed, loop again */ - OldValue = NewValue; - } - - /* Now we have to wait for it */ - KeWaitForGate(&GuardedMutex->Gate, WrGuardedMutex, KernelMode); - ASSERT((GuardedMutex->Count & GM_LOCK_WAITER_WOKEN) != 0); - - /* Ok, the wait is done, so set the new bits */ - BitsToRemove = GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN; - BitsToAdd = GM_LOCK_WAITER_WOKEN; - - /* We depend on these bits being just right */ - C_ASSERT((GM_LOCK_WAITER_WOKEN * 2) == GM_LOCK_WAITER_INC); - } -} - -FORCEINLINE -VOID -FASTCALL -KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - BOOLEAN OldBit; - - /* Remove the lock */ - OldBit = InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V); - if (!OldBit) - { - /* The Guarded Mutex was already locked, enter contented case */ - KiAcquireGuardedMutexContented(GuardedMutex); - } -} - -FORCEINLINE -VOID -FASTCALL -KiReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - LONG OldValue, NewValue; - - /* Destroy the Owner */ - GuardedMutex->Owner = NULL; - - /* Add the Lock Bit */ - OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT); - ASSERT((OldValue & GM_LOCK_BIT) == 0); - - /* Check if it was already locked, but not woken */ - if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN)) - { - /* Update the Oldvalue to what it should be now */ - OldValue += GM_LOCK_BIT; - - /* The mutex will be woken, minus one waiter */ - NewValue = OldValue + GM_LOCK_WAITER_WOKEN - - GM_LOCK_WAITER_INC; - - /* Remove the Woken bit */ - if (InterlockedCompareExchange(&GuardedMutex->Count, - NewValue, - OldValue) == OldValue) - { - /* Signal the Gate */ - KeSignalGateBoostPriority(&GuardedMutex->Gate); - } - } -} +/* Undefine some macros we implement here */ +#undef KeInitializeGuardedMutex +#undef KeAcquireGuardedMutex +#undef KeReleaseGuardedMutex +#undef KeAcquireGuardedMutexUnsafe +#undef KeReleaseGuardedMutexUnsafe +#undef KeTryToAcquireGuardedMutex /* PUBLIC FUNCTIONS **********************************************************/ @@ -145,54 +30,8 @@ VOID FASTCALL KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex) { - /* Setup the Initial Data */ - GuardedMutex->Count = GM_LOCK_BIT; - GuardedMutex->Owner = NULL; - GuardedMutex->Contention = 0; - - /* Initialize the Wait Gate */ - KeInitializeGate(&GuardedMutex->Gate); -} - -/* - * @implemented - */ -VOID -FASTCALL -KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - PKTHREAD Thread = KeGetCurrentThread(); - - /* Sanity checks */ - ASSERT((KeGetCurrentIrql() == APC_LEVEL) || - (Thread->SpecialApcDisable < 0) || - (Thread->Teb == NULL) || - (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); - ASSERT(GuardedMutex->Owner != Thread); - - /* Do the actual acquire */ - KiAcquireGuardedMutex(GuardedMutex); - - /* Set the Owner */ - GuardedMutex->Owner = Thread; -} - -/* - * @implemented - */ -VOID -FASTCALL -KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) -{ - /* Sanity checks */ - ASSERT((KeGetCurrentIrql() == APC_LEVEL) || - (KeGetCurrentThread()->SpecialApcDisable < 0) || - (KeGetCurrentThread()->Teb == NULL) || - (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START)); - ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); - - /* Release the mutex */ - KiReleaseGuardedMutex(GuardedMutex); + /* Call the inline */ + _KeInitializeGuardedMutex(GuardedMutex); } /* @@ -202,21 +41,8 @@ VOID FASTCALL KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex) { - PKTHREAD Thread = KeGetCurrentThread(); - - /* Sanity checks */ - ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); - ASSERT(GuardedMutex->Owner != Thread); - - /* Disable Special APCs */ - KeEnterGuardedRegion(); - - /* Do the actual acquire */ - KiAcquireGuardedMutex(GuardedMutex); - - /* Set the Owner and Special APC Disable state */ - GuardedMutex->Owner = Thread; - GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; + /* Call the inline */ + _KeAcquireGuardedMutex(GuardedMutex); } /* @@ -226,17 +52,30 @@ VOID FASTCALL KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) { - /* Sanity checks */ - ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL); - ASSERT(GuardedMutex->Owner == KeGetCurrentThread()); - ASSERT(GuardedMutex->SpecialApcDisable == - KeGetCurrentThread()->SpecialApcDisable); + /* Call the inline */ + _KeReleaseGuardedMutex(GuardedMutex); +} - /* Release the mutex */ - KiReleaseGuardedMutex(GuardedMutex); +/* + * @implemented + */ +VOID +FASTCALL +KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + /* Call the inline */ + _KeAcquireGuardedMutexUnsafe(GuardedMutex); +} - /* Re-enable APCs */ - KeLeaveGuardedRegion(); +/* + * @implemented + */ +VOID +FASTCALL +KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + /* Call the inline */ + _KeReleaseGuardedMutexUnsafe(GuardedMutex); } /* @@ -246,28 +85,8 @@ BOOLEAN FASTCALL KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) { - PKTHREAD Thread = KeGetCurrentThread(); - BOOLEAN OldBit; - - /* Block APCs */ - KeEnterGuardedRegion(); - - /* Remove the lock */ - OldBit = InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V); - if (!OldBit) - { - /* Re-enable APCs */ - KeLeaveGuardedRegion(); - YieldProcessor(); - - /* Return failure */ - return FALSE; - } - - /* Set the Owner and APC State */ - GuardedMutex->Owner = Thread; - GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable; - return TRUE; + /* Call the inline */ + return _KeTryToAcquireGuardedMutex(GuardedMutex); } /** diff --git a/reactos/ntoskrnl/ke/wait.c b/reactos/ntoskrnl/ke/wait.c index e0713e56e90..86bdbc11e1e 100644 --- a/reactos/ntoskrnl/ke/wait.c +++ b/reactos/ntoskrnl/ke/wait.c @@ -117,6 +117,76 @@ KiAcquireFastMutex(IN PFAST_MUTEX FastMutex) NULL); } +VOID +FASTCALL +KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex) +{ + ULONG BitsToRemove, BitsToAdd; + LONG OldValue, NewValue; + + /* Increase the contention count */ + GuardedMutex->Contention++; + + /* Start by unlocking the Guarded Mutex */ + BitsToRemove = GM_LOCK_BIT; + BitsToAdd = GM_LOCK_WAITER_INC; + + /* Start change loop */ + for (;;) + { + /* Loop sanity checks */ + ASSERT((BitsToRemove == GM_LOCK_BIT) || + (BitsToRemove == (GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN))); + ASSERT((BitsToAdd == GM_LOCK_WAITER_INC) || + (BitsToAdd == GM_LOCK_WAITER_WOKEN)); + + /* Get the Count Bits */ + OldValue = GuardedMutex->Count; + + /* Start internal bit change loop */ + for (;;) + { + /* Check if the Guarded Mutex is locked */ + if (OldValue & GM_LOCK_BIT) + { + /* Sanity check */ + ASSERT((BitsToRemove == GM_LOCK_BIT) || + ((OldValue & GM_LOCK_WAITER_WOKEN) != 0)); + + /* Unlock it by removing the Lock Bit */ + NewValue = OldValue ^ BitsToRemove; + NewValue = InterlockedCompareExchange(&GuardedMutex->Count, + NewValue, + OldValue); + if (NewValue == OldValue) return; + } + else + { + /* The Guarded Mutex isn't locked, so simply set the bits */ + NewValue = OldValue + BitsToAdd; + NewValue = InterlockedCompareExchange(&GuardedMutex->Count, + NewValue, + OldValue); + if (NewValue == OldValue) break; + } + + /* Old value changed, loop again */ + OldValue = NewValue; + } + + /* Now we have to wait for it */ + KeWaitForGate(&GuardedMutex->Gate, WrGuardedMutex, KernelMode); + ASSERT((GuardedMutex->Count & GM_LOCK_WAITER_WOKEN) != 0); + + /* Ok, the wait is done, so set the new bits */ + BitsToRemove = GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN; + BitsToAdd = GM_LOCK_WAITER_WOKEN; + + /* We depend on these bits being just right */ + C_ASSERT((GM_LOCK_WAITER_WOKEN * 2) == GM_LOCK_WAITER_INC); + } +} + // // This routine exits the dispatcher after a compatible operation and // swaps the context to the next scheduled thread on the current CPU if