- Inline Guarded Mutex and Fast Mutex implementations when called from within the kernel

svn path=/trunk/; revision=39214

reactos/ntoskrnl/ex/fmutex.c

@@ -12,61 +12,14 @@
 #define NDEBUG
 #include <debug.h>
 
-VOID
+/* Undefine some macros we implement here */
-FASTCALL
+#undef ExEnterCriticalRegionAndAcquireFastMutexUnsafe
-KiAcquireFastMutex(
+#undef ExReleaseFastMutexUnsafeAndLeaveCriticalRegion
-    IN PFAST_MUTEX FastMutex
+#undef ExAcquireFastMutex
-);
+#undef ExReleaseFastMutex
-
+#undef ExAcquireFastMutexUnsafe
-/* PRIVATE FUNCTIONS *********************************************************/
+#undef ExReleaseFastMutexUnsafe
-
+#undef ExTryToAcquireFastMutex
-FORCEINLINE
-VOID
-ExiAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
-{
-    PKTHREAD Thread = KeGetCurrentThread();
-
-    DPRINT("Sanity print: %d %d %p\n",
-            KeGetCurrentIrql(), Thread->CombinedApcDisable, Thread->Teb);
-
-    /* Sanity check */
-    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
-           (Thread->CombinedApcDisable != 0) ||
-           (Thread->Teb == NULL) ||
-           (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
-    ASSERT(FastMutex->Owner != Thread);
-
-    /* Decrease the count */
-    if (InterlockedDecrement(&FastMutex->Count))
-    {
-        /* Someone is still holding it, use slow path */
-        KiAcquireFastMutex(FastMutex);
-    }
-
-    /* Set the owner */
-    FastMutex->Owner = Thread;
-}
-
-FORCEINLINE
-VOID
-ExiReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
-{
-    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
-           (KeGetCurrentThread()->CombinedApcDisable != 0) ||
-           (KeGetCurrentThread()->Teb == NULL) ||
-           (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
-    ASSERT(FastMutex->Owner == KeGetCurrentThread());
-
-    /* Erase the owner */
-    FastMutex->Owner = NULL;
-
-    /* Increase the count */
-    if (InterlockedIncrement(&FastMutex->Count) <= 0)
-    {
-        /* Someone was waiting for it, signal the waiter */
-        KeSetEventBoostPriority(&FastMutex->Gate, NULL);
-    }
-}
 
 /* PUBLIC FUNCTIONS **********************************************************/
 
@@ -77,11 +30,8 @@ VOID
 FASTCALL
 ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
 {
-    /* Enter the Critical Region */
+    /* Call the inline */
-    KeEnterCriticalRegion();
+	_ExEnterCriticalRegionAndAcquireFastMutexUnsafe(FastMutex);
-
-    /* Acquire the mutex unsafely */
-    ExiAcquireFastMutexUnsafe(FastMutex);
 }
 
 /*
@@ -91,11 +41,8 @@ VOID
 FASTCALL
 ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
 {
-    /* Release the mutex unsafely */
+    /* Call the inline */
-    ExiReleaseFastMutexUnsafe(FastMutex);
+	_ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(FastMutex);
-
-    /* Leave the critical region */
-    KeLeaveCriticalRegion();
 }
 
 /*
@@ -105,22 +52,8 @@ VOID
 FASTCALL
 ExAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
 {
-    KIRQL OldIrql;
+    /* Call the inline */
-    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+    _ExAcquireFastMutex(FastMutex);
-
-    /* Raise IRQL to APC */
-    KeRaiseIrql(APC_LEVEL, &OldIrql);
-
-    /* Decrease the count */
-    if (InterlockedDecrement(&FastMutex->Count) != 0)
-    {
-        /* Someone is still holding it, use slow path */
-        KiAcquireFastMutex(FastMutex);
-    }
-
-    /* Set the owner and IRQL */
-    FastMutex->Owner = KeGetCurrentThread();
-    FastMutex->OldIrql = OldIrql;
 }
 
 /*
@@ -130,22 +63,8 @@ VOID
 FASTCALL
 ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
 {
-    KIRQL OldIrql;
+    /* Call the inline */
-    ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL);
+    _ExReleaseFastMutex(FastMutex);
-
-    /* Erase the owner */
-    FastMutex->Owner = NULL;
-    OldIrql = (KIRQL)FastMutex->OldIrql;
-
-    /* Increase the count */
-    if (InterlockedIncrement(&FastMutex->Count) <= 0)
-    {
-        /* Someone was waiting for it, signal the waiter */
-        KeSetEventBoostPriority(&FastMutex->Gate, IO_NO_INCREMENT);
-    }
-
-    /* Lower IRQL back */
-    KeLowerIrql(OldIrql);
 }
 
 /*
@@ -156,7 +75,7 @@ FASTCALL
 ExAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
 {
     /* Acquire the mutex unsafely */
-    ExiAcquireFastMutexUnsafe(FastMutex);
+    _ExAcquireFastMutexUnsafe(FastMutex);
 }
 
 /*
@@ -167,7 +86,7 @@ FASTCALL
 ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
 {
     /* Release the mutex unsafely */
-    ExiReleaseFastMutexUnsafe(FastMutex);
+    _ExReleaseFastMutexUnsafe(FastMutex);
 }
 
 /*
@@ -177,26 +96,8 @@ BOOLEAN
 FASTCALL
 ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
 {
-    KIRQL OldIrql;
+    /* Call the inline */
-    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+    return _ExTryToAcquireFastMutex(FastMutex);
-
-    /* Raise to APC_LEVEL */
-    KeRaiseIrql(APC_LEVEL, &OldIrql);
-
-    /* Check if we can quickly acquire it */
-    if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
-    {
-        /* We have, set us as owners */
-        FastMutex->Owner = KeGetCurrentThread();
-        FastMutex->OldIrql = OldIrql;
-        return TRUE;
-    }
-    else
-    {
-        /* Acquire attempt failed */
-        KeLowerIrql(OldIrql);
-        return FALSE;
-    }
 }
 
 /* EOF */

reactos/ntoskrnl/include/internal/ex.h

@@ -994,6 +994,146 @@ ExReleasePushLock(PEX_PUSH_LOCK PushLock)
     }
 }
 
+/* FAST MUTEX INLINES *********************************************************/
+
+FORCEINLINE
+VOID
+_ExAcquireFastMutexUnsafe(IN PFAST_MUTEX FastMutex)
+{
+    PKTHREAD Thread = KeGetCurrentThread();
+    
+    /* Sanity check */
+    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
+           (Thread->CombinedApcDisable != 0) ||
+           (Thread->Teb == NULL) ||
+           (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
+    ASSERT(FastMutex->Owner != Thread);
+    
+    /* Decrease the count */
+    if (InterlockedDecrement(&FastMutex->Count))
+    {
+        /* Someone is still holding it, use slow path */
+        KiAcquireFastMutex(FastMutex);
+    }
+    
+    /* Set the owner */
+    FastMutex->Owner = Thread;
+}
+
+FORCEINLINE
+VOID
+_ExReleaseFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
+{
+    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
+           (KeGetCurrentThread()->CombinedApcDisable != 0) ||
+           (KeGetCurrentThread()->Teb == NULL) ||
+           (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
+    ASSERT(FastMutex->Owner == KeGetCurrentThread());
+    
+    /* Erase the owner */
+    FastMutex->Owner = NULL;
+    
+    /* Increase the count */
+    if (InterlockedIncrement(&FastMutex->Count) <= 0)
+    {
+        /* Someone was waiting for it, signal the waiter */
+        KeSetEventBoostPriority(&FastMutex->Gate, NULL);
+    }
+}
+
+FORCEINLINE
+VOID
+_ExAcquireFastMutex(IN PFAST_MUTEX FastMutex)
+{
+    KIRQL OldIrql;
+    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+    
+    /* Raise IRQL to APC */
+    KeRaiseIrql(APC_LEVEL, &OldIrql);
+    
+    /* Decrease the count */
+    if (InterlockedDecrement(&FastMutex->Count))
+    {
+        /* Someone is still holding it, use slow path */
+        KiAcquireFastMutex(FastMutex);
+    }
+    
+    /* Set the owner and IRQL */
+    FastMutex->Owner = KeGetCurrentThread();
+    FastMutex->OldIrql = OldIrql;
+}
+
+FORCEINLINE
+VOID
+_ExReleaseFastMutex(IN OUT PFAST_MUTEX FastMutex)
+{
+    KIRQL OldIrql;
+    ASSERT(KeGetCurrentIrql() == APC_LEVEL);
+    
+    /* Erase the owner */
+    FastMutex->Owner = NULL;
+    OldIrql = (KIRQL)FastMutex->OldIrql;
+    
+    /* Increase the count */
+    if (InterlockedIncrement(&FastMutex->Count) <= 0)
+    {
+        /* Someone was waiting for it, signal the waiter */
+        KeSetEventBoostPriority(&FastMutex->Gate, NULL);
+    }
+    
+    /* Lower IRQL back */
+    KeLowerIrql(OldIrql);
+}
+
+FORCEINLINE
+BOOLEAN
+_ExTryToAcquireFastMutex(IN OUT PFAST_MUTEX FastMutex)
+{
+    KIRQL OldIrql;
+    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+    
+    /* Raise to APC_LEVEL */
+    KeRaiseIrql(APC_LEVEL, &OldIrql);
+    
+    /* Check if we can quickly acquire it */
+    if (InterlockedCompareExchange(&FastMutex->Count, 0, 1) == 1)
+    {
+        /* We have, set us as owners */
+        FastMutex->Owner = KeGetCurrentThread();
+        FastMutex->OldIrql = OldIrql;
+        return TRUE;
+    }
+    else
+    {
+        /* Acquire attempt failed */
+        KeLowerIrql(OldIrql);
+        YieldProcessor();
+        return FALSE;
+    }
+}
+
+FORCEINLINE
+VOID
+_ExEnterCriticalRegionAndAcquireFastMutexUnsafe(IN OUT PFAST_MUTEX FastMutex)
+{
+    /* Enter the Critical Region */
+    KeEnterCriticalRegion();
+
+    /* Acquire the mutex unsafely */
+    _ExAcquireFastMutexUnsafe(FastMutex);
+}
+
+FORCEINLINE
+VOID
+_ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(IN OUT PFAST_MUTEX FastMutex)
+{
+    /* Release the mutex unsafely */
+    _ExReleaseFastMutexUnsafe(FastMutex);
+
+    /* Leave the critical region */
+    KeLeaveCriticalRegion();
+}
+
 /* OTHER FUNCTIONS **********************************************************/
 
 BOOLEAN

reactos/ntoskrnl/include/internal/ke.h

@@ -312,7 +312,15 @@ KiCompleteTimer(
 
 VOID
 FASTCALL
-KiAcquireGuardedMutexContented(PKGUARDED_MUTEX GuardedMutex);
+KiAcquireGuardedMutex(
+    IN OUT PKGUARDED_MUTEX GuardedMutex
+);
+
+VOID
+FASTCALL
+KiAcquireFastMutex(
+    IN PFAST_MUTEX FastMutex
+);
 
 /* gate.c **********************************************************************/
 

reactos/ntoskrnl/include/internal/ke_x.h

@@ -170,10 +170,6 @@ KeFlushProcessTb(VOID)
     }                                                                       \
 }
 
-//
-// TODO: Guarded Mutex Routines
-//
-
 //
 // Enters a Critical Region
 //
@@ -1710,3 +1706,178 @@ KiComputeNewPriority(IN PKTHREAD Thread,
     /* Return the new priority */
     return Priority;
 }
+
+//
+// Guarded Mutex Routines
+//
+FORCEINLINE
+VOID
+_KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    /* Setup the Initial Data */
+    GuardedMutex->Count = GM_LOCK_BIT;
+    GuardedMutex->Owner = NULL;
+    GuardedMutex->Contention = 0;
+    
+    /* Initialize the Wait Gate */
+    KeInitializeGate(&GuardedMutex->Gate);
+}
+
+FORCEINLINE
+VOID
+_KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    PKTHREAD Thread = KeGetCurrentThread();
+    
+    /* Sanity checks */
+    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
+           (Thread->SpecialApcDisable < 0) ||
+           (Thread->Teb == NULL) ||
+           (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
+    ASSERT(GuardedMutex->Owner != Thread);
+    
+    /* Remove the lock */
+    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
+    {
+        /* The Guarded Mutex was already locked, enter contented case */
+        KiAcquireGuardedMutex(GuardedMutex);
+    }
+    
+    /* Set the Owner */
+    GuardedMutex->Owner = Thread;
+}
+
+FORCEINLINE
+VOID
+_KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    LONG OldValue, NewValue;
+    
+    /* Sanity checks */
+    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
+           (KeGetCurrentThread()->SpecialApcDisable < 0) ||
+           (KeGetCurrentThread()->Teb == NULL) ||
+           (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
+    ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
+    
+    /* Destroy the Owner */
+    GuardedMutex->Owner = NULL;
+    
+    /* Add the Lock Bit */
+    OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
+    ASSERT((OldValue & GM_LOCK_BIT) == 0);
+    
+    /* Check if it was already locked, but not woken */
+    if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
+    {
+        /* Update the Oldvalue to what it should be now */
+        OldValue += GM_LOCK_BIT;
+
+        /* The mutex will be woken, minus one waiter */
+        NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
+            GM_LOCK_WAITER_INC;
+        
+        /* Remove the Woken bit */
+        if (InterlockedCompareExchange(&GuardedMutex->Count,
+                                       NewValue,
+                                       OldValue) == OldValue)
+        {
+            /* Signal the Gate */
+            KeSignalGateBoostPriority(&GuardedMutex->Gate);
+        }
+    }
+}
+
+FORCEINLINE
+VOID
+_KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
+{
+    PKTHREAD Thread = KeGetCurrentThread();
+    
+    /* Sanity checks */
+    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+    ASSERT(GuardedMutex->Owner != Thread);
+    
+    /* Disable Special APCs */
+    KeEnterGuardedRegion();
+    
+    /* Remove the lock */
+    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
+    {
+        /* The Guarded Mutex was already locked, enter contented case */
+        KiAcquireGuardedMutex(GuardedMutex);
+    }
+    
+    /* Set the Owner and Special APC Disable state */
+    GuardedMutex->Owner = Thread;
+    GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
+}
+
+FORCEINLINE
+VOID
+_KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    LONG OldValue, NewValue;
+    
+    /* Sanity checks */
+    ASSERT(KeGetCurrentIrql() <= APC_LEVEL);
+    ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
+    ASSERT(KeGetCurrentThread()->SpecialApcDisable ==
+           GuardedMutex->SpecialApcDisable);
+    
+    /* Destroy the Owner */
+    GuardedMutex->Owner = NULL;
+    
+    /* Add the Lock Bit */
+    OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
+    ASSERT((OldValue & GM_LOCK_BIT) == 0);
+    
+    /* Check if it was already locked, but not woken */
+    if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
+    {
+        /* Update the Oldvalue to what it should be now */
+        OldValue += GM_LOCK_BIT;
+
+        /* The mutex will be woken, minus one waiter */
+        NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
+            GM_LOCK_WAITER_INC;
+        
+        /* Remove the Woken bit */
+        if (InterlockedCompareExchange(&GuardedMutex->Count,
+                                       NewValue,
+                                       OldValue) == OldValue)
+        {
+            /* Signal the Gate */
+            KeSignalGateBoostPriority(&GuardedMutex->Gate);
+        }
+    }
+    
+    /* Re-enable APCs */
+    KeLeaveGuardedRegion();
+}
+
+FORCEINLINE
+BOOLEAN
+_KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    PKTHREAD Thread = KeGetCurrentThread();
+    
+    /* Block APCs */
+    KeEnterGuardedRegion();
+    
+    /* Remove the lock */
+    if (!InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V))
+    {
+        /* Re-enable APCs */
+        KeLeaveGuardedRegion();
+        YieldProcessor();
+        
+        /* Return failure */
+        return FALSE;
+    }
+    
+    /* Set the Owner and APC State */
+    GuardedMutex->Owner = Thread;
+    GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
+    return TRUE;
+}

reactos/ntoskrnl/include/internal/ntoskrnl.h

@@ -42,6 +42,24 @@
 #define InterlockedOr                _InterlockedOr
 #define InterlockedAnd               _InterlockedAnd
 
+//
+// Use inlined versions of fast/guarded mutex routines
+//
+#define ExEnterCriticalRegionAndAcquireFastMutexUnsafe _ExEnterCriticalRegionAndAcquireFastMutexUnsafe
+#define ExReleaseFastMutexUnsafeAndLeaveCriticalRegion _ExReleaseFastMutexUnsafeAndLeaveCriticalRegion
+#define ExAcquireFastMutex _ExAcquireFastMutex
+#define ExReleaseFastMutex _ExReleaseFastMutex
+#define ExAcquireFastMutexUnsafe _ExAcquireFastMutexUnsafe
+#define ExReleaseFastMutexUnsafe _ExReleaseFastMutexUnsafe
+#define ExTryToAcquireFastMutex _ExTryToAcquireFastMutex
+
+#define KeInitializeGuardedMutex _KeInitializeGuardedMutex
+#define KeAcquireGuardedMutex _KeAcquireGuardedMutex
+#define KeReleaseGuardedMutex _KeReleaseGuardedMutex
+#define KeAcquireGuardedMutexUnsafe _KeAcquireGuardedMutexUnsafe
+#define KeReleaseGuardedMutexUnsafe _KeReleaseGuardedMutexUnsafe
+#define KeTryToAcquireGuardedMutex _KeTryToAcquireGuardedMutex
+
 #include "ke.h"
 #include "ob.h"
 #include "mm.h"

reactos/ntoskrnl/ke/gmutex.c

@@ -13,128 +13,13 @@
 #define NDEBUG
 #include <debug.h>
 
-/* PRIVATE FUNCTIONS *********************************************************/
+/* Undefine some macros we implement here */
-
+#undef KeInitializeGuardedMutex
-VOID
+#undef KeAcquireGuardedMutex
-FASTCALL
+#undef KeReleaseGuardedMutex
-KiAcquireGuardedMutexContented(IN OUT PKGUARDED_MUTEX GuardedMutex)
+#undef KeAcquireGuardedMutexUnsafe
-{
+#undef KeReleaseGuardedMutexUnsafe
-    ULONG BitsToRemove, BitsToAdd;
+#undef KeTryToAcquireGuardedMutex
-    LONG OldValue, NewValue;
-
-    /* Increase the contention count */
-    GuardedMutex->Contention++;
-
-    /* Start by unlocking the Guarded Mutex */
-    BitsToRemove = GM_LOCK_BIT;
-    BitsToAdd = GM_LOCK_WAITER_INC;
-
-    /* Start change loop */
-    for (;;)
-    {
-        /* Loop sanity checks */
-        ASSERT((BitsToRemove == GM_LOCK_BIT) ||
-               (BitsToRemove == (GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN)));
-        ASSERT((BitsToAdd == GM_LOCK_WAITER_INC) ||
-               (BitsToAdd == GM_LOCK_WAITER_WOKEN));
-
-        /* Get the Count Bits */
-        OldValue = GuardedMutex->Count;
-
-        /* Start internal bit change loop */
-        for (;;)
-        {
-            /* Check if the Guarded Mutex is locked */
-            if (OldValue & GM_LOCK_BIT)
-            {
-                /* Sanity check */
-                ASSERT((BitsToRemove == GM_LOCK_BIT) ||
-                       ((OldValue & GM_LOCK_WAITER_WOKEN) != 0));
-
-                /* Unlock it by removing the Lock Bit */
-                NewValue = OldValue ^ BitsToRemove;
-                NewValue = InterlockedCompareExchange(&GuardedMutex->Count,
-                                                      NewValue,
-                                                      OldValue);
-                if (NewValue == OldValue) return;
-            }
-            else
-            {
-                /* The Guarded Mutex isn't locked, so simply set the bits */
-                NewValue = OldValue + BitsToAdd;
-                NewValue = InterlockedCompareExchange(&GuardedMutex->Count,
-                                                      NewValue,
-                                                      OldValue);
-                if (NewValue == OldValue) break;
-            }
-
-            /* Old value changed, loop again */
-            OldValue = NewValue;
-        }
-
-        /* Now we have to wait for it */
-        KeWaitForGate(&GuardedMutex->Gate, WrGuardedMutex, KernelMode);
-        ASSERT((GuardedMutex->Count & GM_LOCK_WAITER_WOKEN) != 0);
-
-        /* Ok, the wait is done, so set the new bits */
-        BitsToRemove = GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN;
-        BitsToAdd = GM_LOCK_WAITER_WOKEN;
-        
-        /* We depend on these bits being just right */
-        C_ASSERT((GM_LOCK_WAITER_WOKEN * 2) == GM_LOCK_WAITER_INC);
-    }
-}
-
-FORCEINLINE
-VOID
-FASTCALL
-KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
-{
-    BOOLEAN OldBit;
-
-    /* Remove the lock */
-    OldBit = InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V);
-    if (!OldBit)
-    {
-        /* The Guarded Mutex was already locked, enter contented case */
-        KiAcquireGuardedMutexContented(GuardedMutex);
-    }
-}
-
-FORCEINLINE
-VOID
-FASTCALL
-KiReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
-{
-    LONG OldValue, NewValue;
-
-    /* Destroy the Owner */
-    GuardedMutex->Owner = NULL;
-
-    /* Add the Lock Bit */
-    OldValue = InterlockedExchangeAdd(&GuardedMutex->Count, GM_LOCK_BIT);
-    ASSERT((OldValue & GM_LOCK_BIT) == 0);
-
-    /* Check if it was already locked, but not woken */
-    if ((OldValue) && !(OldValue & GM_LOCK_WAITER_WOKEN))
-    {
-        /* Update the Oldvalue to what it should be now */
-        OldValue += GM_LOCK_BIT;
-
-        /* The mutex will be woken, minus one waiter */
-        NewValue = OldValue + GM_LOCK_WAITER_WOKEN -
-            GM_LOCK_WAITER_INC;
-
-        /* Remove the Woken bit */
-        if (InterlockedCompareExchange(&GuardedMutex->Count,
-                                       NewValue,
-                                       OldValue) == OldValue)
-        {
-            /* Signal the Gate */
-            KeSignalGateBoostPriority(&GuardedMutex->Gate);
-        }
-    }
-}
 
 /* PUBLIC FUNCTIONS **********************************************************/
 
@@ -145,54 +30,8 @@ VOID
 FASTCALL
 KeInitializeGuardedMutex(OUT PKGUARDED_MUTEX GuardedMutex)
 {
-    /* Setup the Initial Data */
+    /* Call the inline */
-    GuardedMutex->Count = GM_LOCK_BIT;
+    _KeInitializeGuardedMutex(GuardedMutex);
-    GuardedMutex->Owner = NULL;
-    GuardedMutex->Contention = 0;
-
-    /* Initialize the Wait Gate */
-    KeInitializeGate(&GuardedMutex->Gate);
-}
-
-/*
- * @implemented
- */
-VOID
-FASTCALL
-KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
-{
-    PKTHREAD Thread = KeGetCurrentThread();
-
-    /* Sanity checks */
-    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
-           (Thread->SpecialApcDisable < 0) ||
-           (Thread->Teb == NULL) ||
-           (Thread->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
-    ASSERT(GuardedMutex->Owner != Thread);
-
-    /* Do the actual acquire */
-    KiAcquireGuardedMutex(GuardedMutex);
-
-    /* Set the Owner */
-    GuardedMutex->Owner = Thread;
-}
-
-/*
- * @implemented
- */
-VOID
-FASTCALL
-KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
-{
-    /* Sanity checks */
-    ASSERT((KeGetCurrentIrql() == APC_LEVEL) ||
-           (KeGetCurrentThread()->SpecialApcDisable < 0) ||
-           (KeGetCurrentThread()->Teb == NULL) ||
-           (KeGetCurrentThread()->Teb >= (PTEB)MM_SYSTEM_RANGE_START));
-    ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
-
-    /* Release the mutex */
-    KiReleaseGuardedMutex(GuardedMutex);
 }
 
 /*
@@ -202,21 +41,8 @@ VOID
 FASTCALL
 KeAcquireGuardedMutex(IN PKGUARDED_MUTEX GuardedMutex)
 {
-    PKTHREAD Thread = KeGetCurrentThread();
+    /* Call the inline */
-
+    _KeAcquireGuardedMutex(GuardedMutex);
-    /* Sanity checks */
-    ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL);
-    ASSERT(GuardedMutex->Owner != Thread);
-
-    /* Disable Special APCs */
-    KeEnterGuardedRegion();
-
-    /* Do the actual acquire */
-    KiAcquireGuardedMutex(GuardedMutex);
-
-    /* Set the Owner and Special APC Disable state */
-    GuardedMutex->Owner = Thread;
-    GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
 }
 
 /*
@@ -226,17 +52,30 @@ VOID
 FASTCALL
 KeReleaseGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
 {
-    /* Sanity checks */
+    /* Call the inline */
-    ASSERT_IRQL_LESS_OR_EQUAL(APC_LEVEL);
+    _KeReleaseGuardedMutex(GuardedMutex);
-    ASSERT(GuardedMutex->Owner == KeGetCurrentThread());
+}
-    ASSERT(GuardedMutex->SpecialApcDisable ==
-           KeGetCurrentThread()->SpecialApcDisable);
 
-    /* Release the mutex */
+/*
-    KiReleaseGuardedMutex(GuardedMutex);
+ * @implemented
+ */
+VOID
+FASTCALL
+KeAcquireGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    /* Call the inline */
+    _KeAcquireGuardedMutexUnsafe(GuardedMutex);
+}
 
-    /* Re-enable APCs */
+/*
-    KeLeaveGuardedRegion();
+ * @implemented
+ */
+VOID
+FASTCALL
+KeReleaseGuardedMutexUnsafe(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    /* Call the inline */
+    _KeReleaseGuardedMutexUnsafe(GuardedMutex);
 }
 
 /*
@@ -246,28 +85,8 @@ BOOLEAN
 FASTCALL
 KeTryToAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
 {
-    PKTHREAD Thread = KeGetCurrentThread();
+    /* Call the inline */
-    BOOLEAN OldBit;
+    return _KeTryToAcquireGuardedMutex(GuardedMutex);
-
-    /* Block APCs */
-    KeEnterGuardedRegion();
-
-    /* Remove the lock */
-    OldBit = InterlockedBitTestAndReset(&GuardedMutex->Count, GM_LOCK_BIT_V);
-    if (!OldBit)
-    {
-        /* Re-enable APCs */
-        KeLeaveGuardedRegion();
-        YieldProcessor();
-
-        /* Return failure */
-        return FALSE;
-    }
-
-    /* Set the Owner and APC State */
-    GuardedMutex->Owner = Thread;
-    GuardedMutex->SpecialApcDisable = Thread->SpecialApcDisable;
-    return TRUE;
 }
 
 /**

reactos/ntoskrnl/ke/wait.c

@@ -117,6 +117,76 @@ KiAcquireFastMutex(IN PFAST_MUTEX FastMutex)
                           NULL);
 }
 
+VOID
+FASTCALL
+KiAcquireGuardedMutex(IN OUT PKGUARDED_MUTEX GuardedMutex)
+{
+    ULONG BitsToRemove, BitsToAdd;
+    LONG OldValue, NewValue;
+    
+    /* Increase the contention count */
+    GuardedMutex->Contention++;
+    
+    /* Start by unlocking the Guarded Mutex */
+    BitsToRemove = GM_LOCK_BIT;
+    BitsToAdd = GM_LOCK_WAITER_INC;
+    
+    /* Start change loop */
+    for (;;)
+    {
+        /* Loop sanity checks */
+        ASSERT((BitsToRemove == GM_LOCK_BIT) ||
+               (BitsToRemove == (GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN)));
+        ASSERT((BitsToAdd == GM_LOCK_WAITER_INC) ||
+               (BitsToAdd == GM_LOCK_WAITER_WOKEN));
+        
+        /* Get the Count Bits */
+        OldValue = GuardedMutex->Count;
+        
+        /* Start internal bit change loop */
+        for (;;)
+        {
+            /* Check if the Guarded Mutex is locked */
+            if (OldValue & GM_LOCK_BIT)
+            {
+                /* Sanity check */
+                ASSERT((BitsToRemove == GM_LOCK_BIT) ||
+                       ((OldValue & GM_LOCK_WAITER_WOKEN) != 0));
+                
+                /* Unlock it by removing the Lock Bit */
+                NewValue = OldValue ^ BitsToRemove;
+                NewValue = InterlockedCompareExchange(&GuardedMutex->Count,
+                                                      NewValue,
+                                                      OldValue);
+                if (NewValue == OldValue) return;
+            }
+            else
+            {
+                /* The Guarded Mutex isn't locked, so simply set the bits */
+                NewValue = OldValue + BitsToAdd;
+                NewValue = InterlockedCompareExchange(&GuardedMutex->Count,
+                                                      NewValue,
+                                                      OldValue);
+                if (NewValue == OldValue) break;
+            }
+            
+            /* Old value changed, loop again */
+            OldValue = NewValue;
+        }
+        
+        /* Now we have to wait for it */
+        KeWaitForGate(&GuardedMutex->Gate, WrGuardedMutex, KernelMode);
+        ASSERT((GuardedMutex->Count & GM_LOCK_WAITER_WOKEN) != 0);
+        
+        /* Ok, the wait is done, so set the new bits */
+        BitsToRemove = GM_LOCK_BIT | GM_LOCK_WAITER_WOKEN;
+        BitsToAdd = GM_LOCK_WAITER_WOKEN;
+        
+        /* We depend on these bits being just right */
+        C_ASSERT((GM_LOCK_WAITER_WOKEN * 2) == GM_LOCK_WAITER_INC);
+    }
+}
+
 //
 // This routine exits the dispatcher after a compatible operation and
 // swaps the context to the next scheduled thread on the current CPU if