- Refactor Fast Reference implementation into a generic module with init/acquire/release/compare/insert interfaces, and update the executive callback & object manager referencing code to use it.

- Always capture fast reference values on the stack since a fast reference is by nature volatile.
- Based off earlier work from Alex to fix callback implementation.

svn path=/trunk/; revision=39859

reactos/ntoskrnl/ex/callback.c

@@ -45,8 +45,8 @@ VOID
 NTAPI
 ExInitializeCallBack(IN OUT PEX_CALLBACK Callback)
 {
-    /* Initialize the fast references */
+    /* Initialize the fast reference */
-    Callback->RoutineBlock.Object = NULL;
+    ExInitializeFastReference(&Callback->RoutineBlock, NULL);
 }
 
 PEX_CALLBACK_ROUTINE_BLOCK
@@ -54,93 +54,66 @@ NTAPI
 ExAllocateCallBack(IN PEX_CALLBACK_FUNCTION Function,
                    IN PVOID Context)
 {
-    PEX_CALLBACK_ROUTINE_BLOCK Callback;
+    PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
 
     /* Allocate a callback */
-    Callback = ExAllocatePoolWithTag(PagedPool,
+    CallbackBlock = ExAllocatePoolWithTag(PagedPool,
-                                     sizeof(*Callback),
+                                          sizeof(EX_CALLBACK_ROUTINE_BLOCK),
-                                     TAG('C', 'b', 'r', 'b'));
+                                          'CbRb');
-    if (Callback)
+    if (CallbackBlock)
     {
         /* Initialize it */
-        Callback->Function = Function;
+        CallbackBlock->Function = Function;
-        Callback->Context = Context;
+        CallbackBlock->Context = Context;
-        ExInitializeRundownProtection(&Callback->RundownProtect);
+        ExInitializeRundownProtection(&CallbackBlock->RundownProtect);
     }
 
     /* Return it */
-    return Callback;
+    return CallbackBlock;
 }
 
 VOID
 NTAPI
-ExFreeCallBack(IN PEX_CALLBACK_ROUTINE_BLOCK Callback)
+ExFreeCallBack(IN PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock)
 {
     /* Just free it from memory */
-    ExFreePool(Callback);
+    ExFreePool(CallbackBlock);
 }
 
 VOID
 NTAPI
-ExWaitForCallBacks(IN PEX_CALLBACK_ROUTINE_BLOCK Callback)
+ExWaitForCallBacks(IN PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock)
 {
     /* Wait on the rundown */
-    ExWaitForRundownProtectionRelease(&Callback->RundownProtect);
+    ExWaitForRundownProtectionRelease(&CallbackBlock->RundownProtect);
 }
 
 PEX_CALLBACK_FUNCTION
 NTAPI
-ExGetCallBackBlockRoutine(IN PEX_CALLBACK_ROUTINE_BLOCK Callback)
+ExGetCallBackBlockRoutine(IN PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock)
 {
     /* Return the function */
-    return Callback->Function;
+    return CallbackBlock->Function;
 }
 
 PVOID
 NTAPI
-ExGetCallBackBlockContext(IN PEX_CALLBACK_ROUTINE_BLOCK Callback)
+ExGetCallBackBlockContext(IN PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock)
 {
     /* Return the context */
-    return Callback->Context;
+    return CallbackBlock->Context;
 }
 
 VOID
 NTAPI
 ExDereferenceCallBackBlock(IN OUT PEX_CALLBACK CallBack,
-                           IN PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock)
+                           IN PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock)
 {
-    PEX_FAST_REF FastRef = &CallBack->RoutineBlock;
+    /* Release a fast reference */
-    EX_FAST_REF Value, NewValue;
+    if (!ExReleaseFastReference(&CallBack->RoutineBlock, CallbackBlock))
-
-    /* Sanity checks */
-    ASSERT(CallbackRoutineBlock);
-    ASSERT(!(((ULONG_PTR)CallbackRoutineBlock) & MAX_FAST_REFS));
-
-    /* Start dereference loop */
-    for (;;)
     {
-        /* Get the current count */
+        /* Take slow path */
-        Value = *FastRef;
+        ExReleaseRundownProtection(&CallbackBlock->RundownProtect);
-        if ((Value.Value ^ (ULONG_PTR)CallbackRoutineBlock) < MAX_FAST_REFS)
-        {
-            /* Decrease the reference count */
-            NewValue.Value = Value.Value + 1;
-            NewValue.Object = InterlockedCompareExchangePointer(&FastRef->Object,
-                                                                NewValue.Object,
-                                                                Value.Object);
-            if (NewValue.Object != Value.Object) continue;
-
-            /* We're all done */
-            break;
-        }
-        else
-        {
-            /* Release rundown protection */
-            ExReleaseRundownProtection(&CallbackRoutineBlock->RundownProtect);
-
-            /* We're all done */
-            break;
-        }
     }
 }
 
@@ -148,89 +121,48 @@ PEX_CALLBACK_ROUTINE_BLOCK
 NTAPI
 ExReferenceCallBackBlock(IN OUT PEX_CALLBACK CallBack)
 {
-    PEX_FAST_REF FastRef = &CallBack->RoutineBlock;
+    EX_FAST_REF OldValue;
-    EX_FAST_REF Value, NewValue;
+    ULONG_PTR Count;
-    PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock;
+    PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
     
-    /* Start reference loop */
+    /* Acquire a reference */
-    for (;;)
+    OldValue = ExAcquireFastReference(&CallBack->RoutineBlock);
-    {
+    Count = ExGetCountFastReference(OldValue);
-        /* Get the current count */
-        Value = *FastRef;
-        if (Value.RefCnt != 0)
-        {
-            /* Increase the reference count */
-            NewValue.Value = Value.Value - 1;
-            NewValue.Object = InterlockedCompareExchangePointer(&FastRef->Object,
-                                                                NewValue.Object,
-                                                                Value.Object);
-            if (NewValue.Object != Value.Object) continue;
-        }
-
-        /* All done */
-        break;
-    }
 
     /* Fail if there isn't any object */
-    if (!Value.Value) return NULL;
+    if (!ExGetObjectFastReference(OldValue)) return NULL;
 
     /* Check if we don't have a reference */
-    if (!Value.RefCnt)
+    if (!Count)
     {
         /* FIXME: Race */
-        CallbackRoutineBlock = NULL;
         DPRINT1("Unhandled callback race condition\n");
         ASSERT(FALSE);
+        return NULL;
     }
-    else
+    
-    {
     /* Get the callback block */
-        CallbackRoutineBlock = (PVOID)(Value.Value &~ MAX_FAST_REFS);
+    CallbackBlock = ExGetObjectFastReference(OldValue);
     
     /* Check if this is the last reference */
-        if (Value.RefCnt == 1)
+    if (Count == 1)
     {
         /* Acquire rundown protection */
-            if (ExfAcquireRundownProtectionEx(&CallbackRoutineBlock->
+        if (ExfAcquireRundownProtectionEx(&CallbackBlock->RundownProtect,
-                                              RundownProtect,
                                           MAX_FAST_REFS))
         {
-                /* Sanity check */
+            /* Insert references */
-                ASSERT(!(((ULONG_PTR)CallbackRoutineBlock) & MAX_FAST_REFS));
+            if (!ExInsertFastReference(&CallBack->RoutineBlock, CallbackBlock))
-
-                /* Start reference loop */
-                for (;;)
-                {
-                    /* Check if the current count is too high */
-                    Value = *FastRef;
-                    if (((Value.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
-                        ((Value.Value &~ MAX_FAST_REFS) !=
-                         (ULONG_PTR)CallbackRoutineBlock))
             {
                 /* Backdown the rundown acquire */
-                        ExfReleaseRundownProtectionEx(&CallbackRoutineBlock->
+                ExfReleaseRundownProtectionEx(&CallbackBlock->RundownProtect,
-                                                      RundownProtect,
                                               MAX_FAST_REFS);
-                        break;
-                    }
-
-                    /* Increase the reference count */
-                    NewValue.Value = Value.Value + MAX_FAST_REFS;
-                    NewValue.Object =
-                        InterlockedCompareExchangePointer(&FastRef->Object,
-                                                          NewValue.Object,
-                                                          Value.Object);
-                    if (NewValue.Object != Value.Object) continue;
-
-                    /* Break out if the change was OK */
-                    break;
-                }
             }
         }
     }
 
     /* Return the callback block */
-    return CallbackRoutineBlock;
+    return CallbackBlock;
 }
 
 BOOLEAN
@@ -239,9 +171,9 @@ ExCompareExchangeCallBack(IN OUT PEX_CALLBACK CallBack,
                           IN PEX_CALLBACK_ROUTINE_BLOCK NewBlock,
                           IN PEX_CALLBACK_ROUTINE_BLOCK OldBlock)
 {
-    EX_FAST_REF Value, NewValue;
+    EX_FAST_REF OldValue;
-    PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock;
+    PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
-    PEX_FAST_REF FastRef = &CallBack->RoutineBlock;
+    ULONG_PTR Count;
 
     /* Check that we have a new block */
     if (NewBlock)
@@ -256,46 +188,20 @@ ExCompareExchangeCallBack(IN OUT PEX_CALLBACK CallBack,
         }
     }
     
-    /* Sanity check and start swap loop */
+    /* Do the swap */
-    ASSERT(!(((ULONG_PTR)NewBlock) & MAX_FAST_REFS));
+    OldValue = ExCompareSwapFastReference(&CallBack->RoutineBlock,
-    for (;;)
+                                          NewBlock,
-    {
+                                          OldBlock);
-        /* Get the current value */
-        Value = *FastRef;
-
-        /* Make sure there's enough references to swap */
-        if (!((Value.Value ^ (ULONG_PTR)OldBlock) <= MAX_FAST_REFS)) break;
-
-        /* Check if we have an object to swap */
-        if (NewBlock)
-        {
-            /* Set up the value with maximum fast references */
-            NewValue.Value = (ULONG_PTR)NewBlock | MAX_FAST_REFS;
-        }
-        else
-        {
-            /* Write the object address itself (which is empty) */
-            NewValue.Value = (ULONG_PTR)NewBlock;
-        }
-
-        /* Do the actual compare exchange */
-        NewValue.Object = InterlockedCompareExchangePointer(&FastRef->Object,
-                                                            NewValue.Object,
-                                                            Value.Object);
-        if (NewValue.Object != Value.Object) continue;
-
-        /* All done */
-        break;
-    }
 
     /* Get the routine block */
-    CallbackRoutineBlock = (PVOID)(Value.Value & ~MAX_FAST_REFS);
+    CallbackBlock = ExGetObjectFastReference(OldValue);
+    Count = ExGetCountFastReference(OldValue);
 
     /* Make sure the swap worked */
-    if (CallbackRoutineBlock == OldBlock)
+    if (CallbackBlock == OldBlock)
     {
         /* Make sure we replaced a valid pointer */
-        if (CallbackRoutineBlock)
+        if (CallbackBlock)
         {
             /* Acquire the flush lock and immediately release it */
             KeEnterCriticalRegion();
@@ -303,8 +209,8 @@ ExCompareExchangeCallBack(IN OUT PEX_CALLBACK CallBack,
 
             /* Release rundown protection */
             KeLeaveCriticalRegion();
-            ExfReleaseRundownProtectionEx(&CallbackRoutineBlock->RundownProtect,
+            ExfReleaseRundownProtectionEx(&CallbackBlock->RundownProtect,
-                                          Value.RefCnt + 1);
+                                          Count + 1);
         }
 
         /* Compare worked */

reactos/ntoskrnl/include/internal/ex.h

@@ -119,6 +119,15 @@ typedef struct
 
 #endif
 
+#ifdef _WIN64
+#define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
+#define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
+#else
+#define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
+#define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
+#define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
+#endif
+
 /* INITIALIZATION FUNCTIONS *************************************************/
 
 VOID
@@ -439,34 +448,238 @@ ExDoCallBack(IN OUT PEX_CALLBACK Callback,
              IN PVOID Argument1,
              IN PVOID Argument2)
 {
-    PEX_CALLBACK_ROUTINE_BLOCK CallbackRoutineBlock;
+    PEX_CALLBACK_ROUTINE_BLOCK CallbackBlock;
     PEX_CALLBACK_FUNCTION Function;
 
     /* Reference the block */
-    CallbackRoutineBlock = ExReferenceCallBackBlock(Callback);
+    CallbackBlock = ExReferenceCallBackBlock(Callback);
-    if (CallbackRoutineBlock)
+    if (CallbackBlock)
     {
         /* Get the function */
-        Function = ExGetCallBackBlockRoutine(CallbackRoutineBlock);
+        Function = ExGetCallBackBlockRoutine(CallbackBlock);
 
         /* Do the callback */
         Function(Context, Argument1, Argument2);
 
         /* Now dereference it */
-        ExDereferenceCallBackBlock(Callback, CallbackRoutineBlock);
+        ExDereferenceCallBackBlock(Callback, CallbackBlock);
     }
 }
 
-/* RUNDOWN *******************************************************************/
+/* FAST REFS ******************************************************************/
 
-#ifdef _WIN64
+FORCEINLINE
-#define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
+PVOID
-#define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
+ExGetObjectFastReference(IN EX_FAST_REF FastRef)
-#else
+{
-#define ExpChangeRundown(x, y, z) PtrToUlong(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
+    /* Return the unbiased pointer */
-#define ExpChangePushlock(x, y, z) LongToPtr(InterlockedCompareExchange((PLONG)x, PtrToLong(y), PtrToLong(z)))
+    return (PVOID)(FastRef.Value & ~MAX_FAST_REFS);
-#define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
+}
-#endif
+
+FORCEINLINE
+ULONG
+ExGetCountFastReference(IN EX_FAST_REF FastRef)
+{
+    /* Return the reference count */
+    return FastRef.RefCnt;
+}
+
+FORCEINLINE
+VOID
+ExInitializeFastReference(OUT PEX_FAST_REF FastRef,
+                          IN OPTIONAL PVOID Object)
+{
+    /* Sanity check */
+    ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
+    
+    /* Check if an object is being set */
+    if (!Object)
+    {
+        /* Clear the field */
+        FastRef->Object = NULL;
+    }
+    else
+    {
+        /* Otherwise, we assume the object was referenced and is ready */
+        FastRef->Value = (ULONG_PTR)Object | MAX_FAST_REFS;
+    }
+}
+
+FORCEINLINE
+EX_FAST_REF
+ExAcquireFastReference(IN OUT PEX_FAST_REF FastRef)
+{
+    EX_FAST_REF OldValue, NewValue;
+    
+    /* Start reference loop */
+    for (;;)
+    {
+        /* Get the current reference count */
+        OldValue = *FastRef;
+        if (OldValue.RefCnt)
+        {
+            /* Increase the reference count */
+            NewValue.Value = OldValue.Value - 1;
+            NewValue.Object = ExpChangePushlock(&FastRef->Object,
+                                                NewValue.Object,
+                                                OldValue.Object);
+            if (NewValue.Object != OldValue.Object) continue;
+        }
+        
+        /* We are done */
+        break;
+    }
+    
+    /* Return the old value */
+    return OldValue;
+}
+
+FORCEINLINE
+BOOLEAN
+ExInsertFastReference(IN OUT PEX_FAST_REF FastRef,
+                      IN PVOID Object)
+{
+    EX_FAST_REF OldValue, NewValue;
+    
+    /* Sanity checks */
+    ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
+    
+    /* Start update loop */
+    for (;;)
+    {
+        /* Get the current reference count */
+        OldValue = *FastRef;
+        
+        /* Check if the current count is too high or if the pointer changed */
+        if (((OldValue.RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
+            ((OldValue.Value &~ MAX_FAST_REFS) != (ULONG_PTR)Object))
+        {
+            /* Fail */
+            return FALSE;
+        }
+        
+        /* Update the reference count */
+        NewValue.Value = OldValue.Value + MAX_FAST_REFS;
+        NewValue.Object = ExpChangePushlock(&FastRef->Object,
+                                            NewValue.Object,
+                                            OldValue.Object);
+        if (NewValue.Object != OldValue.Object) continue;
+        
+        /* We are done */
+        break;
+    }
+    
+    /* Return success */
+    return TRUE;   
+}
+
+BOOLEAN
+FORCEINLINE
+ExReleaseFastReference(IN PEX_FAST_REF FastRef,
+                       IN PVOID Object)
+{
+    EX_FAST_REF OldValue, NewValue;
+    
+    /* Sanity checks */
+    ASSERT(Object != NULL);
+    ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
+    
+    /* Start reference loop */
+    for (;;)
+    {
+        /* Get the current reference count */
+        OldValue = *FastRef;
+        
+        /* Check if we're full if if the pointer changed */
+        if ((OldValue.Value ^ (ULONG_PTR)Object) >= MAX_FAST_REFS) return FALSE;
+            
+        /* Decrease the reference count */
+        NewValue.Value = OldValue.Value + 1;
+        NewValue.Object = ExpChangePushlock(&FastRef->Object,
+                                            NewValue.Object,
+                                            OldValue.Object);
+        if (NewValue.Object != OldValue.Object) continue;
+        
+        /* We are done */
+        break;
+    }
+    
+    /* Return success */
+    return TRUE;
+}
+
+EX_FAST_REF
+FORCEINLINE
+ExSwapFastReference(IN PEX_FAST_REF FastRef,
+                    IN PVOID Object)
+{
+    EX_FAST_REF NewValue, OldValue;
+    
+    /* Sanity check */
+    ASSERT((((ULONG_PTR)Object) & MAX_FAST_REFS) == 0);
+    
+    /* Check if an object is being set */
+    if (!Object)
+    {
+        /* Clear the field */
+        NewValue.Object = NULL;
+    }
+    else
+    {
+        /* Otherwise, we assume the object was referenced and is ready */
+        NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
+    }
+    
+    /* Update the object */
+    OldValue.Object = InterlockedExchangePointer(&FastRef->Object, NewValue.Object);
+    return OldValue;
+}
+
+EX_FAST_REF
+FORCEINLINE
+ExCompareSwapFastReference(IN PEX_FAST_REF FastRef,
+                           IN PVOID Object,
+                           IN PVOID OldObject)
+{
+    EX_FAST_REF OldValue, NewValue;
+    
+    /* Sanity check and start swap loop */
+    ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
+    for (;;)
+    {
+        /* Get the current value */
+        OldValue = *FastRef;
+        
+        /* Make sure there's enough references to swap */
+        if (!((OldValue.Value ^ (ULONG_PTR)OldObject) <= MAX_FAST_REFS)) break;
+        
+        /* Check if we have an object to swap */
+        if (Object)
+        {
+            /* Set up the value with maximum fast references */
+            NewValue.Value = (ULONG_PTR)Object | MAX_FAST_REFS;
+        }
+        else
+        {
+            /* Write the object address itself (which is empty) */
+            NewValue.Value = (ULONG_PTR)Object;
+        }
+        
+        /* Do the actual compare exchange */
+        NewValue.Object = ExpChangePushlock(&FastRef->Object,
+                                            NewValue.Object,
+                                            OldValue.Object);
+        if (NewValue.Object != OldValue.Object) continue;
+        
+        /* All done */
+        break;
+    }
+    
+    /* Return the old value */
+    return OldValue;
+}
+
+/* RUNDOWN *******************************************************************/
 
 /*++
  * @name ExfAcquireRundownProtection

reactos/ntoskrnl/ob/obref.c

@@ -110,20 +110,8 @@ ObInitializeFastReference(IN PEX_FAST_REF FastRef,
     /* Check if we were given an object and reference it 7 times */
     if (Object) ObReferenceObjectEx(Object, MAX_FAST_REFS);
     
-    /* Sanity check */
+    /* Setup the fast reference */
-    ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
+    ExInitializeFastReference(FastRef, Object);
-
-    /* Check if the caller gave us an object */
-    if (Object)
-    {
-        /* He did, so write the biased pointer */
-        FastRef->Object = (PVOID)((ULONG_PTR)Object | MAX_FAST_REFS);
-    }
-    else
-    {
-        /* Otherwise, clear the current object */
-        FastRef->Object = NULL;
-    }
 }
 
 PVOID
@@ -131,9 +119,10 @@ FASTCALL
 ObFastReferenceObjectLocked(IN PEX_FAST_REF FastRef)
 {
     PVOID Object;
+    EX_FAST_REF OldValue = *FastRef;
 
     /* Get the object and reference it slowly */
-    Object = (PVOID)((ULONG_PTR)FastRef->Object & MAX_FAST_REFS);
+    Object = ExGetObjectFastReference(OldValue);
     if (Object) ObReferenceObject(Object);
     return Object;
 }
@@ -142,25 +131,16 @@ PVOID
 FASTCALL
 ObFastReferenceObject(IN PEX_FAST_REF FastRef)
 {
-    ULONG_PTR Value, NewValue;
+    EX_FAST_REF OldValue;
     ULONG_PTR Count;
     PVOID Object;
 
-    /* Start reference loop */
+    /* Reference the object and get it pointer */
-    for (;;)
+    OldValue = ExAcquireFastReference(FastRef);
-    {
+    Object = ExGetObjectFastReference(OldValue);
-        /* Get the current count */
-        Value = FastRef->Value;
-        if (!(Value & MAX_FAST_REFS)) break;
 
-        /* Increase the reference count */
+    /* Check how many references are left */
-        NewValue = Value - 1;
+    Count = ExGetCountFastReference(OldValue);
-        if (ExpChangeRundown(FastRef, NewValue, Value) == Value) break;
-    }
-
-    /* Get the object and count */
-    Object = (PVOID)(Value &~ MAX_FAST_REFS);
-    Count = Value & MAX_FAST_REFS;
 
     /* Check if the reference count is over 1 */
     if (Count > 1) return Object;
@@ -170,25 +150,12 @@ ObFastReferenceObject(IN PEX_FAST_REF FastRef)
 
     /* Otherwise, reference the object 7 times */
     ObReferenceObjectEx(Object, MAX_FAST_REFS);
-    ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
     
-    for (;;)
+    /* Now update the reference count */
+    if (!ExInsertFastReference(FastRef, Object))
     {
-        /* Check if the current count is too high */
+        /* We failed: completely dereference the object */
-        Value = FastRef->Value;
-        if (((FastRef->RefCnt + MAX_FAST_REFS) > MAX_FAST_REFS) ||
-            ((PVOID)((ULONG_PTR)FastRef->Object &~ MAX_FAST_REFS) != Object))
-        {
-            /* Completely dereference the object */
         ObDereferenceObjectEx(Object, MAX_FAST_REFS);
-            break;
-        }
-        else
-        {
-            /* Increase the reference count */
-            NewValue = Value + MAX_FAST_REFS;
-            if (ExpChangeRundown(FastRef, NewValue, Value) == Value) break;
-        }
     }
 
     /* Return the Object */
@@ -200,30 +167,8 @@ FASTCALL
 ObFastDereferenceObject(IN PEX_FAST_REF FastRef,
                         IN PVOID Object)
 {
-    ULONG_PTR Value, NewValue;
+    /* Release a fast reference. If this failed, use the slow path */
-
+    if (!ExReleaseFastReference(FastRef, Object)) ObDereferenceObject(Object);
-    /* Sanity checks */
-    ASSERT(Object);
-    ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
-
-    /* Start dereference loop */
-    for (;;)
-    {
-        /* Get the current count */
-        Value = FastRef->Value;
-        if ((Value ^ (ULONG_PTR)Object) < MAX_FAST_REFS)
-        {
-            /* Decrease the reference count */
-            NewValue = Value + 1;
-            if (ExpChangeRundown(FastRef, NewValue, Value) == Value) return;
-        }
-        else
-        {
-            /* Do a normal Dereference */
-            ObDereferenceObject(Object);
-            return;
-        }
-    }
 }
 
 PVOID
@@ -231,36 +176,20 @@ FASTCALL
 ObFastReplaceObject(IN PEX_FAST_REF FastRef,
                     PVOID Object)
 {
-    ULONG_PTR NewValue;
+    EX_FAST_REF OldValue;
-    EX_FAST_REF OldRef;
     PVOID OldObject;
+    ULONG_PTR Count;
 
     /* Check if we were given an object and reference it 7 times */
     if (Object) ObReferenceObjectEx(Object, MAX_FAST_REFS);
     
-    /* Sanity check */
+    /* Do the swap */
-    ASSERT(!(((ULONG_PTR)Object) & MAX_FAST_REFS));
+    OldValue = ExSwapFastReference(FastRef, Object);
+    OldObject = ExGetObjectFastReference(OldValue);
     
-    /* Check if the caller gave us an object */
+    /* Check if we had an active object and dereference it */
-    if (Object)
+    Count = ExGetCountFastReference(OldValue);
-    {
+    if ((OldObject) && (Count)) ObDereferenceObjectEx(OldObject, Count);
-        /* He did, so bias the pointer */
-        NewValue = (ULONG_PTR)Object | MAX_FAST_REFS;
-    }
-    else
-    {
-        /* No object, we're clearing */
-        NewValue = 0;
-    }
-
-    /* Switch objects */
-    OldRef.Value = InterlockedExchange((PLONG)&FastRef->Value, NewValue);
-    OldObject = (PVOID)((ULONG_PTR)OldRef.Object &~ MAX_FAST_REFS);
-    if ((OldObject) && (OldRef.RefCnt))
-    {
-        /* Dereference the old object */
-        ObDereferenceObjectEx(OldObject, OldRef.RefCnt);
-    }
 
     /* Return the old object */
     return OldObject;