- Implemented InterlockedBitTestAndReset, InterlockedBitTestAndSet, InterlockedExchangeAddSizeT.

- Rundown re-implementation:

* Added inlined functions for internal system use for quickest path.
* Correctly named all functions Exf instead of Ex.
* Removed PAGED_CODE(); macro where it shouldn't be used.
* Addded multiple ASSERTS for sanity checks.
* Used macros for win64/32 portability.
* Fixed the following bugs/features:
    * ExfAcquireRundownProtection:
        ** Added specific code instead of calling the generic function. Rundown locks are
           performance critical and a dedicated path is prefered.
    * ExfAcquireRundownProtectionEx:
        ** Added a quick immediate check to see if the rundown is active.
    * ExfReleaseRundownProtection:
        ** Added specific code instead of calling the generic function. Rundown locks are
           performance critical and a dedicated path is prefered.
    * ExfReleaseRundownProtectionEx: 
        ** Simplified the loop code.
        ** Fixed a bug in signaling of the event during waitblock count removal
    * ExfWaitForRundownProtectionRelease:
        ** Add quick case when we don't actually need a full wait.
        ** Simplified loop code.
* Added stubs for cache-aware implementation.
* Documented the functions.

svn path=/trunk/; revision=20435

reactos/include/ndk/exfuncs.h

@@ -37,11 +37,62 @@ typedef struct _EVENT_TRACE_HEADER *PEVENT_TRACE_HEADER;
 //
 VOID
 FASTCALL
-ExEnterCriticalRegionAndAcquireFastMutexUnsafe(PFAST_MUTEX FastMutex);
+ExEnterCriticalRegionAndAcquireFastMutexUnsafe(
+    PFAST_MUTEX FastMutex
+);
 
 VOID
 FASTCALL
-ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(PFAST_MUTEX FastMutex);
+ExReleaseFastMutexUnsafeAndLeaveCriticalRegion(
+    PFAST_MUTEX FastMutex
+);
+
+//
+// Pushlock functions
+//
+VOID
+FASTCALL
+ExfAcquirePushLockExclusive(
+    PEX_PUSH_LOCK PushLock
+);
+
+VOID
+FASTCALL
+ExfAcquirePushLockShared(
+    PEX_PUSH_LOCK PushLock
+);
+
+VOID
+FASTCALL
+ExfReleasePushLock(
+    PEX_PUSH_LOCK PushLock
+);
+
+VOID
+FASTCALL
+ExfReleasePushLockExclusive(
+    PEX_PUSH_LOCK PushLock
+);
+
+VOID
+FASTCALL
+ExfReleasePushLockShared(
+    PEX_PUSH_LOCK PushLock
+);
+
+VOID
+FASTCALL
+ExfTryToWakePushLock(
+    PEX_PUSH_LOCK PushLock
+);
+
+VOID
+FASTCALL
+ExfUnblockPushLock(
+    PEX_PUSH_LOCK PushLock,
+    PVOID CurrentWaitBlock
+);
+
 #endif
 
 //

reactos/ntoskrnl/cm/ntfunc.c

@@ -147,7 +147,7 @@ CmiCallRegisteredCallbacks(IN REG_NOTIFY_CLASS Argument1,
 
     CurrentCallback = CONTAINING_RECORD(CurrentEntry, REGISTRY_CALLBACK, ListEntry);
     if(!CurrentCallback->PendingDelete &&
-       ExAcquireRundownProtectionEx(&CurrentCallback->RundownRef, 1))
+       ExAcquireRundownProtection(&CurrentCallback->RundownRef))
     {
       /* don't hold locks during the callbacks! */
       ExReleaseFastMutex(&CmiCallbackLock);
@@ -160,7 +160,7 @@ CmiCallRegisteredCallbacks(IN REG_NOTIFY_CLASS Argument1,
       /* don't release the rundown protection before holding the callback lock
          so the pointer to the next callback isn't cleared in case this callback
          get's deleted */
-      ExReleaseRundownProtectionEx(&CurrentCallback->RundownRef, 1);
+      ExReleaseRundownProtection(&CurrentCallback->RundownRef);
       if(!NT_SUCCESS(Status))
       {
         /* one callback returned failure, don't call any more callbacks */

reactos/ntoskrnl/ex/rundown.c

@@ -1,10 +1,10 @@
 /*
  * COPYRIGHT:       See COPYING in the top level directory
- * PROJECT:         ReactOS kernel
+ * PROJECT:         ReactOS Kernel
  * FILE:            ntoskrnl/ex/rundown.c
- * PURPOSE:         Rundown Protection Functions
+ * PURPOSE:         Rundown and Cache-Aware Rundown Protection
- *
+ * PROGRAMMERS:     Alex Ionescu (alex@relsoft.net)
- * PROGRAMMERS:     Alex Ionescu & Thomas Weidenmueller - Implementation
+ *                  Thomas Weidenmueller
  */
 
 /* INCLUDES *****************************************************************/
@@ -15,261 +15,489 @@
 
 /* FUNCTIONS *****************************************************************/
 
-/*
+/*++
- * @implemented
+ * @name ExfAcquireRundownProtection
- */
+ * @implemented NT5.1
+ *
+ *     The ExfAcquireRundownProtection routine acquires rundown protection for
+ *     the specified descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return TRUE if access to the protected structure was granted, FALSE otherwise.
+ *
+ * @remarks Callers of ExfAcquireRundownProtection can be running at any IRQL.
+ *
+ *--*/
 BOOLEAN
 FASTCALL
-ExAcquireRundownProtection (
+ExfAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
-    IN PEX_RUNDOWN_REF RunRef
-    )
 {
-    /* Call the general function with only one Reference add */
+    ULONG_PTR Value = RunRef->Count, NewValue;
-    return ExAcquireRundownProtectionEx(RunRef, 1);
+
+    /* Make sure a rundown is not active */
+    if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
+
+    /* Loop until successfully incremented the counter */
+    for (;;)
+    {
+        /* Add a reference */
+        NewValue = Value + EX_RUNDOWN_COUNT_INC;
+
+        /* Change the value */
+        Value = ExpChangeRundown(RunRef, NewValue, Value);
+        if (Value == NewValue) return TRUE;
+
+        /* Make sure a rundown is not active */
+        if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
+    }
 }
 
-/*
+/*++
- * @implemented
+ * @name ExfAcquireRundownProtectionEx
- */
+ * @implemented NT5.2
+ *
+ *     The ExfAcquireRundownProtectionEx routine acquires multiple rundown
+ *     protection references for the specified descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @param Count
+ *         Number of times to reference the descriptor.
+ *
+ * @return TRUE if access to the protected structure was granted, FALSE otherwise.
+ *
+ * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
+ *
+ *--*/
 BOOLEAN
 FASTCALL
-ExAcquireRundownProtectionEx (
+ExfAcquireRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
-    IN PEX_RUNDOWN_REF RunRef,
+                              IN ULONG Count)
-    IN ULONG Count
-    )
 {
-    ULONG_PTR PrevCount, Current;
+    ULONG_PTR Value = RunRef->Count, NewValue;
 
-    PAGED_CODE();
+    /* Make sure a rundown is not active */
+    if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
 
+    /* Convert the count to our internal representation */
     Count <<= EX_RUNDOWN_COUNT_SHIFT;
 
     /* Loop until successfully incremented the counter */
-    do
+    for (;;)
     {
-        Current = RunRef->Count;
+        /* Add references */
+        NewValue = Value + Count;
+
+        /* Change the value */
+        Value = ExpChangeRundown(RunRef, NewValue, Value);
+        if (Value == NewValue) return TRUE;
 
         /* Make sure a rundown is not active */
-        if (Current & EX_RUNDOWN_ACTIVE)
+        if (Value & EX_RUNDOWN_ACTIVE) return FALSE;
-        {
+    }
-            return FALSE;
-        }
-
-#ifdef _WIN64
-        PrevCount = (ULONG_PTR)InterlockedExchangeAdd64((LONGLONG*)&RunRef->Count, (LONGLONG)Count);
-#else
-        PrevCount = (ULONG_PTR)InterlockedExchangeAdd((LONG*)&RunRef->Count, (LONG)Count);
-#endif
-    } while (PrevCount != Current);
-
-    /* Return Success */
-    return TRUE;
 }
 
-/*
+/*++
- * @implemented
+ * @name ExfInitializeRundownProtection
- */
+ * @implemented NT5.1
+ *
+ *     The ExfInitializeRundownProtection routine initializes a rundown
+ *     protection descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExfInitializeRundownProtection can be running at any IRQL.
+ *
+ *--*/
 VOID
 FASTCALL
-ExInitializeRundownProtection (
+ExfInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
-    IN PEX_RUNDOWN_REF RunRef
-    )
 {
-    PAGED_CODE();
-
     /* Set the count to zero */
     RunRef->Count = 0;
 }
 
-/*
+/*++
- * @implemented
+ * @name ExfReInitializeRundownProtection
- */
+ * @implemented NT5.1
+ *
+ *     The ExfReInitializeRundownProtection routine re-initializes a rundown
+ *     protection descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExfReInitializeRundownProtection can be running at any IRQL.
+ *
+ *--*/
 VOID
 FASTCALL
-ExReInitializeRundownProtection (
+ExfReInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
-    IN PEX_RUNDOWN_REF RunRef
-    )
 {
     PAGED_CODE();
 
+    /* Sanity check */
+    ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
+
     /* Reset the count */
-#ifdef _WIN64
+    InterlockedExchange((PLONG)&RunRef->Count, 0);
-    InterlockedExchange64((LONGLONG*)&RunRef->Count, 0LL);
-#else
-    InterlockedExchange((LONG*)&RunRef->Count, 0);
-#endif
 }
 
-
+/*++
-/*
+ * @name ExfRundownCompleted
- * @implemented
+ * @implemented NT5.1
- */
+ *
+ *     The ExfRundownCompleted routine completes the rundown of the specified
+ *     descriptor by setting the active bit.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExfRundownCompleted must be running at IRQL <= APC_LEVEL.
+ *
+ *--*/
 VOID
 FASTCALL
-ExReleaseRundownProtectionEx (
+ExfRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
-    IN PEX_RUNDOWN_REF RunRef,
-    IN ULONG Count
-    )
 {
     PAGED_CODE();
 
-    Count <<= EX_RUNDOWN_COUNT_SHIFT;
+    /* Sanity check */
+    ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
 
-    for (;;)
+    /* Mark the counter as active */
+    InterlockedExchange((PLONG)&RunRef->Count, EX_RUNDOWN_ACTIVE);
+}
+
+/*++
+ * @name ExfReleaseRundownProtection
+ * @implemented NT5.1
+ *
+ *     The ExfReleaseRundownProtection routine releases the rundown protection
+ *     reference for the specified descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExfReleaseRundownProtection can be running at any IRQL.
+ *
+ *--*/
+VOID
+FASTCALL
+ExfReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
+{
+    ULONG_PTR Value = RunRef->Count, NewValue;
+    PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
+
+    /* Check if rundown is not active */
+    if (!(Value & EX_RUNDOWN_ACTIVE))
     {
-        ULONG_PTR Current = RunRef->Count;
+        /* Loop until successfully incremented the counter */
-
-        /* Check if Rundown is active */
-        if (Current & EX_RUNDOWN_ACTIVE)
-        {
-            /* Get Pointer */
-            PEX_RUNDOWN_WAIT_BLOCK RundownDescriptor = (PEX_RUNDOWN_WAIT_BLOCK)(Current & ~EX_RUNDOWN_ACTIVE);
-
-            if (RundownDescriptor == NULL)
-            {
-                /* the rundown was completed and there's no one to notify */
-                break;
-            }
-
-            Current = RundownDescriptor->Count;
-
-            /* Decrease RundownDescriptor->Count by Count Count */
-            for (;;)
-            {
-                ULONG_PTR PrevCount, NewCount;
-
-                if ((Count >> EX_RUNDOWN_COUNT_SHIFT) == Current)
-                {
-                    NewCount = 0;
-                }
-                else
-                {
-                    NewCount = ((RundownDescriptor->Count - (Count >> EX_RUNDOWN_COUNT_SHIFT)) << EX_RUNDOWN_COUNT_SHIFT) | EX_RUNDOWN_ACTIVE;
-                }
-#ifdef _WIN64
-                PrevCount = (ULONG_PTR)InterlockedCompareExchange64((LONGLONG*)&RundownDescriptor->Count, (LONGLONG)NewCount, (LONGLONG)Current);
-#else
-                PrevCount = (ULONG_PTR)InterlockedCompareExchange((LONG*)&RundownDescriptor->Count, (LONG)NewCount, (LONG)Current);
-#endif
-                if (PrevCount == Current)
-                {
-                    if (NewCount == 0)
-                    {
-                        /* Signal the event so anyone waiting on it can now kill it */
-                        KeSetEvent(&RundownDescriptor->RundownEvent, 0, FALSE);
-                    }
-
-                    /* Successfully decremented the counter, so bail! */
-                    break;
-                }
-
-                Current = PrevCount;
-            }
-
-            break;
-        }
-        else
-        {
-            ULONG_PTR PrevCount, NewCount = Current - (ULONG_PTR)Count;
-#ifdef _WIN64
-            PrevCount = (ULONG_PTR)InterlockedCompareExchange64((LONGLONG*)&RunRef->Count, (LONGLONG)NewCount, (LONGLONG)Current);
-#else
-            PrevCount = (ULONG_PTR)InterlockedCompareExchange((LONG*)&RunRef->Count, (LONG)NewCount, (LONG)Current);
-#endif
-            if (PrevCount == Current)
-            {
-                /* Successfully decremented the counter, so bail! */
-                break;
-            }
-        }
-    }
-}
-
-/*
-* @implemented
-*/
-VOID
-FASTCALL
-ExReleaseRundownProtection (
-    IN PEX_RUNDOWN_REF RunRef
-    )
-{
-    /* Call the general function with only 1 reference removal */
-    ExReleaseRundownProtectionEx(RunRef, 1);
-}
-
-/*
- * @implemented
- */
-VOID
-FASTCALL
-ExRundownCompleted (
-    IN PEX_RUNDOWN_REF RunRef
-    )
-{
-    PAGED_CODE();
-
-    /* mark the counter as active */
-#ifdef _WIN64
-    InterlockedExchange64((LONGLONG*)&RunRef->Count, (LONGLONG)EX_RUNDOWN_ACTIVE);
-#else
-    InterlockedExchange((LONG*)&RunRef->Count, EX_RUNDOWN_ACTIVE);
-#endif
-}
-
-/*
- * @implemented
- */
-VOID
-FASTCALL
-ExWaitForRundownProtectionRelease (
-    IN PEX_RUNDOWN_REF RunRef
-    )
-{
-    ULONG_PTR PrevCount, NewPtr, PrevPtr;
-    EX_RUNDOWN_WAIT_BLOCK RundownDescriptor;
-
-    PAGED_CODE();
-
-    PrevCount = RunRef->Count;
-
-    if (PrevCount != 0 && !(PrevCount & EX_RUNDOWN_ACTIVE))
-    {
-        /* save the reference counter */
-        RundownDescriptor.Count = PrevCount >> EX_RUNDOWN_COUNT_SHIFT;
-
-        /* Pending Count... wait on them to be closed with an event */
-        KeInitializeEvent(&RundownDescriptor.RundownEvent, NotificationEvent, FALSE);
-
-        ASSERT(!((ULONG_PTR)&RundownDescriptor & EX_RUNDOWN_ACTIVE));
-
-        NewPtr = (ULONG_PTR)&RundownDescriptor | EX_RUNDOWN_ACTIVE;
-
         for (;;)
         {
-#ifdef _WIN64
+            /* Sanity check */
-            PrevPtr = (ULONG_PTR)InterlockedCompareExchange64((LONGLONG*)&RunRef->Ptr, (LONGLONG)NewPtr, (LONGLONG)PrevCount);
+            ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
-#else
-            PrevPtr = (ULONG_PTR)InterlockedCompareExchange((LONG*)&RunRef->Ptr, (LONG)NewPtr, (LONG)PrevCount);
-#endif
-            if (PrevPtr == PrevCount)
-            {
-                /* Wait for whoever needs to release to notify us */
-                KeWaitForSingleObject(&RundownDescriptor.RundownEvent, Executive, KernelMode, FALSE, NULL);
-                break;
-            }
-            else if (PrevPtr == 0 || (PrevPtr & EX_RUNDOWN_ACTIVE))
-            {
-                /* some one else was faster, let's just bail */
-                break;
-            }
 
-            PrevCount = PrevPtr;
+            /* Get the new value */
+            NewValue = Value - EX_RUNDOWN_COUNT_INC;
 
-            /* save the changed reference counter and try again */
+            /* Change the value */
-            RundownDescriptor.Count = PrevCount >> EX_RUNDOWN_COUNT_SHIFT;
+            Value = ExpChangeRundown(RunRef, NewValue, Value);
+            if (Value == NewValue) return;
+
+            /* Loop again if we're still not active */
+            if (Value & EX_RUNDOWN_ACTIVE) break;
         }
     }
+
+    /* Get the wait block */
+    WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
+    ASSERT((WaitBlock->Count > 0) || (KeNumberProcessors > 1));
+
+    /* Remove the one count */
+    if (InterlockedExchangeAddSizeT(&WaitBlock->Count, -1))
+    {
+        /* We're down to 0 now, so signal the event */
+        KeSetEvent(&WaitBlock->RundownEvent, IO_NO_INCREMENT, FALSE);
+    }
+}
+
+/*++
+ * @name ExfReleaseRundownProtectionEx
+ * @implemented NT5.2
+ *
+ *     The ExfReleaseRundownProtectionEx routine releases multiple rundown
+ *     protection references for the specified descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @param Count
+ *         Number of times to dereference the descriptor.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExfAcquireRundownProtectionEx can be running at any IRQL.
+ *
+ *--*/
+VOID
+FASTCALL
+ExfReleaseRundownProtectionEx(IN PEX_RUNDOWN_REF RunRef,
+                              IN ULONG Count)
+{
+    ULONG_PTR Value = RunRef->Count, NewValue;
+    PEX_RUNDOWN_WAIT_BLOCK WaitBlock;
+
+    /* Check if rundown is not active */
+    if (!(Value & EX_RUNDOWN_ACTIVE))
+    {
+        /* Loop until successfully incremented the counter */
+        for (;;)
+        {
+            /* Sanity check */
+            ASSERT((Value >= EX_RUNDOWN_COUNT_INC * Count) || (KeNumberProcessors > 1));
+
+            /* Get the new value */
+            NewValue = Value - (Count * EX_RUNDOWN_COUNT_INC);
+
+            /* Change the value */
+            Value = ExpChangeRundown(RunRef, NewValue, Value);
+            if (Value == NewValue) return;
+
+            /* Loop again if we're still not active */
+            if (Value & EX_RUNDOWN_ACTIVE) break;
+        }
+    }
+
+    /* Get the wait block */
+    WaitBlock = (PEX_RUNDOWN_WAIT_BLOCK)(Value & ~EX_RUNDOWN_ACTIVE);
+    ASSERT((WaitBlock->Count >= Count) || (KeNumberProcessors > 1));
+
+    /* Remove the count */
+    if (InterlockedExchangeAddSizeT(WaitBlock->Count, -(LONG)Count) ==
+        (LONG)Count)
+    {
+        /* We're down to 0 now, so signal the event */
+        KeSetEvent(&WaitBlock->RundownEvent, IO_NO_INCREMENT, FALSE);
+    }
+}
+
+/*++
+ * @name ExfWaitForRundownProtectionRelease
+ * @implemented NT5.1
+ *
+ *     The ExfWaitForRundownProtectionRelease routine waits until the specified
+ *     rundown descriptor has been released.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return None.
+ *
+ * @remarks Callers of ExfWaitForRundownProtectionRelease must be running
+ *          at IRQL <= APC_LEVEL.
+ *
+ *--*/
+VOID
+FASTCALL
+ExfWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
+{
+    ULONG_PTR Value, Count, NewValue;
+    EX_RUNDOWN_WAIT_BLOCK WaitBlock;
+    PEX_RUNDOWN_WAIT_BLOCK WaitBlockPointer;
+    PKEVENT Event;
+    PAGED_CODE();
+
+    /* Set the active bit */
+    Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
+    if ((Value == 0) || (Value == EX_RUNDOWN_ACTIVE)) return;
+
+    /* No event for now */
+    Event = NULL;
+    WaitBlockPointer = (PEX_RUNDOWN_WAIT_BLOCK)((ULONG_PTR)&WaitBlock |
+                                                EX_RUNDOWN_ACTIVE);
+
+    /* Start waitblock set loop */
+    for(;;)
+    {
+        /* Save the count */
+        Count = Value >> EX_RUNDOWN_COUNT_SHIFT;
+
+        /* If the count is over one or we don't have en event yet, create it */
+        if (Count || !Event)
+        {
+            /* Initialize the event */
+            KeInitializeEvent(&WaitBlock.RundownEvent,
+                              NotificationEvent,
+                              FALSE);
+
+            /* Set the pointer */
+            Event = &WaitBlock.RundownEvent;
+        }
+
+        /* Set the count */
+        WaitBlock.Count = Count;
+
+        /* Now set the pointer */
+        NewValue = ExpChangeRundown(RunRef, PtrToUlong(WaitBlockPointer), Value);
+        if (NewValue == Value) break;
+
+        /* Loop again */
+        Value = NewValue;
+        ASSERT((Value & EX_RUNDOWN_ACTIVE) == 0);
+    }
+
+    /* If the count was 0, we're done */
+    if (!Count) return;
+
+    /* Wait for whoever needs to release to notify us */
+    KeWaitForSingleObject(Event, Executive, KernelMode, FALSE, NULL);
+    ASSERT(WaitBlock.Count == 0);
+}
+
+/* FIXME: STUBS **************************************************************/
+
+/*
+ * @unimplemented NT5.2
+ */
+BOOLEAN
+FASTCALL
+ExfAcquireRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    UNIMPLEMENTED;
+    return FALSE;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+BOOLEAN
+FASTCALL
+ExfAcquireRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
+                                        IN ULONG Count)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    DBG_UNREFERENCED_PARAMETER(Count);
+    UNIMPLEMENTED;
+    return FALSE;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+VOID
+FASTCALL
+ExfReleaseRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    UNIMPLEMENTED;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+VOID
+FASTCALL
+ExfReleaseRundownProtectionCacheAwareEx(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
+                                        IN ULONG Count)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    DBG_UNREFERENCED_PARAMETER(Count);
+    UNIMPLEMENTED;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+VOID
+FASTCALL
+ExfWaitForRundownProtectionReleaseCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    UNIMPLEMENTED;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+VOID
+FASTCALL
+ExfRundownCompletedCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    UNIMPLEMENTED;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+VOID
+FASTCALL
+ExfReInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    UNIMPLEMENTED;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+PEX_RUNDOWN_REF_CACHE_AWARE
+NTAPI
+ExAllocateCacheAwareRundownProtection(IN POOL_TYPE PoolType,
+                                      IN ULONG Tag)
+{
+    DBG_UNREFERENCED_PARAMETER(PoolType);
+    DBG_UNREFERENCED_PARAMETER(Tag);
+    UNIMPLEMENTED;
+    return NULL;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+VOID
+NTAPI
+ExFreeCacheAwareRundownProtection(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    UNIMPLEMENTED;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+VOID
+NTAPI
+ExInitializeRundownProtectionCacheAware(IN PEX_RUNDOWN_REF_CACHE_AWARE RunRefCacheAware,
+                                        IN ULONG Count)
+{
+    DBG_UNREFERENCED_PARAMETER(RunRefCacheAware);
+    DBG_UNREFERENCED_PARAMETER(Count);
+    UNIMPLEMENTED;
+}
+
+/*
+ * @unimplemented NT5.2
+ */
+SIZE_T
+NTAPI
+ExSizeOfRundownProtectionCacheAware(VOID)
+{
+    UNIMPLEMENTED;
+    return 0;
 }
 
-/* EOF */

reactos/ntoskrnl/include/internal/ex.h

@@ -77,6 +77,58 @@ VOID
 STDCALL
 ExpInitializeProfileImplementation(VOID);
 
+/* Rundown Functions ********************************************************/
+
+VOID
+FASTCALL
+ExfInitializeRundownProtection(
+     OUT PEX_RUNDOWN_REF RunRef
+);
+
+VOID
+FASTCALL
+ExfReInitializeRundownProtection(
+     OUT PEX_RUNDOWN_REF RunRef
+);
+
+BOOLEAN
+FASTCALL
+ExfAcquireRundownProtection(
+     IN OUT PEX_RUNDOWN_REF RunRef
+);
+
+BOOLEAN
+FASTCALL
+ExfAcquireRundownProtectionEx(
+     IN OUT PEX_RUNDOWN_REF RunRef,
+     IN ULONG Count
+);
+
+VOID
+FASTCALL
+ExfReleaseRundownProtection(
+     IN OUT PEX_RUNDOWN_REF RunRef
+);
+
+VOID
+FASTCALL
+ExfReleaseRundownProtectionEx(
+     IN OUT PEX_RUNDOWN_REF RunRef,
+     IN ULONG Count
+);
+
+VOID
+FASTCALL
+ExfRundownCompleted(
+     OUT PEX_RUNDOWN_REF RunRef
+);
+
+VOID
+FASTCALL
+ExfWaitForRundownProtectionRelease(
+     IN OUT PEX_RUNDOWN_REF RunRef
+);
+
 /* HANDLE TABLE FUNCTIONS ***************************************************/
 
 #define EX_HANDLE_ENTRY_LOCKED (1 << ((sizeof(PVOID) * 8) - 1))
@@ -192,6 +244,184 @@ static __inline _SEH_FILTER(_SEH_ExSystemExceptionFilter)
     return ExSystemExceptionFilter();
 }
 
+/* RUNDOWN *******************************************************************/
+
+#ifdef _WIN64
+#define ExpChangeRundown(x, y, z) InterlockedCompareExchange64((PLONGLONG)x, y, z)
+#define ExpSetRundown(x, y) InterlockedExchange64((PLONGLONG)x, y)
+#else
+#define ExpChangeRundown(x, y, z) InterlockedCompareExchange((PLONG)x, y, z)
+#define ExpSetRundown(x, y) InterlockedExchange((PLONG)x, y)
+#endif
+
+/*++
+ * @name ExfAcquireRundownProtection
+ * INTERNAL MACRO
+ *
+ *     The ExfAcquireRundownProtection routine acquires rundown protection for
+ *     the specified descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return TRUE if access to the protected structure was granted, FALSE otherwise.
+ *
+ * @remarks This is the internal macro for system use only.In case the rundown
+ *          was active, then the slow-path will be called through the exported
+ *          function.
+ *
+ *--*/
+BOOLEAN
+FORCEINLINE
+ExAcquireRundownProtection(IN PEX_RUNDOWN_REF RunRef)
+{
+    ULONG_PTR Value, NewValue, OldValue;
+
+    /* Get the current value and mask the active bit */
+    Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
+
+    /* Add a reference */
+    NewValue = Value + EX_RUNDOWN_COUNT_INC;
+
+    /* Change the value */
+    OldValue = ExpChangeRundown(RunRef, NewValue, Value);
+    if (OldValue != Value)
+    {
+        /* Rundown was active, use long path */
+        return ExfAcquireRundownProtection(RunRef);
+    }
+
+    /* Success */
+    return TRUE;
+}
+
+/*++
+ * @name ExReleaseRundownProtection
+ * INTERNAL MACRO
+ *
+ *     The ExReleaseRundownProtection routine releases rundown protection for
+ *     the specified descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return TRUE if access to the protected structure was granted, FALSE otherwise.
+ *
+ * @remarks This is the internal macro for system use only.In case the rundown
+ *          was active, then the slow-path will be called through the exported
+ *          function.
+ *
+ *--*/
+VOID
+FORCEINLINE
+ExReleaseRundownProtection(IN PEX_RUNDOWN_REF RunRef)
+{
+    ULONG_PTR Value, NewValue, OldValue;
+
+    /* Get the current value and mask the active bit */
+    Value = RunRef->Count &~ EX_RUNDOWN_ACTIVE;
+
+    /* Remove a reference */
+    NewValue = Value - EX_RUNDOWN_COUNT_INC;
+
+    /* Change the value */
+    OldValue = ExpChangeRundown(RunRef, NewValue, Value);
+
+    /* Check if the rundown was active */
+    if (OldValue != Value)
+    {
+        /* Rundown was active, use long path */
+        ExfReleaseRundownProtection(RunRef);
+    }
+    else
+    {
+        /* Sanity check */
+        ASSERT((Value >= EX_RUNDOWN_COUNT_INC) || (KeNumberProcessors > 1));
+    }
+}
+
+/*++
+ * @name ExInitializeRundownProtection
+ * INTERNAL MACRO
+ *
+ *     The ExInitializeRundownProtection routine initializes a rundown
+ *     protection descriptor.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return None.
+ *
+ * @remarks This is the internal macro for system use only.
+ *
+ *--*/
+VOID
+FORCEINLINE
+ExInitializeRundownProtection(IN PEX_RUNDOWN_REF RunRef)
+{
+    /* Set the count to zero */
+    RunRef->Count = 0;
+}
+
+/*++
+ * @name ExWaitForRundownProtectionRelease
+ * INTERNAL MACRO
+ *
+ *     The ExWaitForRundownProtectionRelease routine waits until the specified
+ *     rundown descriptor has been released.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return None.
+ *
+ * @remarks This is the internal macro for system use only. If a wait is actually
+ *          necessary, then the slow path is taken through the exported function.
+ *
+ *--*/
+VOID
+FORCEINLINE
+ExWaitForRundownProtectionRelease(IN PEX_RUNDOWN_REF RunRef)
+{
+    ULONG_PTR Value;
+
+    /* Set the active bit */
+    Value = ExpChangeRundown(RunRef, EX_RUNDOWN_ACTIVE, 0);
+    if ((Value) || (Value != EX_RUNDOWN_ACTIVE))
+    {
+        /* If the the rundown wasn't already active, then take the long path */
+        ExfWaitForRundownProtectionRelease(RunRef);
+    }
+}
+
+/*++
+ * @name ExRundownCompleted
+ * INTERNAL MACRO
+ *
+ *     The ExRundownCompleted routine completes the rundown of the specified
+ *     descriptor by setting the active bit.
+ *
+ * @param RunRef
+ *        Pointer to a rundown reference descriptor.
+ *
+ * @return None.
+ *
+ * @remarks This is the internal macro for system use only.
+ *
+ *--*/
+VOID
+FORCEINLINE
+ExRundownCompleted(IN PEX_RUNDOWN_REF RunRef)
+{
+    /* Sanity check */
+    ASSERT((RunRef->Count & EX_RUNDOWN_ACTIVE) != 0);
+
+    /* Mark the counter as active */
+    ExpSetRundown(&RunRef->Count, EX_RUNDOWN_ACTIVE);
+}
+
+
+
 /* OTHER FUNCTIONS **********************************************************/
 
 LONGLONG

reactos/ntoskrnl/ntoskrnl.def

@@ -65,11 +65,28 @@ DbgQueryDebugFilterState@8
 DbgSetDebugFilterState@12
 @ExiAcquireFastMutex@4=@ExAcquireFastMutex@4
 @ExAcquireFastMutexUnsafe@4
+ExAcquireRundownProtection=@ExfAcquireRundownProtection@4
+ExAcquireRundownProtectionCacheAware=@ExfAcquireRundownProtectionCacheAware@4
+ExAcquireRundownProtectionCacheAwareEx=@ExfAcquireRundownProtectionCacheAwareEx@8
+ExAcquireRundownProtectionEx=@ExfAcquireRundownProtectionEx@8
+ExInitializeRundownProtection=@ExfInitializeRundownProtection@4
+ExReInitializeRundownProtection=@ExfReInitializeRundownProtection@4
+ExReInitializeRundownProtectionCacheAware=@ExfReInitializeRundownProtectionCacheAware@4
+ExReleaseRundownProtection=@ExfReleaseRundownProtection@4
+ExReleaseRundownProtectionCacheAware=@ExfReleaseRundownProtectionCacheAware@4
+ExReleaseRundownProtectionCacheAwareEx=@ExfReleaseRundownProtectionCacheAwareEx@8
+ExReleaseRundownProtectionEx=@ExfReleaseRundownProtectionEx@8
+ExRundownCompleted=@ExfRundownCompleted@4
+ExRundownCompletedCacheAware=@ExfRundownCompletedCacheAware@4
+ExWaitForRundownProtectionRelease=@ExfWaitForRundownProtectionRelease@4
+ExWaitForRundownProtectionReleaseCacheAware=@ExfWaitForRundownProtectionReleaseCacheAware@4
+ExAllocateCacheAwareRundownProtection@8
+ExFreeCacheAwareRundownProtection@4
+ExInitializeRundownProtectionCacheAware@8
+ExSizeOfRundownProtectionCacheAware@0
 ExAcquireResourceExclusive@8
 ExAcquireResourceExclusiveLite@8
 ExAcquireResourceSharedLite@8
-@ExAcquireRundownProtection@4
-@ExAcquireRundownProtectionEx@8
 ExAcquireSharedStarveExclusive@8
 ExAcquireSharedWaitForExclusive@8
 ExAllocateFromPagedLookasideList@4=ExiAllocateFromPagedLookasideList@4
@@ -98,7 +115,6 @@ ExGetCurrentProcessorCpuUsage@4
 ExGetExclusiveWaiterCount@4
 ExGetPreviousMode@0
 ExGetSharedWaiterCount@4
-@ExInitializeRundownProtection@4
 ExInitializeNPagedLookasideList@28
 ExInitializePagedLookasideList@28
 ExInitializeResource@4
@@ -134,16 +150,12 @@ ExRaiseHardError@24
 ExRaiseStatus@4=RtlRaiseStatus@4
 ExRegisterCallback@12
 ExReinitializeResourceLite@4
-@ExReInitializeRundownProtection@4
 @ExiReleaseFastMutex@4=@ExReleaseFastMutex@4
 @ExReleaseFastMutexUnsafe@4
 @ExReleaseFastMutexUnsafeAndLeaveCriticalRegion@4
 ExReleaseResourceForThread@8
 ExReleaseResourceForThreadLite@8
 @ExReleaseResourceLite@4
-@ExReleaseRundownProtection@4
-@ExReleaseRundownProtectionEx@8
-@ExRundownCompleted@4
 ExSemaphoreObjectType DATA
 ExSetResourceOwnerPointer@8
 ExSetTimerResolution@8
@@ -153,11 +165,7 @@ ExTryToAcquireResourceExclusiveLite@4
 ExUnregisterCallback@4
 ExUuidCreate@4
 ExVerifySuite@4
-@ExWaitForRundownProtectionRelease@4
 ExWindowStationObjectType DATA
-@ExfAcquirePushLockExclusive@4
-@ExfAcquirePushLockShared@4
-@ExfReleasePushLock@4
 @ExfInterlockedAddUlong@12
 @ExfInterlockedInsertHeadList@12
 @ExfInterlockedInsertTailList@12

reactos/ntoskrnl/ntoskrnl.xml

@@ -146,7 +146,6 @@
 			<file>resource.c</file>
 			<file>rundown.c</file>
 			<file>sem.c</file>
-			<file>synch.c</file>
 			<file>sysinfo.c</file>
 			<file>time.c</file>
 			<file>timer.c</file>

reactos/w32api/include/winnt.h

@@ -3692,6 +3692,53 @@ static __inline PVOID GetFiberData(void)
 	return *((PVOID *)GetCurrentFiber());
 }
 
+#if defined(__GNUC__)
+
+static __inline__ BOOLEAN
+InterlockedBitTestAndSet(IN LONG *Base,
+                         IN LONG Bit)
+{
+	LONG OldBit;
+
+	__asm__ __volatile__("lock"
+	                     "btsl %2,%1\n\t"
+	                     "sbbl %0,%0\n\t"
+		             :"=r" (OldBit),"=m" (*Base)
+		             :"Ir" (Bit)
+			     : "memory");
+	return OldBit;
+}
+
+static __inline__ BOOLEAN
+InterlockedBitTestAndReset(IN LONG *Base,
+                          IN LONG Bit)
+{
+	LONG OldBit;
+
+	__asm__ __volatile__("lock"
+	                     "btrl %2,%1\n\t"
+	                     "sbbl %0,%0\n\t"
+		             :"=r" (OldBit),"=m" (*Base)
+		             :"Ir" (Bit)
+			     : "memory");
+	return OldBit;
+}
+
+#endif
+
+#if defined(_AMD64_)
+#if defined(_M_AMD64)
+
+#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd64((LONG64 *)a, b)
+
+#endif
+
+#else
+
+#define InterlockedExchangeAddSizeT(a, b) InterlockedExchangeAdd((LONG *)a, b)
+
+#endif
+
 #endif /* RC_INVOKED */
 
 #ifdef __cplusplus