- Implement KxAcquireSpinLock/KxReleaseSpinLock for inline acquisition of spinlocks, SMP, UP and DBG. (on DBG builds, we OR the spinlock value with the KTHREAD address, then compare on release.)

- Make In-Stack QSLs a complete no-op in UP, we were still touching some fields.
- Cleanup and re-organize spinlock.c

svn path=/trunk/; revision=24090

reactos/ntoskrnl/include/internal/ke_x.h

@@ -292,6 +292,71 @@ KiRecalculateDueTime(IN PLARGE_INTEGER OriginalDueTime,
     }                                                                       \
 }
 
+//
+// Spinlock Acquisition at IRQL >= DISPATCH_LEVEL
+//
+FORCEINLINE
+VOID
+KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+#ifdef CONFIG_SMP
+    for (;;)
+    {
+        /* Try to acquire it */
+        if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))
+        {
+            /* Value changed... wait until it's locked */
+            while (*(volatile KSPIN_LOCK *)SpinLock == 1)
+            {
+#ifdef DBG
+                /* On debug builds, we use a much slower but useful routine */
+                Kii386SpinOnSpinLock(SpinLock, 5);
+#else
+                /* Otherwise, just yield and keep looping */
+                YieldProcessor();
+#endif
+            }
+        }
+        else
+        {
+#ifdef DBG
+            /* On debug builds, we OR in the KTHREAD */
+            *SpinLock = KeGetCurrentThread() | 1;
+#endif
+            /* All is well, break out */
+            break;
+        }
+    }
+#else
+    /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
+    UNREFERENCED_PARAMETER(SpinLock);
+#endif
+}
+
+//
+// Spinlock Release at IRQL >= DISPATCH_LEVEL
+//
+FORCEINLINE
+VOID
+KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+#ifdef CONFIG_SMP
+#ifdef DBG
+    /* Make sure that the threads match */
+    if ((KeGetCurrentThread() | 1) != *SpinLock)
+    {
+        /* They don't, bugcheck */
+        KeBugCheckEx(SPIN_LOCK_NOT_OWNED, SpinLock, 0, 0, 0);
+    }
+#endif
+    /* Clear the lock */
+    InterlockedAnd(SpinLock, 0);
+#else
+    /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */
+    UNREFERENCED_PARAMETER(SpinLock);
+#endif
+}
+
 //
 // Thread Scheduling Routines
 //

reactos/ntoskrnl/ke/spinlock.c

@@ -1,197 +1,21 @@
 /*
- * COPYRIGHT:       See COPYING in the top level directory
  * PROJECT:         ReactOS Kernel
+ * LICENSE:         GPL - See COPYING in the top level directory
  * FILE:            ntoskrnl/ke/spinlock.c
- * PURPOSE:         Implements spinlocks
+ * PURPOSE:         Spinlock and Queued Spinlock Support
- * PROGRAMMERS:     Alex Ionescu (alex@relsoft.net)
+ * PROGRAMMERS:     Alex Ionescu (alex.ionescu@reactos.org)
- *                  David Welch (welch@cwcom.net)
  */
 
-/* INCLUDES ****************************************************************/
+/* INCLUDES ******************************************************************/
 
 #include <ntoskrnl.h>
 #define NDEBUG
 #include <internal/debug.h>
 
-#undef KefAcquireSpinLockAtDpcLevel
-#undef KeAcquireSpinLockAtDpcLevel
-#undef KefReleaseSpinLockFromDpcLevel
-#undef KeReleaseSpinLockFromDpcLevel
-
 #define LQ_WAIT     1
 #define LQ_OWN      2
 
-/* FUNCTIONS ***************************************************************/
+/* PRIVATE FUNCTIONS *********************************************************/
-
-/*
- * @implemented
- *
- * FUNCTION: Synchronizes the execution of a given routine with the ISR
- * of a given interrupt object
- * ARGUMENTS:
- *       Interrupt = Interrupt object to synchronize with
- *       SynchronizeRoutine = Routine to call whose execution is
- *                            synchronized with the ISR
- *       SynchronizeContext = Parameter to pass to the synchronized routine
- * RETURNS: TRUE if the operation succeeded
- */
-BOOLEAN
-STDCALL
-KeSynchronizeExecution(PKINTERRUPT Interrupt,
-                       PKSYNCHRONIZE_ROUTINE SynchronizeRoutine,
-                       PVOID SynchronizeContext)
-{
-    KIRQL OldIrql;
-    BOOLEAN Status;
-
-    /* Raise IRQL and acquire lock on MP */
-    OldIrql = KeAcquireInterruptSpinLock(Interrupt);
-
-    /* Call the routine */
-    Status = SynchronizeRoutine(SynchronizeContext);
-
-    /* Release lock and lower IRQL */
-    KeReleaseInterruptSpinLock(Interrupt, OldIrql);
-
-    /* Return routine status */
-    return Status;
-}
-
-/*
- * @implemented
- */
-KIRQL
-STDCALL
-KeAcquireInterruptSpinLock(IN PKINTERRUPT Interrupt)
-{
-    KIRQL OldIrql;
-
-    /* Raise IRQL */
-    KeRaiseIrql(Interrupt->SynchronizeIrql, &OldIrql);
-
-    /* Acquire spinlock on MP */
-    KiAcquireSpinLock(Interrupt->ActualLock);
-    return OldIrql;
-}
-
-/*
- * @implemented
- *
- * FUNCTION: Initalizes a spinlock
- * ARGUMENTS:
- *           SpinLock = Caller supplied storage for the spinlock
- */
-VOID
-STDCALL
-KeInitializeSpinLock(PKSPIN_LOCK SpinLock)
-{
-    *SpinLock = 0;
-}
-
-/*
- * @implemented
- */
-VOID
-FASTCALL
-KefAcquireSpinLockAtDpcLevel(PKSPIN_LOCK SpinLock)
-{
-    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
-    KiAcquireSpinLock(SpinLock);
-}
-
-/*
- * @implemented
- *
- * FUNCTION: Acquires a spinlock when the caller is already running at
- * dispatch level
- * ARGUMENTS:
- *        SpinLock = Spinlock to acquire
- */
-VOID
-STDCALL
-KeAcquireSpinLockAtDpcLevel (PKSPIN_LOCK SpinLock)
-{
-    KefAcquireSpinLockAtDpcLevel(SpinLock);
-}
-
-/*
- * @implemented
- */
-VOID
-FASTCALL
-KefReleaseSpinLockFromDpcLevel(PKSPIN_LOCK SpinLock)
-{
-    ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL);
-    KiReleaseSpinLock(SpinLock);
-}
-
-/*
- * @implemented
- *
- * FUNCTION: Releases a spinlock when the caller was running at dispatch
- * level before acquiring it
- * ARGUMENTS:
- *         SpinLock = Spinlock to release
- */
-VOID
-STDCALL
-KeReleaseSpinLockFromDpcLevel (PKSPIN_LOCK SpinLock)
-{
-    KefReleaseSpinLockFromDpcLevel(SpinLock);
-}
-
-/*
- * @implemented
- */
-VOID
-FASTCALL
-KiAcquireSpinLock(PKSPIN_LOCK SpinLock)
-{
-#ifdef CONFIG_SMP
-    for (;;)
-    {
-        /* Try to acquire it */
-        if (InterlockedBitTestAndSet((PLONG)SpinLock, 0))
-        {
-            /* Value changed... wait until it's locked */
-            while (*(volatile KSPIN_LOCK *)SpinLock == 1) YieldProcessor();
-        }
-        else
-        {
-            /* All is well, break out */
-            break;
-        }
-    }
-#endif /* CONFIG_SMP */
-}
-
-/*
- * @implemented
- */
-VOID
-STDCALL
-KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt,
-                           IN KIRQL OldIrql)
-{
-    /* Release lock on MP */
-    KiReleaseSpinLock(Interrupt->ActualLock);
-
-    /* Lower IRQL */
-    KeLowerIrql(OldIrql);
-}
-
-/*
- * @implemented
- */
-VOID
-FASTCALL
-KiReleaseSpinLock(PKSPIN_LOCK SpinLock)
-{
-#ifdef CONFIG_SMP
-    /* Simply clear it */
-    *SpinLock = 0;
-#endif
-}
 
 VOID
 FASTCALL
@@ -260,6 +84,87 @@ KeReleaseQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
 #endif
 }
 
+/* PUBLIC FUNCTIONS **********************************************************/
+
+/*
+ * @implemented
+ */
+VOID
+NTAPI
+KeInitializeSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+    /* Clear it */
+    *SpinLock = 0;
+}
+
+/*
+ * @implemented
+ */
+#undef KeAcquireSpinLockAtDpcLevel
+VOID
+NTAPI
+KeAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock)
+{
+    /* Do the inlined function */
+    KxAcquireSpinLock(SpinLock);
+}
+
+/*
+ * @implemented
+ */
+#undef KeReleaseSpinLockFromDpcLevel
+VOID
+NTAPI
+KeReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock)
+{
+    /* Do the lined function */
+    KxReleaseSpinLock(SpinLock);
+}
+
+/*
+ * @implemented
+ */
+VOID
+FASTCALL
+KefAcquireSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock)
+{
+    /* Do the inlined function */
+    KxAcquireSpinLock(SpinLock);
+}
+
+/*
+ * @implemented
+ */
+VOID
+FASTCALL
+KefReleaseSpinLockFromDpcLevel(IN PKSPIN_LOCK SpinLock)
+{
+    /* Do the lined function */
+    KxReleaseSpinLock(SpinLock);
+}
+
+/*
+ * @implemented
+ */
+VOID
+FASTCALL
+KiAcquireSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+    /* Do the inlined function */
+    KxAcquireSpinLock(SpinLock);
+}
+
+/*
+ * @implemented
+ */
+VOID
+FASTCALL
+KiReleaseSpinLock(IN PKSPIN_LOCK SpinLock)
+{
+    /* Do the lined function */
+    KxReleaseSpinLock(SpinLock);
+}
+
 /*
  * @implemented
  */
@@ -268,11 +173,13 @@ FASTCALL
 KeAcquireInStackQueuedSpinLockAtDpcLevel(IN PKSPIN_LOCK SpinLock,
                                          IN PKLOCK_QUEUE_HANDLE LockHandle)
 {
+#ifdef CONFIG_SMP
     /* Set it up properly */
     LockHandle->LockQueue.Next = NULL;
     LockHandle->LockQueue.Lock = SpinLock;
     KeAcquireQueuedSpinLockAtDpcLevel((PKLOCK_QUEUE_HANDLE)
                                       &LockHandle->LockQueue.Next);
+#endif
 }
 
 /*
@@ -282,9 +189,68 @@ VOID
 FASTCALL
 KeReleaseInStackQueuedSpinLockFromDpcLevel(IN PKLOCK_QUEUE_HANDLE LockHandle)
 {
+#ifdef CONFIG_SMP
     /* Call the internal function */
     KeReleaseQueuedSpinLockFromDpcLevel((PKLOCK_QUEUE_HANDLE)
                                         &LockHandle->LockQueue.Next);
+#endif
+}
+
+/*
+ * @implemented
+ */
+KIRQL
+NTAPI
+KeAcquireInterruptSpinLock(IN PKINTERRUPT Interrupt)
+{
+    KIRQL OldIrql;
+
+    /* Raise IRQL */
+    KeRaiseIrql(Interrupt->SynchronizeIrql, &OldIrql);
+
+    /* Acquire spinlock on MP */
+    KefAcquireSpinLockAtDpcLevel(Interrupt->ActualLock);
+    return OldIrql;
+}
+
+/*
+ * @implemented
+ */
+BOOLEAN
+NTAPI
+KeSynchronizeExecution(IN PKINTERRUPT Interrupt,
+                       IN PKSYNCHRONIZE_ROUTINE SynchronizeRoutine,
+                       IN PVOID SynchronizeContext)
+{
+    KIRQL OldIrql;
+    BOOLEAN Status;
+
+    /* Raise IRQL and acquire lock on MP */
+    OldIrql = KeAcquireInterruptSpinLock(Interrupt);
+
+    /* Call the routine */
+    Status = SynchronizeRoutine(SynchronizeContext);
+
+    /* Release lock and lower IRQL */
+    KeReleaseInterruptSpinLock(Interrupt, OldIrql);
+
+    /* Return routine status */
+    return Status;
+}
+
+/*
+ * @implemented
+ */
+VOID
+STDCALL
+KeReleaseInterruptSpinLock(IN PKINTERRUPT Interrupt,
+                           IN KIRQL OldIrql)
+{
+    /* Release lock on MP */
+    KefReleaseSpinLock(Interrupt->ActualLock);
+
+    /* Lower IRQL */
+    KeLowerIrql(OldIrql);
 }
 
 /* EOF */