From b19da39918def7fac16588033b1517f52c8daccd Mon Sep 17 00:00:00 2001 From: Timo Kreuzer Date: Sat, 4 Jun 2011 12:33:54 +0000 Subject: [PATCH] [NTOSKRNL/HAL] - Add explicit memory barriers to KxAcquireSpinLock, KxReleaseSpinLock inline functions and KeTryToAcquireQueuedSpinLock, KeTryToAcquireQueuedSpinLockRaiseToSynch in the UP case. This will prevent the compiler from reordering memory access instructions across the boundaries of these functions, even when being inlined. - Use the inline functions in x64 spinlock functions, too svn path=/trunk/; revision=52078 --- reactos/hal/halx86/generic/spinlock.c | 8 ++ reactos/ntoskrnl/include/internal/spinlock.h | 8 ++ reactos/ntoskrnl/ke/amd64/spinlock.c | 105 +++++++++++-------- 3 files changed, 75 insertions(+), 46 deletions(-) diff --git a/reactos/hal/halx86/generic/spinlock.c b/reactos/hal/halx86/generic/spinlock.c index 1aab8d5a314..26829ef438f 100644 --- a/reactos/hal/halx86/generic/spinlock.c +++ b/reactos/hal/halx86/generic/spinlock.c @@ -188,6 +188,10 @@ KeTryToAcquireQueuedSpinLockRaiseToSynch(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber, /* Simply raise to synch */ KeRaiseIrql(SYNCH_LEVEL, OldIrql); + /* Add an explicit memory barrier to prevent the compiler from reordering + memory accesses across the borders of spinlocks */ + _ReadWriteBarrier(); + /* Always return true on UP Machines */ return TRUE; } @@ -208,6 +212,10 @@ KeTryToAcquireQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber, /* Simply raise to dispatch */ KeRaiseIrql(DISPATCH_LEVEL, OldIrql); + /* Add an explicit memory barrier to prevent the compiler from reordering + memory accesses across the borders of spinlocks */ + _ReadWriteBarrier(); + /* Always return true on UP Machines */ return TRUE; } diff --git a/reactos/ntoskrnl/include/internal/spinlock.h b/reactos/ntoskrnl/include/internal/spinlock.h index 6d1ca71c231..e2d9fd0ff83 100644 --- a/reactos/ntoskrnl/include/internal/spinlock.h +++ b/reactos/ntoskrnl/include/internal/spinlock.h @@ -21,6 +21,10 @@ KxAcquireSpinLock(IN PKSPIN_LOCK SpinLock) { /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ UNREFERENCED_PARAMETER(SpinLock); + + /* Add an explicit memory barrier to prevent the compiler from reordering + memory accesses across the borders of spinlocks */ + _ReadWriteBarrier(); } // @@ -32,6 +36,10 @@ KxReleaseSpinLock(IN PKSPIN_LOCK SpinLock) { /* On UP builds, spinlocks don't exist at IRQL >= DISPATCH */ UNREFERENCED_PARAMETER(SpinLock); + + /* Add an explicit memory barrier to prevent the compiler from reordering + memory accesses across the borders of spinlocks */ + _ReadWriteBarrier(); } #else diff --git a/reactos/ntoskrnl/ke/amd64/spinlock.c b/reactos/ntoskrnl/ke/amd64/spinlock.c index 9c485fd8d43..7909c1ac7f4 100644 --- a/reactos/ntoskrnl/ke/amd64/spinlock.c +++ b/reactos/ntoskrnl/ke/amd64/spinlock.c @@ -23,14 +23,14 @@ KIRQL KeAcquireSpinLockRaiseToSynch(PKSPIN_LOCK SpinLock) { -#ifndef CONFIG_SMP KIRQL OldIrql; - /* Simply raise to dispatch */ - KeRaiseIrql(DISPATCH_LEVEL, &OldIrql); + + /* Raise to sync */ + KeRaiseIrql(SYNCH_LEVEL, &OldIrql); + + /* Acquire the lock and return */ + KxAcquireSpinLock(SpinLock); return OldIrql; -#else - UNIMPLEMENTED; -#endif } /* @@ -40,14 +40,14 @@ KIRQL NTAPI KeAcquireSpinLockRaiseToDpc(PKSPIN_LOCK SpinLock) { -#ifndef CONFIG_SMP KIRQL OldIrql; - /* Simply raise to dispatch */ + + /* Raise to dispatch */ KeRaiseIrql(DISPATCH_LEVEL, &OldIrql); + + /* Acquire the lock and return */ + KxAcquireSpinLock(SpinLock); return OldIrql; -#else - UNIMPLEMENTED; -#endif } /* @@ -58,12 +58,9 @@ NTAPI KeReleaseSpinLock(PKSPIN_LOCK SpinLock, KIRQL OldIrql) { -#ifndef CONFIG_SMP - /* Simply lower IRQL back */ + /* Release the lock and lower IRQL back */ + KxReleaseSpinLock(SpinLock); KeLowerIrql(OldIrql); -#else - UNIMPLEMENTED; -#endif } /* @@ -72,14 +69,14 @@ KeReleaseSpinLock(PKSPIN_LOCK SpinLock, KIRQL KeAcquireQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber) { -#ifndef CONFIG_SMP KIRQL OldIrql; - /* Simply raise to dispatch */ + + /* Raise to dispatch */ KeRaiseIrql(DISPATCH_LEVEL, &OldIrql); + + /* Acquire the lock */ + KxAcquireSpinLock(KeGetCurrentPrcb()->LockQueue[LockNumber].Lock); // HACK return OldIrql; -#else - UNIMPLEMENTED; -#endif } /* @@ -88,14 +85,14 @@ KeAcquireQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber) KIRQL KeAcquireQueuedSpinLockRaiseToSynch(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber) { -#ifndef CONFIG_SMP KIRQL OldIrql; - /* Simply raise to dispatch */ - KeRaiseIrql(DISPATCH_LEVEL, &OldIrql); + + /* Raise to synch */ + KeRaiseIrql(SYNCH_LEVEL, &OldIrql); + + /* Acquire the lock */ + KxAcquireSpinLock(KeGetCurrentPrcb()->LockQueue[LockNumber].Lock); // HACK return OldIrql; -#else - UNIMPLEMENTED; -#endif } /* @@ -105,14 +102,18 @@ VOID KeAcquireInStackQueuedSpinLock(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle) { -#ifndef CONFIG_SMP - /* Simply raise to dispatch */ + /* Set up the lock */ + LockHandle->LockQueue.Next = NULL; + LockHandle->LockQueue.Lock = SpinLock; + + /* Raise to dispatch */ KeRaiseIrql(DISPATCH_LEVEL, &LockHandle->OldIrql); -#else - UNIMPLEMENTED; -#endif + + /* Acquire the lock */ + KxAcquireSpinLock(LockHandle->LockQueue.Lock); // HACK } + /* * @implemented */ @@ -120,14 +121,18 @@ VOID KeAcquireInStackQueuedSpinLockRaiseToSynch(IN PKSPIN_LOCK SpinLock, IN PKLOCK_QUEUE_HANDLE LockHandle) { -#ifndef CONFIG_SMP - /* Simply raise to synch */ + /* Set up the lock */ + LockHandle->LockQueue.Next = NULL; + LockHandle->LockQueue.Lock = SpinLock; + + /* Raise to synch */ KeRaiseIrql(SYNCH_LEVEL, &LockHandle->OldIrql); -#else - UNIMPLEMENTED; -#endif + + /* Acquire the lock */ + KxAcquireSpinLock(LockHandle->LockQueue.Lock); // HACK } + /* * @implemented */ @@ -135,28 +140,26 @@ VOID KeReleaseQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber, IN KIRQL OldIrql) { -#ifndef CONFIG_SMP - /* Simply lower IRQL back */ + /* Release the lock */ + KxReleaseSpinLock(KeGetCurrentPrcb()->LockQueue[LockNumber].Lock); // HACK + + /* Lower IRQL back */ KeLowerIrql(OldIrql); -#else - UNIMPLEMENTED; -#endif } + /* * @implemented */ VOID KeReleaseInStackQueuedSpinLock(IN PKLOCK_QUEUE_HANDLE LockHandle) { -#ifndef CONFIG_SMP /* Simply lower IRQL back */ + KxReleaseSpinLock(LockHandle->LockQueue.Lock); // HACK KeLowerIrql(LockHandle->OldIrql); -#else - UNIMPLEMENTED; -#endif } + /* * @implemented */ @@ -168,10 +171,15 @@ KeTryToAcquireQueuedSpinLockRaiseToSynch(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber, /* Simply raise to dispatch */ KeRaiseIrql(DISPATCH_LEVEL, OldIrql); + /* Add an explicit memory barrier to prevent the compiler from reordering + memory accesses across the borders of spinlocks */ + _ReadWriteBarrier(); + /* Always return true on UP Machines */ return TRUE; #else UNIMPLEMENTED; + ASSERT(FALSE); #endif } @@ -186,10 +194,15 @@ KeTryToAcquireQueuedSpinLock(IN KSPIN_LOCK_QUEUE_NUMBER LockNumber, /* Simply raise to dispatch */ KeRaiseIrql(DISPATCH_LEVEL, OldIrql); + /* Add an explicit memory barrier to prevent the compiler from reordering + memory accesses across the borders of spinlocks */ + _ReadWriteBarrier(); + /* Always return true on UP Machines */ return TRUE; #else UNIMPLEMENTED; + ASSERT(FALSE); #endif }