Intravision/reactos
1
0
Fork 0
mirror of https://github.com/reactos/reactos.git synced 2024-11-06 06:33:08 +00:00
Code Issues Projects Releases Packages Wiki Activity
reactos/ntoskrnl/ex/pushlock.c
Amine Khaldi 527f2f9057 [SHELL/EXPERIMENTS] 
* Create a branch for some evul shell experiments.

svn path=/branches/shell-experiments/; revision=61927
2014-02-02 19:37:27 +00:00

1247 lines
38 KiB
C
Raw Blame History

/*
* COPYRIGHT: See COPYING in the top level directory
* PROJECT: ReactOS Kernel
* FILE: ntoskrnl/ex/pushlock.c
* PURPOSE: Pushlock and Cache-Aware Pushlock Implementation
* PROGRAMMER: Alex Ionescu (alex.ionescu@reactos.com)
*/
/* INCLUDES *****************************************************************/
#include <ntoskrnl.h>
#define NDEBUG
#include <debug.h>
/* DATA **********************************************************************/
ULONG ExPushLockSpinCount = 0;
#undef EX_PUSH_LOCK
#undef PEX_PUSH_LOCK
/* PRIVATE FUNCTIONS *********************************************************/
#ifdef _WIN64
#define InterlockedAndPointer(ptr,val) InterlockedAnd64((PLONGLONG)ptr,(LONGLONG)val)
#else
#define InterlockedAndPointer(ptr,val) InterlockedAnd((PLONG)ptr,(LONG)val)
#endif
/*++
* @name ExpInitializePushLocks
*
* The ExpInitializePushLocks routine initialized Pushlock support.
*
* @param None.
*
* @return None.
*
* @remarks The ExpInitializePushLocks routine sets up the spin on SMP machines.
*
*--*/
VOID
NTAPI
INIT_FUNCTION
ExpInitializePushLocks(VOID)
{
#ifdef CONFIG_SMP
/* Initialize an internal 1024-iteration spin for MP CPUs */
if (KeNumberProcessors > 1)
ExPushLockSpinCount = 1024;
#endif
}
/*++
* @name ExfWakePushLock
*
* The ExfWakePushLock routine wakes a Pushlock that is in the waiting
* state.
*
* @param PushLock
* Pointer to a pushlock that is waiting.
*
* @param OldValue
* Last known value of the pushlock before this routine was called.
*
* @return None.
*
* @remarks This is an internal routine; do not call it manually. Only the system
* can properly know if the pushlock is ready to be awakened or not.
* External callers should use ExfTrytoWakePushLock.
*
*--*/
VOID
FASTCALL
ExfWakePushLock(PEX_PUSH_LOCK PushLock,
EX_PUSH_LOCK OldValue)
{
EX_PUSH_LOCK NewValue;
PEX_PUSH_LOCK_WAIT_BLOCK PreviousWaitBlock, FirstWaitBlock, LastWaitBlock;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock;
KIRQL OldIrql;
/* Start main wake loop */
for (;;)
{
/* Sanity checks */
ASSERT(!OldValue.MultipleShared);
/* Check if it's locked */
while (OldValue.Locked)
{
/* It's not waking anymore */
NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
/* Sanity checks */
ASSERT(!NewValue.Waking);
ASSERT(NewValue.Locked);
ASSERT(NewValue.Waiting);
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
/* Someone changed the value behind our back, update it*/
OldValue = NewValue;
}
/* Save the First Block */
FirstWaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value &
~EX_PUSH_LOCK_PTR_BITS);
WaitBlock = FirstWaitBlock;
/* Try to find the last block */
while (TRUE)
{
/* Get the last wait block */
LastWaitBlock = WaitBlock->Last;
/* Check if we found it */
if (LastWaitBlock)
{
/* Use it */
WaitBlock = LastWaitBlock;
break;
}
/* Save the previous block */
PreviousWaitBlock = WaitBlock;
/* Move to next block */
WaitBlock = WaitBlock->Next;
/* Save the previous block */
WaitBlock->Previous = PreviousWaitBlock;
}
/* Check if the last Wait Block is not Exclusive or if it's the only one */
PreviousWaitBlock = WaitBlock->Previous;
if (!(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE) ||
!(PreviousWaitBlock))
{
/* Destroy the pushlock */
NewValue.Value = 0;
ASSERT(!NewValue.Waking);
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) break;
/* Someone changed the value behind our back, update it*/
OldValue = NewValue;
}
else
{
/* Link the wait blocks */
FirstWaitBlock->Last = PreviousWaitBlock;
WaitBlock->Previous = NULL;
/* Sanity checks */
ASSERT(FirstWaitBlock != WaitBlock);
ASSERT(PushLock->Waiting);
/* Remove waking bit from pushlock */
InterlockedAndPointer(&PushLock->Value, ~EX_PUSH_LOCK_WAKING);
/* Leave the loop */
break;
}
}
/* Check if there's a previous block */
OldIrql = DISPATCH_LEVEL;
if (WaitBlock->Previous)
{
/* Raise to Dispatch */
KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
}
/* Signaling loop */
for (;;)
{
/* Get the previous Wait block */
PreviousWaitBlock = WaitBlock->Previous;
/* Sanity check */
ASSERT(!WaitBlock->Signaled);
#if DBG
/* We are about to get signaled */
WaitBlock->Signaled = TRUE;
#endif
/* Set the Wait Bit in the Wait Block */
if (!InterlockedBitTestAndReset(&WaitBlock->Flags, 1))
{
/* Nobody signaled us, so do it */
KeSignalGateBoostPriority(&WaitBlock->WakeGate);
}
/* Set the wait block and check if there still is one to loop*/
WaitBlock = PreviousWaitBlock;
if (!WaitBlock) break;
}
/* Check if we have to lower back the IRQL */
if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql);
}
/*++
* @name ExpOptimizePushLockList
*
* The ExpOptimizePushLockList routine optimizes the list of waiters
* associated to a pushlock's wait block.
*
* @param PushLock
* Pointer to a pushlock whose waiter list needs to be optimized.
*
* @param OldValue
* Last known value of the pushlock before this routine was called.
*
* @return None.
*
* @remarks At the end of the optimization, the pushlock will also be wakened.
*
*--*/
VOID
FASTCALL
ExpOptimizePushLockList(PEX_PUSH_LOCK PushLock,
EX_PUSH_LOCK OldValue)
{
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock, PreviousWaitBlock, FirstWaitBlock;
EX_PUSH_LOCK NewValue;
/* Start main loop */
for (;;)
{
/* Check if we've been unlocked */
if (!OldValue.Locked)
{
/* Wake us up and leave */
ExfWakePushLock(PushLock, OldValue);
break;
}
/* Get the wait block */
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value &
~EX_PUSH_LOCK_PTR_BITS);
/* Loop the blocks */
FirstWaitBlock = WaitBlock;
while (TRUE)
{
/* Get the last wait block */
LastWaitBlock = WaitBlock->Last;
if (LastWaitBlock)
{
/* Set this as the new last block, we're done */
FirstWaitBlock->Last = LastWaitBlock;
break;
}
/* Save the block */
PreviousWaitBlock = WaitBlock;
/* Get the next block */
WaitBlock = WaitBlock->Next;
/* Save the previous */
WaitBlock->Previous = PreviousWaitBlock;
}
/* Remove the wake bit */
NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_WAKING;
/* Sanity checks */
ASSERT(NewValue.Locked);
ASSERT(!NewValue.Waking);
/* Update the value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
/* If we updated correctly, leave */
if (NewValue.Value == OldValue.Value) break;
/* Update value */
OldValue = NewValue;
}
}
/*++
* @name ExTimedWaitForUnblockPushLock
*
* The ExTimedWaitForUnblockPushLock routine waits for a pushlock
* to be unblocked, for a specified internal.
*
* @param PushLock
* Pointer to a pushlock whose waiter list needs to be optimized.
*
* @param WaitBlock
* Pointer to the pushlock's wait block.
*
* @param Timeout
* Amount of time to wait for this pushlock to be unblocked.
*
* @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error
* code returned by KeWaitForSingleObject.
*
* @remarks If the wait fails, then a manual unblock is attempted.
*
*--*/
NTSTATUS
FASTCALL
ExTimedWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock,
IN PVOID WaitBlock,
IN PLARGE_INTEGER Timeout)
{
NTSTATUS Status;
/* Initialize the wait event */
KeInitializeEvent(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->WakeEvent,
SynchronizationEvent,
FALSE);
#ifdef CONFIG_SMP
/* Spin on the push lock if necessary */
if (ExPushLockSpinCount)
{
ULONG i = ExPushLockSpinCount;
do
{
/* Check if we got lucky and can leave early */
if (!(*(volatile LONG *)&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags & EX_PUSH_LOCK_WAITING))
return STATUS_SUCCESS;
YieldProcessor();
} while (--i);
}
#endif
/* Now try to remove the wait bit */
if (InterlockedBitTestAndReset(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->Flags,
EX_PUSH_LOCK_FLAGS_WAIT_V))
{
/* Nobody removed it already, let's do a full wait */
Status = KeWaitForSingleObject(&((PEX_PUSH_LOCK_WAIT_BLOCK)WaitBlock)->
WakeEvent,
WrPushLock,
KernelMode,
FALSE,
Timeout);
/* Check if the wait was satisfied */
if (Status != STATUS_SUCCESS)
{
/* Try unblocking the pushlock if it was not */
ExfUnblockPushLock(PushLock, WaitBlock);
}
}
else
{
/* Someone beat us to it, no need to wait */
Status = STATUS_SUCCESS;
}
/* Return status */
return Status;
}
/*++
* @name ExWaitForUnblockPushLock
*
* The ExWaitForUnblockPushLock routine waits for a pushlock
* to be unblocked, for a specified internal.
*
* @param PushLock
* Pointer to a pushlock whose waiter list needs to be optimized.
*
* @param WaitBlock
* Pointer to the pushlock's wait block.
*
* @return STATUS_SUCCESS is the pushlock is now unblocked, otherwise the error
* code returned by KeWaitForSingleObject.
*
* @remarks If the wait fails, then a manual unblock is attempted.
*
*--*/
VOID
FASTCALL
ExWaitForUnblockPushLock(IN PEX_PUSH_LOCK PushLock,
IN PVOID WaitBlock)
{
/* Call the timed function with no timeout */
ExTimedWaitForUnblockPushLock(PushLock, WaitBlock, NULL);
}
/*++
* @name ExBlockPushLock
*
* The ExBlockPushLock routine blocks a pushlock.
*
* @param PushLock
* Pointer to a pushlock whose waiter list needs to be optimized.
*
* @param WaitBlock
* Pointer to the pushlock's wait block.
*
* @return None.
*
* @remarks None.
*
*--*/
VOID
FASTCALL
ExBlockPushLock(PEX_PUSH_LOCK PushLock,
PVOID pWaitBlock)
{
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock = pWaitBlock;
EX_PUSH_LOCK NewValue, OldValue;
/* Detect invalid wait block alignment */
ASSERT(((ULONG_PTR)pWaitBlock & 0xF) == 0);
/* Set the waiting bit */
WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_WAIT;
/* Get the old value */
OldValue = *PushLock;
/* Start block loop */
for (;;)
{
/* Link the wait blocks */
WaitBlock->Next = OldValue.Ptr;
/* Set the new wait block value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
WaitBlock,
OldValue.Ptr);
if (OldValue.Ptr == NewValue.Ptr) break;
/* Try again with the new value */
OldValue = NewValue;
}
}
/* PUBLIC FUNCTIONS **********************************************************/
/*++
* @name ExAcquirePushLockExclusive
* @implemented NT5.1
*
* The ExAcquirePushLockExclusive macro exclusively acquires a PushLock.
*
* @params PushLock
* Pointer to the pushlock which is to be acquired.
*
* @return None.
*
* @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeAcquireCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfAcquirePushLockExclusive(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue, TempValue;
BOOLEAN NeedWake;
EX_PUSH_LOCK_WAIT_BLOCK Block;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock = &Block;
/* Start main loop */
for (;;)
{
/* Check if it's unlocked */
if (!OldValue.Locked)
{
/* Lock it */
NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
ASSERT(NewValue.Locked);
/* Set the new value */
if (InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr) != OldValue.Ptr)
{
/* Retry */
OldValue = *PushLock;
continue;
}
/* Break out of the loop */
break;
}
else
{
/* We'll have to create a Waitblock */
WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_EXCLUSIVE |
EX_PUSH_LOCK_FLAGS_WAIT;
WaitBlock->Previous = NULL;
NeedWake = FALSE;
/* Check if there is already a waiter */
if (OldValue.Waiting)
{
/* Nobody is the last waiter yet */
WaitBlock->Last = NULL;
/* We are an exclusive waiter */
WaitBlock->ShareCount = 0;
/* Set the current Wait Block pointer */
WaitBlock->Next = (PEX_PUSH_LOCK_WAIT_BLOCK)(
OldValue.Value &~ EX_PUSH_LOCK_PTR_BITS);
/* Point to ours */
NewValue.Value = (OldValue.Value & EX_PUSH_LOCK_MULTIPLE_SHARED) |
EX_PUSH_LOCK_LOCK |
EX_PUSH_LOCK_WAKING |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
/* Check if the pushlock was already waking */
if (!OldValue.Waking) NeedWake = TRUE;
}
else
{
/* We are the first waiter, so loop the wait block */
WaitBlock->Last = WaitBlock;
/* Set the share count */
WaitBlock->ShareCount = (LONG)OldValue.Shared;
/* Check if someone is sharing this pushlock */
if (OldValue.Shared > 1)
{
/* Point to our wait block */
NewValue.Value = EX_PUSH_LOCK_MULTIPLE_SHARED |
EX_PUSH_LOCK_LOCK |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
}
else
{
/* No shared count */
WaitBlock->ShareCount = 0;
/* Point to our wait block */
NewValue.Value = EX_PUSH_LOCK_LOCK |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
}
}
#if DBG
/* Setup the Debug Wait Block */
WaitBlock->Signaled = 0;
WaitBlock->OldValue = OldValue;
WaitBlock->NewValue = NewValue;
WaitBlock->PushLock = PushLock;
#endif
/* Sanity check */
ASSERT(NewValue.Waiting);
ASSERT(NewValue.Locked);
/* Write the new value */
TempValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value != OldValue.Value)
{
/* Retry */
OldValue = *PushLock;
continue;
}
/* Check if the pushlock needed waking */
if (NeedWake)
{
/* Scan the Waiters and Wake PushLocks */
ExpOptimizePushLockList(PushLock, TempValue);
}
/* Set up the Wait Gate */
KeInitializeGate(&WaitBlock->WakeGate);
#ifdef CONFIG_SMP
/* Now spin on the push lock if necessary */
if (ExPushLockSpinCount)
{
ULONG i = ExPushLockSpinCount;
do
{
if (!(*(volatile LONG *)&WaitBlock->Flags & EX_PUSH_LOCK_WAITING))
break;
YieldProcessor();
} while (--i);
}
#endif
/* Now try to remove the wait bit */
if (InterlockedBitTestAndReset(&WaitBlock->Flags, 1))
{
/* Nobody removed it already, let's do a full wait */
KeWaitForGate(&WaitBlock->WakeGate, WrPushLock, KernelMode);
ASSERT(WaitBlock->Signaled);
}
/* We shouldn't be shared anymore */
ASSERT((WaitBlock->ShareCount == 0));
/* Loop again */
OldValue = NewValue;
}
}
}
/*++
* @name ExAcquirePushLockShared
* @implemented NT5.1
*
* The ExAcquirePushLockShared routine acquires a shared PushLock.
*
* @params PushLock
* Pointer to the pushlock which is to be acquired.
*
* @return None.
*
* @remarks Callers of ExAcquirePushLockShared must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeAcquireCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfAcquirePushLockShared(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue;
BOOLEAN NeedWake;
EX_PUSH_LOCK_WAIT_BLOCK Block;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock = &Block;
/* Start main loop */
for (;;)
{
/* Check if it's unlocked or if it's waiting without any sharers */
if (!(OldValue.Locked) || (!(OldValue.Waiting) && (OldValue.Shared > 0)))
{
/* Check if anyone is waiting on it */
if (!OldValue.Waiting)
{
/* Increase the share count and lock it */
NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
NewValue.Shared++;
}
else
{
/* Simply set the lock bit */
NewValue.Value = OldValue.Value | EX_PUSH_LOCK_LOCK;
}
/* Sanity check */
ASSERT(NewValue.Locked);
/* Set the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value != OldValue.Value)
{
/* Retry */
OldValue = *PushLock;
continue;
}
/* Break out of the loop */
break;
}
else
{
/* We'll have to create a Waitblock */
WaitBlock->Flags = EX_PUSH_LOCK_FLAGS_WAIT;
WaitBlock->ShareCount = 0;
NeedWake = FALSE;
WaitBlock->Previous = NULL;
/* Check if there is already a waiter */
if (OldValue.Waiting)
{
/* Set the current Wait Block pointer */
WaitBlock->Next = (PEX_PUSH_LOCK_WAIT_BLOCK)(
OldValue.Value &~ EX_PUSH_LOCK_PTR_BITS);
/* Nobody is the last waiter yet */
WaitBlock->Last = NULL;
/* Point to ours */
NewValue.Value = (OldValue.Value & (EX_PUSH_LOCK_MULTIPLE_SHARED |
EX_PUSH_LOCK_LOCK)) |
EX_PUSH_LOCK_WAKING |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
/* Check if the pushlock was already waking */
if (!OldValue.Waking) NeedWake = TRUE;
}
else
{
/* We are the first waiter, so loop the wait block */
WaitBlock->Last = WaitBlock;
/* Point to our wait block */
NewValue.Value = (OldValue.Value & EX_PUSH_LOCK_PTR_BITS) |
EX_PUSH_LOCK_WAITING |
(ULONG_PTR)WaitBlock;
}
/* Sanity check */
ASSERT(NewValue.Waiting);
#if DBG
/* Setup the Debug Wait Block */
WaitBlock->Signaled = 0;
WaitBlock->OldValue = OldValue;
WaitBlock->NewValue = NewValue;
WaitBlock->PushLock = PushLock;
#endif
/* Write the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Ptr != OldValue.Ptr)
{
/* Retry */
OldValue = *PushLock;
continue;
}
/* Update the value now */
OldValue = NewValue;
/* Check if the pushlock needed waking */
if (NeedWake)
{
/* Scan the Waiters and Wake PushLocks */
ExpOptimizePushLockList(PushLock, OldValue);
}
/* Set up the Wait Gate */
KeInitializeGate(&WaitBlock->WakeGate);
#ifdef CONFIG_SMP
/* Now spin on the push lock if necessary */
if (ExPushLockSpinCount)
{
ULONG i = ExPushLockSpinCount;
do
{
if (!(*(volatile LONG *)&WaitBlock->Flags & EX_PUSH_LOCK_WAITING))
break;
YieldProcessor();
} while (--i);
}
#endif
/* Now try to remove the wait bit */
if (InterlockedBitTestAndReset(&WaitBlock->Flags, 1))
{
/* Fast-path did not work, we need to do a full wait */
KeWaitForGate(&WaitBlock->WakeGate, WrPushLock, KernelMode);
ASSERT(WaitBlock->Signaled);
}
/* We shouldn't be shared anymore */
ASSERT((WaitBlock->ShareCount == 0));
}
}
}
/*++
* @name ExfReleasePushLock
* @implemented NT5.1
*
* The ExReleasePushLock routine releases a previously acquired PushLock.
*
*
* @params PushLock
* Pointer to a previously acquired pushlock.
*
* @return None.
*
* @remarks Callers of ExfReleasePushLock must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeLeaveCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfReleasePushLock(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue, WakeValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock;
/* Sanity check */
ASSERT(OldValue.Locked);
/* Start main loop */
while (TRUE)
{
/* Check if someone is waiting on the lock */
if (!OldValue.Waiting)
{
/* Check if it's shared */
if (OldValue.Shared > 1)
{
/* Write the Old Value but decrease share count */
NewValue = OldValue;
NewValue.Shared--;
}
else
{
/* Simply clear the lock */
NewValue.Value = 0;
}
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
/* Did it enter a wait state? */
OldValue = NewValue;
}
else
{
/* Ok, we do know someone is waiting on it. Are there more then one? */
if (OldValue.MultipleShared)
{
/* Get the wait block */
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value &
~EX_PUSH_LOCK_PTR_BITS);
/* Loop until we find the last wait block */
while (TRUE)
{
/* Get the last wait block */
LastWaitBlock = WaitBlock->Last;
/* Did it exist? */
if (LastWaitBlock)
{
/* Choose it */
WaitBlock = LastWaitBlock;
break;
}
/* Keep searching */
WaitBlock = WaitBlock->Next;
}
/* Make sure the Share Count is above 0 */
if (WaitBlock->ShareCount > 0)
{
/* This shouldn't be an exclusive wait block */
ASSERT(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
/* Do the decrease and check if the lock isn't shared anymore */
if (InterlockedDecrement(&WaitBlock->ShareCount) > 0) return;
}
}
/*
* If nobody was waiting on the block, then we possibly reduced the number
* of times the pushlock was shared, and we unlocked it.
* If someone was waiting, and more then one person is waiting, then we
* reduced the number of times the pushlock is shared in the wait block.
* Therefore, at this point, we can now 'satisfy' the wait.
*/
for (;;)
{
/* Now we need to see if it's waking */
if (OldValue.Waking)
{
/* Remove the lock and multiple shared bits */
NewValue.Value = OldValue.Value;
NewValue.MultipleShared = FALSE;
NewValue.Locked = FALSE;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
}
else
{
/* Remove the lock and multiple shared bits */
NewValue.Value = OldValue.Value;
NewValue.MultipleShared = FALSE;
NewValue.Locked = FALSE;
/* It's not already waking, so add the wake bit */
NewValue.Waking = TRUE;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */
WakeValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value != OldValue.Value) continue;
/* The write was successful. The pushlock is Unlocked and Waking */
ExfWakePushLock(PushLock, WakeValue);
return;
}
}
}
}
}
/*++
* @name ExfReleasePushLockShared
* @implemented NT5.2
*
* The ExfReleasePushLockShared macro releases a previously acquired PushLock.
*
* @params PushLock
* Pointer to a previously acquired pushlock.
*
* @return None.
*
* @remarks Callers of ExReleasePushLockShared must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeLeaveCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfReleasePushLockShared(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue, WakeValue;
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, LastWaitBlock;
/* Check if someone is waiting on the lock */
while (!OldValue.Waiting)
{
/* Check if it's shared */
if (OldValue.Shared > 1)
{
/* Write the Old Value but decrease share count */
NewValue = OldValue;
NewValue.Shared--;
}
else
{
/* Simply clear the lock */
NewValue.Value = 0;
}
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
/* Did it enter a wait state? */
OldValue = NewValue;
}
/* Ok, we do know someone is waiting on it. Are there more then one? */
if (OldValue.MultipleShared)
{
/* Get the wait block */
WaitBlock = (PEX_PUSH_LOCK_WAIT_BLOCK)(OldValue.Value &
~EX_PUSH_LOCK_PTR_BITS);
/* Loop until we find the last wait block */
while (TRUE)
{
/* Get the last wait block */
LastWaitBlock = WaitBlock->Last;
/* Did it exist? */
if (LastWaitBlock)
{
/* Choose it */
WaitBlock = LastWaitBlock;
break;
}
/* Keep searching */
WaitBlock = WaitBlock->Next;
}
/* Sanity checks */
ASSERT(WaitBlock->ShareCount > 0);
ASSERT(WaitBlock->Flags & EX_PUSH_LOCK_FLAGS_EXCLUSIVE);
/* Do the decrease and check if the lock isn't shared anymore */
if (InterlockedDecrement(&WaitBlock->ShareCount) > 0) return;
}
/*
* If nobody was waiting on the block, then we possibly reduced the number
* of times the pushlock was shared, and we unlocked it.
* If someone was waiting, and more then one person is waiting, then we
* reduced the number of times the pushlock is shared in the wait block.
* Therefore, at this point, we can now 'satisfy' the wait.
*/
for (;;)
{
/* Now we need to see if it's waking */
if (OldValue.Waking)
{
/* Remove the lock and multiple shared bits */
NewValue.Value = OldValue.Value;
NewValue.MultipleShared = FALSE;
NewValue.Locked = FALSE;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value == OldValue.Value) return;
}
else
{
/* Remove the lock and multiple shared bits */
NewValue.Value = OldValue.Value;
NewValue.MultipleShared = FALSE;
NewValue.Locked = FALSE;
/* It's not already waking, so add the wake bit */
NewValue.Waking = TRUE;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Locked && !NewValue.MultipleShared);
/* Write the new value */
WakeValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
if (NewValue.Value != OldValue.Value) continue;
/* The write was successful. The pushlock is Unlocked and Waking */
ExfWakePushLock(PushLock, WakeValue);
return;
}
}
}
/*++
* ExfReleasePushLockExclusive
* @implemented NT5.2
*
* The ExfReleasePushLockExclusive routine releases a previously
* exclusively acquired PushLock.
*
* @params PushLock
* Pointer to a previously acquired pushlock.
*
* @return None.
*
* @remarks Callers of ExReleasePushLockExclusive must be running at IRQL <= APC_LEVEL.
* This macro should usually be paired up with KeLeaveCriticalRegion.
*
*--*/
VOID
FASTCALL
ExfReleasePushLockExclusive(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK NewValue, WakeValue;
EX_PUSH_LOCK OldValue = *PushLock;
/* Loop until we can change */
for (;;)
{
/* Sanity checks */
ASSERT(OldValue.Locked);
ASSERT(OldValue.Waiting || OldValue.Shared == 0);
/* Check if it's waiting and not yet waking */
if ((OldValue.Waiting) && !(OldValue.Waking))
{
/* Remove the lock bit, and add the wake bit */
NewValue.Value = (OldValue.Value &~ EX_PUSH_LOCK_LOCK) |
EX_PUSH_LOCK_WAKING;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Locked);
/* Write the New Value. Save our original value for waking */
WakeValue = NewValue;
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
/* Check if the value changed behind our back */
if (NewValue.Value == OldValue.Value)
{
/* Wake the Pushlock */
ExfWakePushLock(PushLock, WakeValue);
break;
}
}
else
{
/* A simple unlock */
NewValue.Value = OldValue.Value &~ EX_PUSH_LOCK_LOCK;
/* Sanity check */
ASSERT(NewValue.Waking && !NewValue.Waiting);
/* Write the New Value */
NewValue.Ptr = InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr);
/* Check if the value changed behind our back */
if (NewValue.Value == OldValue.Value) break;
}
/* Loop again */
OldValue = NewValue;
}
}
/*++
* @name ExfTryToWakePushLock
* @implemented NT5.2
*
* The ExfTryToWakePushLock attemps to wake a waiting pushlock.
*
* @param PushLock
* Pointer to a PushLock which is in the wait state.
*
* @return None.
*
* @remarks The pushlock must be in a wait state and must not be already waking.
*
*--*/
VOID
FASTCALL
ExfTryToWakePushLock(PEX_PUSH_LOCK PushLock)
{
EX_PUSH_LOCK OldValue = *PushLock, NewValue;
/*
* If the Pushlock is not waiting on anything, or if it's already waking up
* and locked, don't do anything
*/
if ((OldValue.Waking) || (OldValue.Locked) || !(OldValue.Waiting)) return;
/* Make it Waking */
NewValue = OldValue;
NewValue.Waking = TRUE;
/* Write the New Value */
if (InterlockedCompareExchangePointer(&PushLock->Ptr,
NewValue.Ptr,
OldValue.Ptr) == OldValue.Ptr)
{
/* Wake the Pushlock */
ExfWakePushLock(PushLock, NewValue);
}
}
/*++
* @name ExfUnblockPushLock
* @implemented NT5.1
*
* The ExfUnblockPushLock routine unblocks a previously blocked PushLock.
*
* @param PushLock
* Pointer to a previously blocked PushLock.
*
* @return None.
*
* @remarks Callers of ExfUnblockPushLock can be running at any IRQL.
*
*--*/
VOID
FASTCALL
ExfUnblockPushLock(PEX_PUSH_LOCK PushLock,
PVOID CurrentWaitBlock)
{
PEX_PUSH_LOCK_WAIT_BLOCK WaitBlock, NextWaitBlock;
KIRQL OldIrql = DISPATCH_LEVEL;
/* Get the wait block and erase the previous one */
WaitBlock = InterlockedExchangePointer(&PushLock->Ptr, NULL);
if (WaitBlock)
{
/* Check if there is a linked pushlock and raise IRQL appropriately */
if (WaitBlock->Next) KeRaiseIrql(DISPATCH_LEVEL, &OldIrql);
/* Start block loop */
while (WaitBlock)
{
/* Get the next block */
NextWaitBlock = WaitBlock->Next;
/* Remove the wait flag from the Wait block */
if (!InterlockedBitTestAndReset(&WaitBlock->Flags, EX_PUSH_LOCK_FLAGS_WAIT_V))
{
/* Nobody removed the flag before us, so signal the event */
KeSetEventBoostPriority(&WaitBlock->WakeEvent, NULL);
}
/* Try the next one */
WaitBlock = NextWaitBlock;
}
/* Lower IRQL if needed */
if (OldIrql != DISPATCH_LEVEL) KeLowerIrql(OldIrql);
}
/* Check if we got a wait block that's pending */
if ((CurrentWaitBlock) &&
(((PEX_PUSH_LOCK_WAIT_BLOCK)CurrentWaitBlock)->Flags &
EX_PUSH_LOCK_FLAGS_WAIT))
{
/* Wait for the pushlock to be unblocked */
ExWaitForUnblockPushLock(PushLock, CurrentWaitBlock);
}
}
Reference in a new issue View git blame Copy permalink