reactos/ntoskrnl/ex/i386/fastinterlck_asm.S

806 lines
15 KiB
ArmAsm
Raw Normal View History

/*
* COPYRIGHT: See COPYING in the top level directory
* PROJECT: ReactOS kernel
* FILE: ntoskrnl/ex/i386/fastinterlck_asm.S
* PURPOSE: FASTCALL Interlocked Functions
* PROGRAMMERS: Alex Ionescu (alex@relsoft.net)
*/
/* INCLUDES ******************************************************************/
#include <reactos/asm.h>
#include <ndk/asm.h>
#include <internal/i386/asmmacro.S>
/* FUNCTIONS ****************************************************************/
.code32
.text
/*
* NOTE: These functions must obey the following rules:
* - Acquire locks only on MP systems.
* - Be safe at HIGH_LEVEL (no paged access).
* - Preserve flags.
* - Disable interrups.
*/
/*VOID
*FASTCALL
*ExInterlockedAddLargeStatistic(IN PLARGE_INTEGER Addend,
* IN ULONG Increment)
*/
.global @ExInterlockedAddLargeStatistic@8
@ExInterlockedAddLargeStatistic@8:
#ifdef CONFIG_SMP
/* Do the addition */
lock add [ecx], edx
/* Check for carry bit and return */
jb 1f
ret
1:
/* Add carry */
lock adc dword ptr [ecx+4], 0
#else
/* Do the addition and add the carry */
add dword ptr [ecx], edx
adc dword ptr [ecx+4], 0
#endif
/* Return */
ret
/*ULONG
*FASTCALL
*ExfInterlockedAddUlong(IN PULONG Addend,
* IN ULONG Increment,
* IN PKSPIN_LOCK Lock)
*/
.global @ExfInterlockedAddUlong@12
@ExfInterlockedAddUlong@12:
/* Save flags */
pushfd
#ifdef CONFIG_SMP
/* Get lock address */
mov eax, [esp+8]
.start1:
#endif
/* Disable interrupts */
cli
/* Acquire lock */
ACQUIRE_SPINLOCK(eax, .spin1)
/* Do the add */
mov eax, [ecx]
add [ecx], edx
#ifdef CONFIG_SMP
/* Get spinlock address and release it */
mov edx, [esp+8]
RELEASE_SPINLOCK(edx)
#endif
/* Restore flags and return */
popfd
ret 4
#ifdef CONFIG_SMP
.spin1:
/* Restore flags and spin */
popfd
pushfd
SPIN_ON_LOCK(eax, .start1)
#endif
/*PLIST_ENTRY
*FASTCALL
*ExfInterlockedInsertHeadList(IN PLIST_ENTRY ListHead,
* IN PLIST_ENTRY ListEntry,
* IN PKSPIN_LOCK Lock)
*/
.global @ExfInterlockedInsertHeadList@12
@ExfInterlockedInsertHeadList@12:
#ifdef CONFIG_SMP
/* Save lock address */
push esi
mov esi, [esp+8]
#endif
/* Save flags and disable interrupts */
pushfd
.start2:
cli
/* Acquire lock */
ACQUIRE_SPINLOCK(esi, .spin2)
/* Get list pointer */
mov eax, [ecx]
/* Do the insert */
mov [edx], eax
mov [edx+4], ecx
mov [ecx], edx
mov [eax+4], edx
/* Release lock and restore flags */
RELEASE_SPINLOCK(esi)
popfd
#ifdef CONFIG_SMP
pop esi
#endif
/* Check if list was empty */
xor eax, ecx
jz 2f
/* Return list pointer */
xor eax, ecx
2:
ret 4
#ifdef CONFIG_SMP
.spin2:
popfd
pushfd
SPIN_ON_LOCK(esi, .start2)
#endif
/*PLIST_ENTRY
*FASTCALL
*ExfInterlockedInsertTailList(IN PLIST_ENTRY ListHead,
* IN PLIST_ENTRY ListEntry,
* IN PKSPIN_LOCK Lock)
*/
.global @ExfInterlockedInsertTailList@12
@ExfInterlockedInsertTailList@12:
#ifdef CONFIG_SMP
/* Save lock address */
push esi
mov esi, [esp+8]
#endif
/* Save flags and disable interrupts */
pushfd
.start3:
cli
/* Acquire lock */
ACQUIRE_SPINLOCK(esi, .spin3)
/* Get list pointer */
mov eax, [ecx+4]
/* Do the insert */
mov [edx], ecx
mov [edx+4], eax
mov [ecx+4], edx
mov [eax], edx
/* Release lock and restore flags */
RELEASE_SPINLOCK(esi)
popfd
#ifdef CONFIG_SMP
pop esi
#endif
/* Check if list was empty */
xor eax, ecx
jz 2f
/* Return list pointer */
xor eax, ecx
2:
ret 4
#ifdef CONFIG_SMP
.spin3:
popfd
pushfd
SPIN_ON_LOCK(esi, .start3)
#endif
/*PLIST_ENTRY
*FASTCALL
*ExfInterlockedRemoveHeadList(IN PLIST_ENTRY ListHead,
* IN PKSPIN_LOCK Lock)
*/
.global @ExfInterlockedRemoveHeadList@8
@ExfInterlockedRemoveHeadList@8:
/* Save flags and disable interrupts */
.start4:
pushfd
cli
ACQUIRE_SPINLOCK(edx, .spin4)
/* Get list pointer */
mov eax, [ecx]
/* Check if it's empty */
cmp eax, ecx
je 2f
/* Get the next entry and do the deletion */
#ifdef CONFIG_SMP
push ebx
mov ebx, [eax]
mov [ecx], ebx
mov [ebx+4], ecx
#else
mov edx, [eax]
mov [ecx], edx
mov [edx+4], ecx
#endif
/* Release lock */
#ifdef CONFIG_SMP
RELEASE_SPINLOCK(edx)
pop ebx
#endif
/* Restore flags */
popfd
/* Return */
ret
2:
/* Release lock */
RELEASE_SPINLOCK(edx)
/* Restore flags */
popfd
/* Return empty list */
xor eax, eax
ret
#ifdef CONFIG_SMP
.spin4:
popfd
SPIN_ON_LOCK(edx, .start4)
#endif
/*PSINGLE_LIST_ENTRY
*FASTCALL
*ExfInterlockedPopEntryList(IN PSINGLE_LIST_ENTRY ListHead,
* IN PKSPIN_LOCK Lock)
*/
.global @ExfInterlockedPopEntryList@8
@ExfInterlockedPopEntryList@8:
/* Save flags and disable interrupts */
.start5:
pushfd
cli
ACQUIRE_SPINLOCK(edx, .spin5)
/* Get list pointer */
mov eax, [ecx]
/* Check if it's empty */
or eax, eax
je 3f
/* Get next entry and do deletion */
#ifdef CONFIG_SMP
push edx
#endif
mov edx, [eax]
mov [ecx], edx
#ifdef CONFIG_SMP
pop edx
#endif
2:
/* Release lock */
RELEASE_SPINLOCK(edx)
/* Restore flags */
popfd
/* Return */
ret
3:
/* Return empty list */
xor eax, eax
jmp 2b
#ifdef CONFIG_SMP
.spin5:
popfd
SPIN_ON_LOCK(edx, .start5)
#endif
/*PSINGLE_LIST_ENTRY
*FASTCALL
*ExfInterlockedPushEntryList(IN PSINGLE_LIST_ENTRY ListHead,
* IN PSINGLE_LIST_ENTRY ListEntry,
* IN PKSPIN_LOCK Lock)
*/
.global @ExfInterlockedPushEntryList@12
@ExfInterlockedPushEntryList@12:
/* Save flags */
pushfd
/* Save lock pointer */
#ifdef CONFIG_SMP
push edx
mov edx, [esp+12]
#endif
/* Disable interrupts */
.start6:
cli
#ifdef CONFIG_SMP
ACQUIRE_SPINLOCK(edx, .spin6)
pop edx
#endif
/* Get list pointer */
mov eax, [ecx]
/* Do push */
mov [edx], eax
mov [ecx], edx
/* Release lock */
#ifdef CONFIG_SMP
mov edx, [esp+8]
RELEASE_SPINLOCK(edx)
#endif
/* Restore flags */
popfd
/* Return */
ret 4
#ifdef CONFIG_SMP
.spin6:
pop edx
popfd
pushfd
push edx
mov edx, [esp+12]
SPIN_ON_LOCK(edx, .start6)
#endif
/*PSINGLE_LIST_ENTRY
*FASTCALL
*ExInterlockedPopEntrySList(IN PSINGLE_LIST_ENTRY ListHead,
* IN PKSPIN_LOCK Lock)
*/
.global @ExInterlockedPopEntrySList@8
.global @InterlockedPopEntrySList@4
.global _ExpInterlockedPopEntrySListResume@0
.global _ExpInterlockedPopEntrySListFault@0
.global _ExpInterlockedPopEntrySListEnd@0
@ExInterlockedPopEntrySList@8:
@InterlockedPopEntrySList@4:
/* Save registers */
push ebx
push ebp
/* Pointer to list */
mov ebp, ecx
/* Get sequence number and link pointer */
_ExpInterlockedPopEntrySListResume@0:
mov edx, [ebp+4]
mov eax, [ebp]
/* Check if the list is empty */
or eax, eax
jz 2f
/* Copy sequence number and adjust it */
lea ecx, [edx-1]
/* Get next pointer and do the exchange */
_ExpInterlockedPopEntrySListFault@0:
mov ebx, [eax]
_ExpInterlockedPopEntrySListEnd@0:
LOCK cmpxchg8b qword ptr [ebp]
jnz _ExpInterlockedPopEntrySListResume@0
/* Restore registers and return */
2:
pop ebp
pop ebx
ret
/*PSINGLE_LIST_ENTRY
*FASTCALL
*ExInterlockedPushEntrySList(IN PSINGLE_LIST_ENTRY ListHead,
* IN PSINGLE_LIST_ENTRY ListEntry,
* IN PKSPIN_LOCK Lock)
*/
.global @ExInterlockedPushEntrySList@12
@ExInterlockedPushEntrySList@12:
/* So we can fall through below */
pop [esp]
.global @InterlockedPushEntrySList@8
@InterlockedPushEntrySList@8:
/* Save registers */
push ebx
push ebp
/* Pointer to list */
mov ebp, ecx
mov ebx, edx
/* Get sequence number and link pointer */
mov edx, [ebp+4]
mov eax, [ebp]
1:
/* Set link pointer */
mov [ebx], eax
/* Copy sequence number and adjust it */
lea ecx, [edx+0x10001]
/* Do the exchange */
LOCK cmpxchg8b qword ptr [ebp]
jnz 1b
/* Restore registers and return */
2:
pop ebp
pop ebx
ret
/*PSINGLE_LIST_ENTRY
*FASTCALL
*ExInterlockedFlushSList(IN PSINGLE_LIST_ENTRY ListHead)
*/
.global @ExInterlockedFlushSList@4
@ExInterlockedFlushSList@4:
/* Save registers */
push ebx
push ebp
/* Clear ebx */
xor ebx, ebx
/* Pointer to list */
mov ebp, ecx
/* Get sequence number and link pointer */
mov edx, [ebp+4]
mov eax, [ebp]
1:
/* Check if the list is empty */
or eax, eax
jz 2f
/* Clear sequence and pointer */
mov ecx, edx
mov cx, bx
/* Do the exchange */
LOCK cmpxchg8b qword ptr [ebp]
jnz 1b
/* Restore registers and return */
2:
pop ebp
pop ebx
ret
/*INTERLOCKED_RESULT
*FASTCALL
*Exfi386InterlockedIncrementLong(IN PLONG Addend)
*/
.global @Exfi386InterlockedIncrementLong@4
@Exfi386InterlockedIncrementLong@4:
/* Do the op */
LOCK add dword ptr [ecx], 1
/* Return */
lahf
and eax, EFLAG_SELECT
ret
/*INTERLOCKED_RESULT
*FASTCALL
*Exfi386InterlockedDecrementLong(IN PLONG Addend)
*/
.global @Exfi386InterlockedDecrementLong@4
@Exfi386InterlockedDecrementLong@4:
/* Do the op */
LOCK sub dword ptr [ecx], 1
/* Return */
lahf
and eax, EFLAG_SELECT
ret
/*ULONG
*FASTCALL
*Exfi386InterlockedExchangeUlong(IN PULONG Taget,
* IN ULONG Value)
*/
.global @Exfi386InterlockedExchangeUlong@8
@Exfi386InterlockedExchangeUlong@8:
#ifdef CONFIG_SMP
/* On MP, do the exchange */
xchg [ecx], edx
mov eax, edx
#else
/* On UP, use cmpxchg */
mov eax, [ecx]
1:
cmpxchg [ecx], edx
jnz 1b
#endif
/* Return */
ret
/*PVOID
*FASTCALL
*ExfInterlockedCompareExchange64(IN PLONGLONG Destination,
* IN PLONGLONG Exchange,
* IN PLONGLONG Comperand)
*/
.global @ExfInterlockedCompareExchange64@12
@ExfInterlockedCompareExchange64@12:
/* Save registers */
push ebx
push ebp
/* Get destination pointer, exchange value and comperand value/address */
mov ebp, ecx
mov ebx, [edx]
mov ecx, [edx+4]
mov edx, [esp+12]
mov eax, [edx]
mov edx, [edx+4]
/* Do the op */
LOCK cmpxchg8b qword ptr [ebp]
/* Restore volatiles */
pop ebp
pop ebx
/* Return */
ret 4
/*PVOID
*FASTCALL
*ExfInterlockedCompareExchange64(IN PLONGLONG Destination,
* IN PLONGLONG Exchange,
* IN PLONGLONG Comperand,
* IN PKSPIN_LOCK Lock)
*/
.global @ExInterlockedCompareExchange64@16
@ExInterlockedCompareExchange64@16:
/* Save registers */
push ebp
push ebp
/* Get destination pointer, exchange value and comperand value/address */
mov ebp, ecx
mov ebx, [edx]
mov ecx, [edx+4]
mov edx, [esp+12]
mov eax, [edx]
mov edx, [edx+4]
/* Do the op */
LOCK cmpxchg8b qword ptr [ebp]
/* Restore volatiles */
pop ebp
pop ebx
/* Return */
ret 8
/*** Non-586 functions ***/
/*PSINGLE_LIST_ENTRY
*FASTCALL
*ExfInterlockedPopEntrySList(IN PSINGLE_LIST_ENTRY ListHead,
* IN PKSPIN_LOCK Lock)
*/
.global @ExfInterlockedPopEntrySList@8
@ExfInterlockedPopEntrySList@8:
/* Save flags */
.starta:
pushfd
/* Disable interrupts */
cli
/* Acquire lock */
ACQUIRE_SPINLOCK(edx, .spina)
/* Get the next link and check if it's empty */
mov eax, [ecx]
or eax, eax
jz 1f
/* Get address of the next link and store it */
push [eax]
pop [ecx]
/* Decrement list depth */
dec dword ptr [ecx+4]
1:
#ifdef CONFIG_SMP
/* Release spinlock */
RELEASE_SPINLOCK(edx)
#endif
/* Restore flags and return */
popfd
ret 0
#ifdef CONFIG_SMP
.spina:
/* Restore flags and spin */
popfd
SPIN_ON_LOCK(edx, .starta)
#endif
/*PSINGLE_LIST_ENTRY
*FASTCALL
*ExfInterlockedPushEntrySList(IN PSINGLE_LIST_ENTRY ListHead,
* IN PSINGLE_LIST_ENTRY ListEntry,
* IN PKSPIN_LOCK Lock)
*/
.global @ExfInterlockedPushEntrySList@12
@ExfInterlockedPushEntrySList@12:
/* Save flags */
.startb:
pushfd
/* Disable interrupts */
cli
/* Acquire lock */
#ifndef CONFIG_SMP
mov eax, [esp+8]
ACQUIRE_SPINLOCK(eax, .spinb)
#endif
/* Get the next link and check if it's empty */
push [ecx]
/* Get address of the next link and store it */
pop [edx]
mov [ecx], edx
/* Increment list depth */
inc dword ptr [ecx+4]
#ifdef CONFIG_SMP
/* Release spinlock */
RELEASE_SPINLOCK(eax)
#endif
/* Restore flags and return */
popfd
ret 4
#ifdef CONFIG_SMP
.spinb:
/* Restore flags and spin */
popfd
SPIN_ON_LOCK(eax, .startb)
#endif
/*PVOID
*FASTCALL
*ExpInterlockedCompareExchange64(IN PLONGLONG Destination,
* IN PLONGLONG Exchange,
* IN PLONGLONG Comperand,
* IN PKSPIN_LOCK Lock)
*/
.global @ExpInterlockedCompareExchange64@16
@ExpInterlockedCompareExchange64@16:
/* Save registers */
push ebp
push ebp
/* Get destination pointer, exchange value and comperand value/address */
mov ebp, ecx
mov ebx, [edx]
mov ecx, [edx+4]
mov edx, [esp+12]
mov eax, [edx]
mov edx, [edx+4]
#ifdef CONFIG_SMP
/* Save ESI so we can store KSPINLOCK in it */
push esi
/* Save flags and lock, and disable interrupts */
pushfd
mov esi, [esp+24]
.startc:
cli
/* Acquire the spinlock */
ACQUIRE_SPINLOCK(esi, .spinc)
#else
/* Save flags and disable interrupts */
pushfd
cli
#endif
/* Compare bottom */
cmp eax, [ebp]
jne NoMatch
/* Compare top */
cmp edx, [ebp+4]
jne NoMatch
/* Save new value */
mov [ebp], ebx
mov [ebp+4], ecx
AfterSave:
#ifdef CONFIG_SMP
/* Release lock, restore volatiles and flags */
RELEASE_SPINLOCK(esi)
popfd
pop esi
#else
popfd
#endif
/* Restore the other volatiles and return */
pop ebp
pop ebx
/* Return */
ret 8
NoMatch:
/* Return the current value */
mov eax, [ebp]
mov edx, [ebp+4]
jmp AfterSave
#ifdef CONFIG_SMP
.spinc:
/* Restore flags and spin */
popfd
pushfd
SPIN_ON_LOCK(esi, .startc)
#endif
/* EOF */