2010-02-26 11:43:19 +00:00
|
|
|
#pragma once
|
2002-08-14 20:58:39 +00:00
|
|
|
|
2002-09-08 10:23:54 +00:00
|
|
|
#ifndef __ASM__
|
2002-08-14 20:58:39 +00:00
|
|
|
|
2008-02-12 06:51:39 +00:00
|
|
|
#include "intrin_i.h"
|
|
|
|
|
2009-11-08 01:13:49 +00:00
|
|
|
//
|
|
|
|
// Thread Dispatcher Header DebugActive Mask
|
|
|
|
//
|
|
|
|
#define DR_MASK(x) (1 << (x))
|
|
|
|
#define DR_REG_MASK 0x4F
|
|
|
|
|
2009-10-04 16:53:15 +00:00
|
|
|
//
|
|
|
|
// INT3 is 1 byte long
|
|
|
|
//
|
2009-10-10 19:27:54 +00:00
|
|
|
#define KD_BREAKPOINT_TYPE UCHAR
|
|
|
|
#define KD_BREAKPOINT_SIZE sizeof(UCHAR)
|
|
|
|
#define KD_BREAKPOINT_VALUE 0xCC
|
2009-10-04 16:53:15 +00:00
|
|
|
|
|
|
|
//
|
|
|
|
// Macros for getting and setting special purpose registers in portable code
|
|
|
|
//
|
|
|
|
#define KeGetContextPc(Context) \
|
|
|
|
((Context)->Eip)
|
|
|
|
|
|
|
|
#define KeSetContextPc(Context, ProgramCounter) \
|
|
|
|
((Context)->Eip = (ProgramCounter))
|
|
|
|
|
|
|
|
#define KeGetTrapFramePc(TrapFrame) \
|
|
|
|
((TrapFrame)->Eip)
|
|
|
|
|
2010-02-01 03:51:45 +00:00
|
|
|
#define KiGetLinkedTrapFrame(x) \
|
|
|
|
(PKTRAP_FRAME)((x)->Edx)
|
2010-12-22 16:14:58 +00:00
|
|
|
|
2009-10-04 16:53:15 +00:00
|
|
|
#define KeGetContextReturnRegister(Context) \
|
|
|
|
((Context)->Eax)
|
|
|
|
|
|
|
|
#define KeSetContextReturnRegister(Context, ReturnValue) \
|
|
|
|
((Context)->Eax = (ReturnValue))
|
|
|
|
|
2009-12-17 20:58:58 +00:00
|
|
|
//
|
|
|
|
// Macro to get trap and exception frame from a thread stack
|
|
|
|
//
|
|
|
|
#define KeGetTrapFrame(Thread) \
|
|
|
|
(PKTRAP_FRAME)((ULONG_PTR)((Thread)->InitialStack) - \
|
|
|
|
sizeof(KTRAP_FRAME) - \
|
|
|
|
sizeof(FX_SAVE_AREA))
|
|
|
|
|
|
|
|
#define KeGetExceptionFrame(Thread) \
|
|
|
|
NULL
|
|
|
|
|
|
|
|
//
|
|
|
|
// Macro to get context switches from the PRCB
|
|
|
|
// All architectures but x86 have it in the PRCB's KeContextSwitches
|
|
|
|
//
|
|
|
|
#define KeGetContextSwitches(Prcb) \
|
|
|
|
CONTAINING_RECORD(Prcb, KIPCR, PrcbData)->ContextSwitches
|
|
|
|
|
2010-05-17 21:26:51 +00:00
|
|
|
//
|
|
|
|
// Macro to get the second level cache size field name which differs between
|
|
|
|
// CISC and RISC architectures, as the former has unified I/D cache
|
|
|
|
//
|
|
|
|
#define KiGetSecondLevelDCacheSize() ((PKIPCR)KeGetPcr())->SecondLevelCacheSize
|
|
|
|
|
2009-10-10 19:27:54 +00:00
|
|
|
//
|
|
|
|
// Returns the Interrupt State from a Trap Frame.
|
|
|
|
// ON = TRUE, OFF = FALSE
|
|
|
|
//
|
|
|
|
#define KeGetTrapFrameInterruptState(TrapFrame) \
|
|
|
|
BooleanFlagOn((TrapFrame)->EFlags, EFLAGS_INTERRUPT_MASK)
|
2010-01-10 15:00:44 +00:00
|
|
|
|
|
|
|
//
|
|
|
|
// Flags for exiting a trap
|
|
|
|
//
|
|
|
|
#define KTE_SKIP_PM_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipPreviousMode = TRUE } }).Bits)
|
|
|
|
#define KTE_SKIP_SEG_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipSegments = TRUE } }).Bits)
|
|
|
|
#define KTE_SKIP_VOL_BIT (((KTRAP_EXIT_SKIP_BITS) { { .SkipVolatiles = TRUE } }).Bits)
|
2010-12-22 16:14:58 +00:00
|
|
|
|
2010-01-10 15:00:44 +00:00
|
|
|
typedef union _KTRAP_EXIT_SKIP_BITS
|
|
|
|
{
|
|
|
|
struct
|
|
|
|
{
|
|
|
|
UCHAR SkipPreviousMode:1;
|
|
|
|
UCHAR SkipSegments:1;
|
|
|
|
UCHAR SkipVolatiles:1;
|
|
|
|
UCHAR Reserved:5;
|
|
|
|
};
|
|
|
|
UCHAR Bits;
|
|
|
|
} KTRAP_EXIT_SKIP_BITS, *PKTRAP_EXIT_SKIP_BITS;
|
2010-01-10 15:40:00 +00:00
|
|
|
|
|
|
|
|
|
|
|
//
|
|
|
|
// Flags used by the VDM/V8086 emulation engine for determining instruction prefixes
|
|
|
|
//
|
|
|
|
#define PFX_FLAG_ES 0x00000100
|
|
|
|
#define PFX_FLAG_CS 0x00000200
|
|
|
|
#define PFX_FLAG_SS 0x00000400
|
|
|
|
#define PFX_FLAG_DS 0x00000800
|
|
|
|
#define PFX_FLAG_FS 0x00001000
|
|
|
|
#define PFX_FLAG_GS 0x00002000
|
|
|
|
#define PFX_FLAG_OPER32 0x00004000
|
|
|
|
#define PFX_FLAG_ADDR32 0x00008000
|
|
|
|
#define PFX_FLAG_LOCK 0x00010000
|
|
|
|
#define PFX_FLAG_REPNE 0x00020000
|
|
|
|
#define PFX_FLAG_REP 0x00040000
|
|
|
|
|
|
|
|
//
|
|
|
|
// VDM Helper Macros
|
|
|
|
//
|
|
|
|
// All VDM/V8086 opcode emulators have the same FASTCALL function definition.
|
|
|
|
// We need to keep 2 parameters while the original ASM implementation uses 4:
|
|
|
|
// TrapFrame, PrefixFlags, Eip, InstructionSize;
|
|
|
|
//
|
|
|
|
// We pass the trap frame, and prefix flags, in our two parameters.
|
|
|
|
//
|
|
|
|
// We then realize that since the smallest prefix flag is 0x100, this gives us
|
|
|
|
// a count of up to 0xFF. So we OR in the instruction size with the prefix flags
|
|
|
|
//
|
|
|
|
// We further realize that we always have access to EIP from the trap frame, and
|
|
|
|
// that if we want the *current instruction* EIP, we simply have to add the
|
|
|
|
// instruction size *MINUS ONE*, and that gives us the EIP we should be looking
|
|
|
|
// at now, so we don't need to use the stack to push this parameter.
|
|
|
|
//
|
|
|
|
// We actually only care about the *current instruction* EIP in one location,
|
|
|
|
// so although it may be slightly more expensive to re-calculate the EIP one
|
|
|
|
// more time, this way we don't redefine ALL opcode handlers to have 3 parameters,
|
|
|
|
// which would be forcing stack usage in all other scenarios.
|
|
|
|
//
|
|
|
|
#define KiVdmSetVdmEFlags(x) InterlockedOr((PLONG)KiNtVdmState, (x));
|
|
|
|
#define KiVdmClearVdmEFlags(x) InterlockedAnd((PLONG)KiNtVdmState, ~(x))
|
|
|
|
#define KiCallVdmHandler(x) KiVdmOpcode##x(TrapFrame, Flags)
|
|
|
|
#define KiCallVdmPrefixHandler(x) KiVdmOpcodePrefix(TrapFrame, Flags | x)
|
|
|
|
#define KiVdmUnhandledOpcode(x) \
|
|
|
|
BOOLEAN \
|
|
|
|
FASTCALL \
|
|
|
|
KiVdmOpcode##x(IN PKTRAP_FRAME TrapFrame, \
|
|
|
|
IN ULONG Flags) \
|
|
|
|
{ \
|
|
|
|
/* Not yet handled */ \
|
|
|
|
UNIMPLEMENTED; \
|
|
|
|
while (TRUE); \
|
2010-01-10 17:50:50 +00:00
|
|
|
return TRUE; \
|
2010-01-10 15:40:00 +00:00
|
|
|
}
|
|
|
|
|
2010-01-11 03:47:17 +00:00
|
|
|
C_ASSERT(NPX_FRAME_LENGTH == sizeof(FX_SAVE_AREA));
|
|
|
|
|
|
|
|
//
|
|
|
|
// Local parameters
|
|
|
|
//
|
|
|
|
typedef struct _KV86_FRAME
|
|
|
|
{
|
|
|
|
PVOID ThreadStack;
|
|
|
|
PVOID ThreadTeb;
|
|
|
|
PVOID PcrTeb;
|
|
|
|
} KV86_FRAME, *PKV86_FRAME;
|
|
|
|
|
|
|
|
//
|
|
|
|
// Virtual Stack Frame
|
|
|
|
//
|
|
|
|
typedef struct _KV8086_STACK_FRAME
|
|
|
|
{
|
|
|
|
KTRAP_FRAME TrapFrame;
|
|
|
|
FX_SAVE_AREA NpxArea;
|
|
|
|
KV86_FRAME V86Frame;
|
|
|
|
} KV8086_STACK_FRAME, *PKV8086_STACK_FRAME;
|
2010-12-22 16:14:58 +00:00
|
|
|
|
2010-01-01 15:09:14 +00:00
|
|
|
//
|
|
|
|
// Registers an interrupt handler with an IDT vector
|
|
|
|
//
|
|
|
|
FORCEINLINE
|
|
|
|
VOID
|
|
|
|
KeRegisterInterruptHandler(IN ULONG Vector,
|
|
|
|
IN PVOID Handler)
|
2010-12-22 16:14:58 +00:00
|
|
|
{
|
2010-01-01 15:09:14 +00:00
|
|
|
UCHAR Entry;
|
|
|
|
ULONG_PTR Address;
|
|
|
|
PKIPCR Pcr = (PKIPCR)KeGetPcr();
|
|
|
|
|
|
|
|
//
|
|
|
|
// Get the entry from the HAL
|
|
|
|
//
|
|
|
|
Entry = HalVectorToIDTEntry(Vector);
|
|
|
|
Address = PtrToUlong(Handler);
|
|
|
|
|
|
|
|
//
|
|
|
|
// Now set the data
|
|
|
|
//
|
|
|
|
Pcr->IDT[Entry].ExtendedOffset = (USHORT)(Address >> 16);
|
|
|
|
Pcr->IDT[Entry].Offset = (USHORT)Address;
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Returns the registered interrupt handler for a given IDT vector
|
|
|
|
//
|
|
|
|
FORCEINLINE
|
|
|
|
PVOID
|
|
|
|
KeQueryInterruptHandler(IN ULONG Vector)
|
|
|
|
{
|
|
|
|
PKIPCR Pcr = (PKIPCR)KeGetPcr();
|
|
|
|
UCHAR Entry;
|
|
|
|
|
|
|
|
//
|
|
|
|
// Get the entry from the HAL
|
|
|
|
//
|
|
|
|
Entry = HalVectorToIDTEntry(Vector);
|
|
|
|
|
|
|
|
//
|
|
|
|
// Read the entry from the IDT
|
|
|
|
//
|
|
|
|
return (PVOID)(((Pcr->IDT[Entry].ExtendedOffset << 16) & 0xFFFF0000) |
|
|
|
|
(Pcr->IDT[Entry].Offset & 0xFFFF));
|
|
|
|
}
|
2009-10-10 19:27:54 +00:00
|
|
|
|
2009-11-04 22:40:18 +00:00
|
|
|
//
|
|
|
|
// Invalidates the TLB entry for a specified address
|
|
|
|
//
|
|
|
|
FORCEINLINE
|
|
|
|
VOID
|
|
|
|
KeInvalidateTlbEntry(IN PVOID Address)
|
|
|
|
{
|
|
|
|
/* Invalidate the TLB entry for this address */
|
|
|
|
__invlpg(Address);
|
|
|
|
}
|
|
|
|
|
2009-12-20 13:30:35 +00:00
|
|
|
FORCEINLINE
|
|
|
|
VOID
|
|
|
|
KeFlushProcessTb(VOID)
|
|
|
|
{
|
|
|
|
/* Flush the TLB by resetting CR3 */
|
|
|
|
__writecr3(__readcr3());
|
|
|
|
}
|
|
|
|
|
|
|
|
FORCEINLINE
|
|
|
|
PRKTHREAD
|
|
|
|
KeGetCurrentThread(VOID)
|
|
|
|
{
|
|
|
|
/* Return the current thread */
|
|
|
|
return ((PKIPCR)KeGetPcr())->PrcbData.CurrentThread;
|
|
|
|
}
|
|
|
|
|
|
|
|
FORCEINLINE
|
|
|
|
VOID
|
|
|
|
KiRundownThread(IN PKTHREAD Thread)
|
|
|
|
{
|
|
|
|
#ifndef CONFIG_SMP
|
|
|
|
/* Check if this is the NPX Thread */
|
|
|
|
if (KeGetCurrentPrcb()->NpxThread == Thread)
|
|
|
|
{
|
|
|
|
/* Clear it */
|
|
|
|
KeGetCurrentPrcb()->NpxThread = NULL;
|
|
|
|
Ke386FnInit();
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
/* Nothing to do */
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2002-08-14 20:58:39 +00:00
|
|
|
VOID
|
2006-09-30 05:42:22 +00:00
|
|
|
FASTCALL
|
|
|
|
Ki386InitializeTss(
|
|
|
|
IN PKTSS Tss,
|
2006-10-01 07:31:33 +00:00
|
|
|
IN PKIDTENTRY Idt,
|
|
|
|
IN PKGDTENTRY Gdt
|
2006-09-30 05:42:22 +00:00
|
|
|
);
|
|
|
|
|
2006-09-01 03:05:30 +00:00
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiSetCR0Bits(VOID);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiGetCacheInformation(VOID);
|
|
|
|
|
|
|
|
BOOLEAN
|
|
|
|
NTAPI
|
|
|
|
KiIsNpxPresent(
|
|
|
|
VOID
|
|
|
|
);
|
|
|
|
|
2007-01-18 06:23:14 +00:00
|
|
|
BOOLEAN
|
|
|
|
NTAPI
|
|
|
|
KiIsNpxErrataPresent(
|
|
|
|
VOID
|
|
|
|
);
|
|
|
|
|
2006-09-01 03:05:30 +00:00
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiSetProcessorType(VOID);
|
|
|
|
|
|
|
|
ULONG
|
|
|
|
NTAPI
|
|
|
|
KiGetFeatureBits(VOID);
|
|
|
|
|
2005-04-22 12:52:25 +00:00
|
|
|
VOID
|
2006-09-06 12:19:00 +00:00
|
|
|
NTAPI
|
2010-01-19 06:34:15 +00:00
|
|
|
KiThreadStartup(VOID);
|
2005-05-09 01:38:29 +00:00
|
|
|
|
2009-10-29 10:04:15 +00:00
|
|
|
NTSTATUS
|
|
|
|
NTAPI
|
|
|
|
Ke386GetGdtEntryThread(
|
|
|
|
IN PKTHREAD Thread,
|
|
|
|
IN ULONG Offset,
|
|
|
|
IN PKGDTENTRY Descriptor
|
|
|
|
);
|
|
|
|
|
2009-11-08 01:13:49 +00:00
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiFlushNPXState(
|
|
|
|
IN FLOATING_SAVE_AREA *SaveArea
|
|
|
|
);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
Ki386AdjustEsp0(
|
|
|
|
IN PKTRAP_FRAME TrapFrame
|
|
|
|
);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
Ki386SetupAndExitToV86Mode(
|
|
|
|
OUT PTEB VdmTeb
|
|
|
|
);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KeI386VdmInitialize(
|
|
|
|
VOID
|
|
|
|
);
|
|
|
|
|
|
|
|
ULONG_PTR
|
|
|
|
NTAPI
|
|
|
|
Ki386EnableGlobalPage(
|
|
|
|
IN volatile ULONG_PTR Context
|
|
|
|
);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiI386PentiumLockErrataFixup(
|
|
|
|
VOID
|
|
|
|
);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiInitializePAT(
|
|
|
|
VOID
|
|
|
|
);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiInitializeMTRR(
|
|
|
|
IN BOOLEAN FinalCpu
|
|
|
|
);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiAmdK6InitializeMTRR(
|
|
|
|
VOID
|
|
|
|
);
|
|
|
|
|
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiRestoreFastSyscallReturnState(
|
|
|
|
VOID
|
|
|
|
);
|
|
|
|
|
|
|
|
ULONG_PTR
|
|
|
|
NTAPI
|
|
|
|
Ki386EnableDE(
|
|
|
|
IN ULONG_PTR Context
|
|
|
|
);
|
|
|
|
|
|
|
|
ULONG_PTR
|
|
|
|
NTAPI
|
|
|
|
Ki386EnableFxsr(
|
|
|
|
IN ULONG_PTR Context
|
|
|
|
);
|
|
|
|
|
|
|
|
ULONG_PTR
|
|
|
|
NTAPI
|
|
|
|
Ki386EnableXMMIExceptions(
|
|
|
|
IN ULONG_PTR Context
|
|
|
|
);
|
|
|
|
|
2010-01-10 15:00:44 +00:00
|
|
|
BOOLEAN
|
|
|
|
NTAPI
|
|
|
|
VdmDispatchBop(
|
|
|
|
IN PKTRAP_FRAME TrapFrame
|
|
|
|
);
|
2010-12-22 16:14:58 +00:00
|
|
|
|
2010-01-10 15:00:44 +00:00
|
|
|
BOOLEAN
|
|
|
|
FASTCALL
|
|
|
|
KiVdmOpcodePrefix(
|
|
|
|
IN PKTRAP_FRAME TrapFrame,
|
|
|
|
IN ULONG Flags
|
|
|
|
);
|
|
|
|
|
|
|
|
BOOLEAN
|
|
|
|
FASTCALL
|
|
|
|
Ki386HandleOpcodeV86(
|
|
|
|
IN PKTRAP_FRAME TrapFrame
|
|
|
|
);
|
|
|
|
|
2010-03-18 16:30:54 +00:00
|
|
|
DECLSPEC_NORETURN
|
2010-01-11 03:47:17 +00:00
|
|
|
VOID
|
|
|
|
FASTCALL
|
|
|
|
KiEoiHelper(
|
|
|
|
IN PKTRAP_FRAME TrapFrame
|
|
|
|
);
|
|
|
|
|
2010-01-11 05:53:57 +00:00
|
|
|
VOID
|
|
|
|
FASTCALL
|
2010-01-11 18:26:46 +00:00
|
|
|
Ki386BiosCallReturnAddress(
|
|
|
|
IN PKTRAP_FRAME TrapFrame
|
|
|
|
);
|
|
|
|
|
|
|
|
ULONG_PTR
|
|
|
|
FASTCALL
|
2010-01-11 05:53:57 +00:00
|
|
|
KiExitV86Mode(
|
|
|
|
IN PKTRAP_FRAME TrapFrame
|
|
|
|
);
|
|
|
|
|
2010-03-18 16:30:54 +00:00
|
|
|
DECLSPEC_NORETURN
|
2010-01-24 05:40:04 +00:00
|
|
|
VOID
|
|
|
|
NTAPI
|
|
|
|
KiDispatchExceptionFromTrapFrame(
|
|
|
|
IN NTSTATUS Code,
|
|
|
|
IN ULONG_PTR Address,
|
|
|
|
IN ULONG ParameterCount,
|
|
|
|
IN ULONG_PTR Parameter1,
|
|
|
|
IN ULONG_PTR Parameter2,
|
|
|
|
IN ULONG_PTR Parameter3,
|
|
|
|
IN PKTRAP_FRAME TrapFrame
|
|
|
|
);
|
|
|
|
|
2009-11-08 01:13:49 +00:00
|
|
|
//
|
|
|
|
// Global x86 only Kernel data
|
|
|
|
//
|
|
|
|
extern PVOID Ki386IopmSaveArea;
|
|
|
|
extern ULONG KeI386EFlagsAndMaskV86;
|
|
|
|
extern ULONG KeI386EFlagsOrMaskV86;
|
|
|
|
extern BOOLEAN KeI386VirtualIntExtensions;
|
|
|
|
extern KIDTENTRY KiIdt[MAXIMUM_IDTVECTOR];
|
|
|
|
extern KDESCRIPTOR KiIdtDescriptor;
|
|
|
|
extern BOOLEAN KiI386PentiumLockErrataPresent;
|
|
|
|
extern ULONG KeI386NpxPresent;
|
|
|
|
extern ULONG KeI386XMMIPresent;
|
|
|
|
extern ULONG KeI386FxsrPresent;
|
|
|
|
extern ULONG KiMXCsrMask;
|
|
|
|
extern ULONG KeI386CpuType;
|
|
|
|
extern ULONG KeI386CpuStep;
|
2010-01-19 06:34:15 +00:00
|
|
|
extern ULONG Ke386CacheAlignment;
|
|
|
|
extern ULONG KiFastSystemCallDisable;
|
2009-11-08 01:13:49 +00:00
|
|
|
extern UCHAR KiDebugRegisterTrapOffsets[9];
|
|
|
|
extern UCHAR KiDebugRegisterContextOffsets[9];
|
2010-07-26 21:32:38 +00:00
|
|
|
extern DECLSPEC_NORETURN VOID __cdecl KiTrap02(VOID);
|
Patch that fixes VMWare boot (and should fix QEMu/KVM boot on the testbot):
[NTOS]: A trap can get us into a state where DS/ES are invalid, making any pointer dereference (on DS/ES segmented memory, not SS, the stack) crash (and probably double-fault). Therefore, we have to be careful to switch to a good DS/ES before touching the TrapFrame pointer, which we don't have in ESP like the ASM code, but in a DS/ES-segmented register. For V8086 traps we can switch to the good DS/ES immediately, but for other kinds of traps, we actually need to save the current (bad) segments first. So we save them on the stack now, then switch to the good ones, then store the stack values into the trap frame. This is what happens on a non-optimized (-O0) build. On an optimized build, the segments will end up in registers instead, which is fine too (they'll be direct values). The order of instructions is guaranteed since the segment macros are volatile.
[NTOS]: The GPF and Invalid Opcode handlers are performance critical when talking about V8086 traps, because they control the main flow of execution during that mode (GPFs will be issued for any privileged instruction we need to emulate, and invalid opcode might be generated for BOPs). Because of this, we employ a fast entry/exit macro into V8086 mode since we can make certain assumptions. We detect, and use, such scenarios when the V8086 flag is enabled in EFLAGS. However, because we can land in a GPF handler with an invalid DS/ES, as some V8086 code could trample this during BIOS calls for example, we must make sure that we are on a valid DS/ES before dereferencing any pointer. We fixup DS/ES either in KiEnterTrap (for normal entry/exit) or, for V86, in KiEnterV86Trap. Notice the problem: we need to detect which of these to use early on but we can't touch the EFLAGS in the frame because DS/ES could be invalid. Thankfully SS is always guaranteed valid, so stack dereferences are game! We therefore read the EFLAGS here, in assembly, where we can touch ESP as we please. We save this in EDX, which will be used as the second argument for the FASTCALL C trap entry. When we make the fast V86 check, we use the parameter instead of the trap frame, leading us to using the correct trap entry function, which fixes up DS/ES and lets us go on our merry way...
[NTOS]: Make appropriate changes to GENERATE_TRAP_HANDLERS macro.
[NTOS]: Switch to using well-known NT trap handler names (hex-based, double-zeroed) instead of decimal-based trap handler names which are confusing.
[NTOS]: Clean up some debug spew.
svn path=/trunk/; revision=45052
2010-01-12 05:50:45 +00:00
|
|
|
extern VOID __cdecl KiTrap08(VOID);
|
|
|
|
extern VOID __cdecl KiTrap13(VOID);
|
2009-11-08 01:13:49 +00:00
|
|
|
extern VOID __cdecl KiFastCallEntry(VOID);
|
2010-01-10 15:00:44 +00:00
|
|
|
extern VOID NTAPI ExpInterlockedPopEntrySListFault(VOID);
|
|
|
|
extern VOID __cdecl CopyParams(VOID);
|
|
|
|
extern VOID __cdecl ReadBatch(VOID);
|
|
|
|
extern VOID __cdecl FrRestore(VOID);
|
2010-01-27 03:05:10 +00:00
|
|
|
extern CHAR KiSystemCallExitBranch[];
|
|
|
|
extern CHAR KiSystemCallExit[];
|
|
|
|
extern CHAR KiSystemCallExit2[];
|
2010-01-11 03:47:17 +00:00
|
|
|
|
2010-01-24 05:40:04 +00:00
|
|
|
//
|
|
|
|
// Trap Macros
|
|
|
|
//
|
|
|
|
#include "../trap_x.h"
|
|
|
|
|
|
|
|
//
|
|
|
|
// Returns a thread's FPU save area
|
|
|
|
//
|
2010-01-11 03:47:17 +00:00
|
|
|
PFX_SAVE_AREA
|
|
|
|
FORCEINLINE
|
|
|
|
KiGetThreadNpxArea(IN PKTHREAD Thread)
|
|
|
|
{
|
|
|
|
return (PFX_SAVE_AREA)((ULONG_PTR)Thread->InitialStack - sizeof(FX_SAVE_AREA));
|
|
|
|
}
|
2009-11-08 01:13:49 +00:00
|
|
|
|
|
|
|
//
|
|
|
|
// Sanitizes a selector
|
|
|
|
//
|
|
|
|
FORCEINLINE
|
|
|
|
ULONG
|
|
|
|
Ke386SanitizeSeg(IN ULONG Cs,
|
|
|
|
IN KPROCESSOR_MODE Mode)
|
|
|
|
{
|
|
|
|
//
|
|
|
|
// Check if we're in kernel-mode, and force CPL 0 if so.
|
|
|
|
// Otherwise, force CPL 3.
|
|
|
|
//
|
|
|
|
return ((Mode == KernelMode) ?
|
|
|
|
(Cs & (0xFFFF & ~RPL_MASK)) :
|
|
|
|
(RPL_MASK | (Cs & 0xFFFF)));
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Sanitizes EFLAGS
|
|
|
|
//
|
|
|
|
FORCEINLINE
|
|
|
|
ULONG
|
|
|
|
Ke386SanitizeFlags(IN ULONG Eflags,
|
|
|
|
IN KPROCESSOR_MODE Mode)
|
|
|
|
{
|
|
|
|
//
|
|
|
|
// Check if we're in kernel-mode, and sanitize EFLAGS if so.
|
|
|
|
// Otherwise, also force interrupt mask on.
|
|
|
|
//
|
|
|
|
return ((Mode == KernelMode) ?
|
|
|
|
(Eflags & (EFLAGS_USER_SANITIZE | EFLAGS_INTERRUPT_MASK)) :
|
|
|
|
(EFLAGS_INTERRUPT_MASK | (Eflags & EFLAGS_USER_SANITIZE)));
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Gets a DR register from a CONTEXT structure
|
|
|
|
//
|
|
|
|
FORCEINLINE
|
|
|
|
PVOID
|
|
|
|
KiDrFromContext(IN ULONG Dr,
|
|
|
|
IN PCONTEXT Context)
|
|
|
|
{
|
|
|
|
return *(PVOID*)((ULONG_PTR)Context + KiDebugRegisterContextOffsets[Dr]);
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Gets a DR register from a KTRAP_FRAME structure
|
|
|
|
//
|
|
|
|
FORCEINLINE
|
|
|
|
PVOID*
|
|
|
|
KiDrFromTrapFrame(IN ULONG Dr,
|
|
|
|
IN PKTRAP_FRAME TrapFrame)
|
|
|
|
{
|
|
|
|
return (PVOID*)((ULONG_PTR)TrapFrame + KiDebugRegisterTrapOffsets[Dr]);
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Sanitizes a Debug Register
|
|
|
|
//
|
|
|
|
FORCEINLINE
|
|
|
|
PVOID
|
|
|
|
Ke386SanitizeDr(IN PVOID DrAddress,
|
|
|
|
IN KPROCESSOR_MODE Mode)
|
|
|
|
{
|
|
|
|
//
|
|
|
|
// Check if we're in kernel-mode, and return the address directly if so.
|
|
|
|
// Otherwise, make sure it's not inside the kernel-mode address space.
|
|
|
|
// If it is, then clear the address.
|
|
|
|
//
|
|
|
|
return ((Mode == KernelMode) ? DrAddress :
|
|
|
|
(DrAddress <= MM_HIGHEST_USER_ADDRESS) ? DrAddress : 0);
|
|
|
|
}
|
|
|
|
|
2010-01-24 05:40:04 +00:00
|
|
|
//
|
|
|
|
// Exception with no arguments
|
|
|
|
//
|
|
|
|
VOID
|
|
|
|
FORCEINLINE
|
2010-01-24 15:18:50 +00:00
|
|
|
DECLSPEC_NORETURN
|
2010-01-24 05:40:04 +00:00
|
|
|
KiDispatchException0Args(IN NTSTATUS Code,
|
|
|
|
IN ULONG_PTR Address,
|
|
|
|
IN PKTRAP_FRAME TrapFrame)
|
|
|
|
{
|
|
|
|
/* Helper for exceptions with no arguments */
|
|
|
|
KiDispatchExceptionFromTrapFrame(Code, Address, 0, 0, 0, 0, TrapFrame);
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Exception with one argument
|
|
|
|
//
|
|
|
|
VOID
|
|
|
|
FORCEINLINE
|
2010-01-24 15:18:50 +00:00
|
|
|
DECLSPEC_NORETURN
|
2010-01-24 05:40:04 +00:00
|
|
|
KiDispatchException1Args(IN NTSTATUS Code,
|
|
|
|
IN ULONG_PTR Address,
|
|
|
|
IN ULONG P1,
|
|
|
|
IN PKTRAP_FRAME TrapFrame)
|
|
|
|
{
|
|
|
|
/* Helper for exceptions with no arguments */
|
|
|
|
KiDispatchExceptionFromTrapFrame(Code, Address, 1, P1, 0, 0, TrapFrame);
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Exception with two arguments
|
|
|
|
//
|
|
|
|
VOID
|
|
|
|
FORCEINLINE
|
2010-01-24 15:18:50 +00:00
|
|
|
DECLSPEC_NORETURN
|
2010-01-24 05:40:04 +00:00
|
|
|
KiDispatchException2Args(IN NTSTATUS Code,
|
|
|
|
IN ULONG_PTR Address,
|
|
|
|
IN ULONG P1,
|
|
|
|
IN ULONG P2,
|
|
|
|
IN PKTRAP_FRAME TrapFrame)
|
|
|
|
{
|
|
|
|
/* Helper for exceptions with no arguments */
|
|
|
|
KiDispatchExceptionFromTrapFrame(Code, Address, 2, P1, P2, 0, TrapFrame);
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Performs a system call
|
|
|
|
//
|
|
|
|
NTSTATUS
|
|
|
|
FORCEINLINE
|
|
|
|
KiSystemCallTrampoline(IN PVOID Handler,
|
|
|
|
IN PVOID Arguments,
|
|
|
|
IN ULONG StackBytes)
|
|
|
|
{
|
|
|
|
NTSTATUS Result;
|
2010-12-22 16:14:58 +00:00
|
|
|
|
2010-01-24 05:40:04 +00:00
|
|
|
/*
|
|
|
|
* This sequence does a RtlCopyMemory(Stack - StackBytes, Arguments, StackBytes)
|
|
|
|
* and then calls the function associated with the system call.
|
|
|
|
*
|
|
|
|
* It's done in assembly for two reasons: we need to muck with the stack,
|
|
|
|
* and the call itself restores the stack back for us. The only way to do
|
|
|
|
* this in C is to do manual C handlers for every possible number of args on
|
|
|
|
* the stack, and then have the handler issue a call by pointer. This is
|
|
|
|
* wasteful since it'll basically push the values twice and require another
|
|
|
|
* level of call indirection.
|
|
|
|
*
|
|
|
|
* The ARM kernel currently does this, but it should probably be changed
|
|
|
|
* later to function like this as well.
|
|
|
|
*
|
|
|
|
*/
|
2010-03-18 16:30:54 +00:00
|
|
|
#ifdef __GNUC__
|
2010-01-24 05:40:04 +00:00
|
|
|
__asm__ __volatile__
|
|
|
|
(
|
|
|
|
"subl %1, %%esp\n"
|
|
|
|
"movl %%esp, %%edi\n"
|
|
|
|
"movl %2, %%esi\n"
|
|
|
|
"shrl $2, %1\n"
|
|
|
|
"rep movsd\n"
|
|
|
|
"call *%3\n"
|
|
|
|
"movl %%eax, %0\n"
|
|
|
|
: "=r"(Result)
|
|
|
|
: "c"(StackBytes),
|
|
|
|
"d"(Arguments),
|
|
|
|
"r"(Handler)
|
|
|
|
: "%esp", "%esi", "%edi"
|
|
|
|
);
|
2010-03-18 16:30:54 +00:00
|
|
|
#elif defined(_MSC_VER)
|
|
|
|
__asm
|
|
|
|
{
|
|
|
|
mov ecx, StackBytes
|
|
|
|
mov edx, Arguments
|
2011-06-22 19:04:28 +00:00
|
|
|
mov ebx, Handler
|
2010-03-18 16:30:54 +00:00
|
|
|
sub esp, ecx
|
|
|
|
mov edi, esp
|
|
|
|
mov esi, edx
|
|
|
|
shr ecx, 2
|
|
|
|
rep movsd
|
2011-06-22 19:04:28 +00:00
|
|
|
call ebx
|
2010-03-18 16:30:54 +00:00
|
|
|
mov Result, eax
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
#error Unknown Compiler
|
|
|
|
#endif
|
|
|
|
|
2010-01-24 05:40:04 +00:00
|
|
|
return Result;
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Checks for pending APCs
|
|
|
|
//
|
|
|
|
VOID
|
2010-01-11 05:53:57 +00:00
|
|
|
FORCEINLINE
|
2010-01-24 05:40:04 +00:00
|
|
|
KiCheckForApcDelivery(IN PKTRAP_FRAME TrapFrame)
|
|
|
|
{
|
|
|
|
PKTHREAD Thread;
|
|
|
|
KIRQL OldIrql;
|
|
|
|
|
|
|
|
/* Check for V8086 or user-mode trap */
|
|
|
|
if ((TrapFrame->EFlags & EFLAGS_V86_MASK) || (KiUserTrap(TrapFrame)))
|
|
|
|
{
|
|
|
|
/* Get the thread */
|
|
|
|
Thread = KeGetCurrentThread();
|
|
|
|
while (TRUE)
|
|
|
|
{
|
|
|
|
/* Turn off the alerted state for kernel mode */
|
|
|
|
Thread->Alerted[KernelMode] = FALSE;
|
|
|
|
|
|
|
|
/* Are there pending user APCs? */
|
|
|
|
if (!Thread->ApcState.UserApcPending) break;
|
|
|
|
|
|
|
|
/* Raise to APC level and enable interrupts */
|
|
|
|
OldIrql = KfRaiseIrql(APC_LEVEL);
|
|
|
|
_enable();
|
|
|
|
|
|
|
|
/* Deliver APCs */
|
|
|
|
KiDeliverApc(UserMode, NULL, TrapFrame);
|
|
|
|
|
|
|
|
/* Restore IRQL and disable interrupts once again */
|
|
|
|
KfLowerIrql(OldIrql);
|
|
|
|
_disable();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Converts a base thread to a GUI thread
|
|
|
|
//
|
|
|
|
NTSTATUS
|
|
|
|
FORCEINLINE
|
|
|
|
KiConvertToGuiThread(VOID)
|
|
|
|
{
|
2010-12-22 16:14:58 +00:00
|
|
|
NTSTATUS Result;
|
2010-01-24 05:40:04 +00:00
|
|
|
PVOID StackFrame;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Converting to a GUI thread safely updates ESP in-place as well as the
|
|
|
|
* current Thread->TrapFrame and EBP when KeSwitchKernelStack is called.
|
|
|
|
*
|
|
|
|
* However, PsConvertToGuiThread "helpfully" restores EBP to the original
|
|
|
|
* caller's value, since it is considered a nonvolatile register. As such,
|
|
|
|
* as soon as we're back after the conversion and we try to store the result
|
|
|
|
* which will probably be in some stack variable (EBP-based), we'll crash as
|
|
|
|
* we are touching the de-allocated non-expanded stack.
|
|
|
|
*
|
|
|
|
* Thus we need a way to update our EBP before EBP is touched, and the only
|
|
|
|
* way to guarantee this is to do the call itself in assembly, use the EAX
|
|
|
|
* register to store the result, fixup EBP, and then let the C code continue
|
|
|
|
* on its merry way.
|
|
|
|
*
|
|
|
|
*/
|
2010-03-18 16:30:54 +00:00
|
|
|
#ifdef __GNUC__
|
2010-01-24 05:40:04 +00:00
|
|
|
__asm__ __volatile__
|
|
|
|
(
|
|
|
|
"movl %%ebp, %1\n"
|
|
|
|
"subl %%esp, %1\n"
|
|
|
|
"call _PsConvertToGuiThread@0\n"
|
|
|
|
"addl %%esp, %1\n"
|
|
|
|
"movl %1, %%ebp\n"
|
|
|
|
"movl %%eax, %0\n"
|
|
|
|
: "=r"(Result), "=r"(StackFrame)
|
|
|
|
:
|
2010-02-18 23:25:12 +00:00
|
|
|
: "%esp", "%ecx", "%edx", "memory"
|
2010-01-24 05:40:04 +00:00
|
|
|
);
|
2010-03-18 16:30:54 +00:00
|
|
|
#elif defined(_MSC_VER)
|
|
|
|
NTSTATUS NTAPI PsConvertToGuiThread(VOID);
|
|
|
|
__asm
|
|
|
|
{
|
|
|
|
mov StackFrame, ebp
|
|
|
|
sub StackFrame, esp
|
|
|
|
call PsConvertToGuiThread
|
|
|
|
add StackFrame, esp
|
|
|
|
mov ebp, StackFrame
|
|
|
|
mov Result, eax
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
#error Unknown Compiler
|
|
|
|
#endif
|
2010-01-24 05:40:04 +00:00
|
|
|
return Result;
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// Switches from boot loader to initial kernel stack
|
|
|
|
//
|
2010-01-11 05:53:57 +00:00
|
|
|
VOID
|
2010-01-24 05:40:04 +00:00
|
|
|
FORCEINLINE
|
|
|
|
KiSwitchToBootStack(IN ULONG_PTR InitialStack)
|
2010-01-11 05:53:57 +00:00
|
|
|
{
|
2010-01-24 05:40:04 +00:00
|
|
|
/* We have to switch to a new stack before continuing kernel initialization */
|
2010-03-18 16:30:54 +00:00
|
|
|
#ifdef __GNUC__
|
|
|
|
__asm__
|
2010-01-11 05:53:57 +00:00
|
|
|
(
|
|
|
|
"movl %0, %%esp\n"
|
2010-01-24 05:40:04 +00:00
|
|
|
"subl %1, %%esp\n"
|
|
|
|
"pushl %2\n"
|
|
|
|
"jmp _KiSystemStartupBootStack@0\n"
|
2010-12-22 16:14:58 +00:00
|
|
|
:
|
2010-01-24 05:40:04 +00:00
|
|
|
: "c"(InitialStack),
|
|
|
|
"i"(NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH),
|
|
|
|
"i"(CR0_EM | CR0_TS | CR0_MP)
|
2010-01-11 05:53:57 +00:00
|
|
|
: "%esp"
|
|
|
|
);
|
2010-03-18 16:30:54 +00:00
|
|
|
#elif defined(_MSC_VER)
|
|
|
|
VOID NTAPI KiSystemStartupBootStack(VOID);
|
|
|
|
__asm
|
|
|
|
{
|
2010-05-31 12:52:16 +00:00
|
|
|
mov esp, InitialStack
|
2010-03-18 16:30:54 +00:00
|
|
|
sub esp, (NPX_FRAME_LENGTH + KTRAP_FRAME_ALIGN + KTRAP_FRAME_LENGTH)
|
|
|
|
push (CR0_EM | CR0_TS | CR0_MP)
|
|
|
|
jmp KiSystemStartupBootStack
|
|
|
|
}
|
|
|
|
#else
|
|
|
|
#error Unknown Compiler
|
|
|
|
#endif
|
2010-01-11 05:53:57 +00:00
|
|
|
}
|
2010-01-24 05:40:04 +00:00
|
|
|
|
2010-05-31 12:52:16 +00:00
|
|
|
//
|
|
|
|
// Emits the iret instruction for C code
|
|
|
|
//
|
|
|
|
DECLSPEC_NORETURN
|
|
|
|
VOID
|
|
|
|
FORCEINLINE
|
|
|
|
KiIret(VOID)
|
|
|
|
{
|
|
|
|
#if defined(__GNUC__)
|
|
|
|
__asm__ __volatile__
|
|
|
|
(
|
|
|
|
"iret\n"
|
|
|
|
);
|
|
|
|
#elif defined(_MSC_VER)
|
|
|
|
__asm
|
|
|
|
{
|
2011-06-22 19:04:28 +00:00
|
|
|
iretd
|
2010-05-31 12:52:16 +00:00
|
|
|
}
|
|
|
|
#else
|
|
|
|
#error Unsupported compiler
|
|
|
|
#endif
|
|
|
|
UNREACHABLE;
|
|
|
|
}
|
|
|
|
|
2010-02-01 03:51:45 +00:00
|
|
|
//
|
|
|
|
// Normally this is done by the HAL, but on x86 as an optimization, the kernel
|
|
|
|
// initiates the end by calling back into the HAL and exiting the trap here.
|
|
|
|
//
|
|
|
|
VOID
|
|
|
|
FORCEINLINE
|
|
|
|
KiEndInterrupt(IN KIRQL Irql,
|
|
|
|
IN PKTRAP_FRAME TrapFrame)
|
|
|
|
{
|
|
|
|
/* Disable interrupts and end the interrupt */
|
|
|
|
_disable();
|
|
|
|
HalEndSystemInterrupt(Irql, TrapFrame);
|
2010-12-22 16:14:58 +00:00
|
|
|
|
2010-02-01 03:51:45 +00:00
|
|
|
/* Exit the interrupt */
|
|
|
|
KiEoiHelper(TrapFrame);
|
|
|
|
}
|
|
|
|
|
|
|
|
//
|
|
|
|
// PERF Code
|
|
|
|
//
|
|
|
|
VOID
|
|
|
|
FORCEINLINE
|
|
|
|
Ki386PerfEnd(VOID)
|
|
|
|
{
|
|
|
|
extern ULONGLONG BootCyclesEnd, BootCycles;
|
|
|
|
BootCyclesEnd = __rdtsc();
|
|
|
|
DbgPrint("Boot took %I64d cycles!\n", BootCyclesEnd - BootCycles);
|
|
|
|
DbgPrint("Interrupts: %d System Calls: %d Context Switches: %d\n",
|
|
|
|
KeGetCurrentPrcb()->InterruptCount,
|
|
|
|
KeGetCurrentPrcb()->KeSystemCalls,
|
|
|
|
KeGetContextSwitches(KeGetCurrentPrcb()));
|
|
|
|
}
|
|
|
|
|
2005-06-25 17:01:17 +00:00
|
|
|
#endif
|