Intravision/reactos
1
0
Fork 0
mirror of https://github.com/reactos/reactos.git synced 2024-06-29 01:12:06 +00:00
Code Issues Packages Projects Releases Wiki Activity
reactos/ntoskrnl/ke/i386/cpu.c
Timo Kreuzer 71fefa32db
[NDK][NTOS] Add global definition of INIT_FUNCTION/INIT_SECTION (#779) 
* Add an NDK header to define INIT_FUNCTION/INIT_SECTION globally
* Use _declspec(allocate(x)) and _declspec(code_seg(x)) on MSVC versions that support it
* Use INIT_FUNCTION on functions only and INIT_SECTION on data only (required by MSVC)
* Place INIT_FUNCTION before the return type (required by MSVC)
* Make sure declarations and implementations share the same modifiers (required by MSVC)
* Add a global linker option to suppress warnings about defined but unused INIT section
* Merge INIT section into .text in freeldr
2018-12-30 12:19:11 +01:00

1539 lines
43 KiB
C
Raw Blame History

/*
* PROJECT: ReactOS Kernel
* LICENSE: GPL - See COPYING in the top level directory
* FILE: ntoskrnl/ke/i386/cpu.c
* PURPOSE: Routines for CPU-level support
* PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
*/
/* INCLUDES *****************************************************************/
#include <ntoskrnl.h>
#define NDEBUG
#include <debug.h>
/* GLOBALS *******************************************************************/
/* The TSS to use for Double Fault Traps (INT 0x9) */
UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
/* The TSS to use for NMI Fault Traps (INT 0x2) */
UCHAR KiNMITSS[KTSS_IO_MAPS];
/* CPU Features and Flags */
ULONG KeI386CpuType;
ULONG KeI386CpuStep;
ULONG KiFastSystemCallDisable = 0;
ULONG KeI386NpxPresent = 0;
ULONG KiMXCsrMask = 0;
ULONG MxcsrFeatureMask = 0;
ULONG KeI386XMMIPresent = 0;
ULONG KeI386FxsrPresent = 0;
ULONG KeI386MachineType;
ULONG Ke386Pae = FALSE;
ULONG Ke386NoExecute = FALSE;
ULONG KeLargestCacheLine = 0x40;
ULONG KeDcacheFlushCount = 0;
ULONG KeIcacheFlushCount = 0;
ULONG KiDmaIoCoherency = 0;
ULONG KePrefetchNTAGranularity = 32;
BOOLEAN KiI386PentiumLockErrataPresent;
BOOLEAN KiSMTProcessorsPresent;
/* The distance between SYSEXIT and IRETD return modes */
UCHAR KiSystemCallExitAdjust;
/* The offset that was applied -- either 0 or the value above */
UCHAR KiSystemCallExitAdjusted;
/* Whether the adjustment was already done once */
BOOLEAN KiFastCallCopyDoneOnce;
/* Flush data */
volatile LONG KiTbFlushTimeStamp;
/* CPU Signatures */
static const CHAR CmpIntelID[] = "GenuineIntel";
static const CHAR CmpAmdID[] = "AuthenticAMD";
static const CHAR CmpCyrixID[] = "CyrixInstead";
static const CHAR CmpTransmetaID[] = "GenuineTMx86";
static const CHAR CmpCentaurID[] = "CentaurHauls";
static const CHAR CmpRiseID[] = "RiseRiseRise";
/* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
/* NSC/Cyrix CPU configuration register index */
#define CX86_CCR1 0xc1
/* NSC/Cyrix CPU indexed register access macros */
static __inline
UCHAR
getCx86(UCHAR reg)
{
WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22, reg);
return READ_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23);
}
static __inline
void
setCx86(UCHAR reg, UCHAR data)
{
WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22, reg);
WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23, data);
}
/* FUNCTIONS *****************************************************************/
INIT_FUNCTION
VOID
NTAPI
KiSetProcessorType(VOID)
{
ULONG EFlags, NewEFlags;
CPU_INFO CpuInfo;
ULONG Stepping, Type;
/* Start by assuming no CPUID data */
KeGetCurrentPrcb()->CpuID = 0;
/* Save EFlags */
EFlags = __readeflags();
/* XOR out the ID bit and update EFlags */
NewEFlags = EFlags ^ EFLAGS_ID;
__writeeflags(NewEFlags);
/* Get them back and see if they were modified */
NewEFlags = __readeflags();
if (NewEFlags != EFlags)
{
/* The modification worked, so CPUID exists. Set the ID Bit again. */
EFlags |= EFLAGS_ID;
__writeeflags(EFlags);
/* Peform CPUID 0 to see if CPUID 1 is supported */
KiCpuId(&CpuInfo, 0);
if (CpuInfo.Eax > 0)
{
/* Do CPUID 1 now */
KiCpuId(&CpuInfo, 1);
/*
* Get the Stepping and Type. The stepping contains both the
* Model and the Step, while the Type contains the returned Type.
* We ignore the family.
*
* For the stepping, we convert this: zzzzzzxy into this: x0y
*/
Stepping = CpuInfo.Eax & 0xF0;
Stepping <<= 4;
Stepping += (CpuInfo.Eax & 0xFF);
Stepping &= 0xF0F;
Type = CpuInfo.Eax & 0xF00;
Type >>= 8;
/* Save them in the PRCB */
KeGetCurrentPrcb()->CpuID = TRUE;
KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
}
else
{
DPRINT1("CPUID Support lacking\n");
}
}
else
{
DPRINT1("CPUID Support lacking\n");
}
/* Restore EFLAGS */
__writeeflags(EFlags);
}
INIT_FUNCTION
ULONG
NTAPI
KiGetCpuVendor(VOID)
{
PKPRCB Prcb = KeGetCurrentPrcb();
CPU_INFO CpuInfo;
/* Assume no Vendor ID and fail if no CPUID Support. */
Prcb->VendorString[0] = 0;
if (!Prcb->CpuID) return 0;
/* Get the Vendor ID */
KiCpuId(&CpuInfo, 0);
/* Copy it to the PRCB and null-terminate it */
*(ULONG*)&Prcb->VendorString[0] = CpuInfo.Ebx;
*(ULONG*)&Prcb->VendorString[4] = CpuInfo.Edx;
*(ULONG*)&Prcb->VendorString[8] = CpuInfo.Ecx;
Prcb->VendorString[12] = 0;
/* Now check the CPU Type */
if (!strcmp(Prcb->VendorString, CmpIntelID))
{
return CPU_INTEL;
}
else if (!strcmp(Prcb->VendorString, CmpAmdID))
{
return CPU_AMD;
}
else if (!strcmp(Prcb->VendorString, CmpCyrixID))
{
DPRINT1("Cyrix CPU support not fully tested!\n");
return CPU_CYRIX;
}
else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
{
DPRINT1("Transmeta CPU support not fully tested!\n");
return CPU_TRANSMETA;
}
else if (!strcmp(Prcb->VendorString, CmpCentaurID))
{
DPRINT1("Centaur CPU support not fully tested!\n");
return CPU_CENTAUR;
}
else if (!strcmp(Prcb->VendorString, CmpRiseID))
{
DPRINT1("Rise CPU support not fully tested!\n");
return CPU_RISE;
}
/* Unknown CPU */
DPRINT1("%s CPU support not fully tested!\n", Prcb->VendorString);
return CPU_UNKNOWN;
}
INIT_FUNCTION
ULONG
NTAPI
KiGetFeatureBits(VOID)
{
PKPRCB Prcb = KeGetCurrentPrcb();
ULONG Vendor;
ULONG FeatureBits = KF_WORKING_PTE;
CPU_INFO CpuInfo, DummyCpuInfo;
UCHAR Ccr1;
BOOLEAN ExtendedCPUID = TRUE;
ULONG CpuFeatures = 0;
/* Get the Vendor ID */
Vendor = KiGetCpuVendor();
/* Make sure we got a valid vendor ID at least. */
if (!Vendor) return FeatureBits;
/* Get the CPUID Info. Features are in Reg[3]. */
KiCpuId(&CpuInfo, 1);
/* Set the initial APIC ID */
Prcb->InitialApicId = (UCHAR)(CpuInfo.Ebx >> 24);
switch (Vendor)
{
/* Intel CPUs */
case CPU_INTEL:
/* Check if it's a P6 */
if (Prcb->CpuType == 6)
{
/* Perform the special sequence to get the MicroCode Signature */
__writemsr(0x8B, 0);
KiCpuId(&DummyCpuInfo, 1);
Prcb->UpdateSignature.QuadPart = __readmsr(0x8B);
}
else if (Prcb->CpuType == 5)
{
/* On P5, enable workaround for the LOCK errata. */
KiI386PentiumLockErrataPresent = TRUE;
}
/* Check for broken P6 with bad SMP PTE implementation */
if (((CpuInfo.Eax & 0x0FF0) == 0x0610 && (CpuInfo.Eax & 0x000F) <= 0x9) ||
((CpuInfo.Eax & 0x0FF0) == 0x0630 && (CpuInfo.Eax & 0x000F) <= 0x4))
{
/* Remove support for correct PTE support. */
FeatureBits &= ~KF_WORKING_PTE;
}
/* Check if the CPU is too old to support SYSENTER */
if ((Prcb->CpuType < 6) ||
((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
{
/* Disable it */
CpuInfo.Edx &= ~0x800;
}
break;
/* AMD CPUs */
case CPU_AMD:
/* Check if this is a K5 or K6. (family 5) */
if ((CpuInfo.Eax & 0x0F00) == 0x0500)
{
/* Get the Model Number */
switch (CpuInfo.Eax & 0x00F0)
{
/* Model 1: K5 - 5k86 (initial models) */
case 0x0010:
/* Check if this is Step 0 or 1. They don't support PGE */
if ((CpuInfo.Eax & 0x000F) > 0x03) break;
/* Model 0: K5 - SSA5 */
case 0x0000:
/* Model 0 doesn't support PGE at all. */
CpuInfo.Edx &= ~0x2000;
break;
/* Model 8: K6-2 */
case 0x0080:
/* K6-2, Step 8 and over have support for MTRR. */
if ((CpuInfo.Eax & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
break;
/* Model 9: K6-III
Model D: K6-2+, K6-III+ */
case 0x0090:
case 0x00D0:
FeatureBits |= KF_AMDK6MTRR;
break;
}
}
else if((CpuInfo.Eax & 0x0F00) < 0x0500)
{
/* Families below 5 don't support PGE, PSE or CMOV at all */
CpuInfo.Edx &= ~(0x08 | 0x2000 | 0x8000);
/* They also don't support advanced CPUID functions. */
ExtendedCPUID = FALSE;
}
break;
/* Cyrix CPUs */
case CPU_CYRIX:
/* Workaround the "COMA" bug on 6x family of Cyrix CPUs */
if (Prcb->CpuType == 6 &&
Prcb->CpuStep <= 1)
{
/* Get CCR1 value */
Ccr1 = getCx86(CX86_CCR1);
/* Enable the NO_LOCK bit */
Ccr1 |= 0x10;
/* Set the new CCR1 value */
setCx86(CX86_CCR1, Ccr1);
}
break;
/* Transmeta CPUs */
case CPU_TRANSMETA:
/* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
if ((CpuInfo.Eax & 0x0FFF) >= 0x0542)
{
__writemsr(0x80860004, __readmsr(0x80860004) | 0x0100);
FeatureBits |= KF_CMPXCHG8B;
}
break;
/* Centaur, IDT, Rise and VIA CPUs */
case CPU_CENTAUR:
case CPU_RISE:
/* These CPUs don't report the presence of CMPXCHG8B through CPUID.
However, this feature exists and operates properly without any additional steps. */
FeatureBits |= KF_CMPXCHG8B;
break;
}
/* Set the current features */
CpuFeatures = CpuInfo.Edx;
/* Convert all CPUID Feature bits into our format */
if (CpuFeatures & 0x00000002) FeatureBits |= KF_V86_VIS | KF_CR4;
if (CpuFeatures & 0x00000008) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
if (CpuFeatures & 0x00000010) FeatureBits |= KF_RDTSC;
if (CpuFeatures & 0x00000100) FeatureBits |= KF_CMPXCHG8B;
if (CpuFeatures & 0x00000800) FeatureBits |= KF_FAST_SYSCALL;
if (CpuFeatures & 0x00001000) FeatureBits |= KF_MTRR;
if (CpuFeatures & 0x00002000) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
if (CpuFeatures & 0x00008000) FeatureBits |= KF_CMOV;
if (CpuFeatures & 0x00010000) FeatureBits |= KF_PAT;
if (CpuFeatures & 0x00200000) FeatureBits |= KF_DTS;
if (CpuFeatures & 0x00800000) FeatureBits |= KF_MMX;
if (CpuFeatures & 0x01000000) FeatureBits |= KF_FXSR;
if (CpuFeatures & 0x02000000) FeatureBits |= KF_XMMI;
if (CpuFeatures & 0x04000000) FeatureBits |= KF_XMMI64;
/* Check if the CPU has hyper-threading */
if (CpuFeatures & 0x10000000)
{
/* Set the number of logical CPUs */
Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(CpuInfo.Ebx >> 16);
if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
{
/* We're on dual-core */
KiSMTProcessorsPresent = TRUE;
}
}
else
{
/* We only have a single CPU */
Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
}
/* Check if CPUID 0x80000000 is supported */
if (ExtendedCPUID)
{
/* Do the call */
KiCpuId(&CpuInfo, 0x80000000);
if ((CpuInfo.Eax & 0xffffff00) == 0x80000000)
{
/* Check if CPUID 0x80000001 is supported */
if (CpuInfo.Eax >= 0x80000001)
{
/* Check which extended features are available. */
KiCpuId(&CpuInfo, 0x80000001);
/* Check if NX-bit is supported */
if (CpuInfo.Edx & 0x00100000) FeatureBits |= KF_NX_BIT;
/* Now handle each features for each CPU Vendor */
switch (Vendor)
{
case CPU_AMD:
case CPU_CENTAUR:
if (CpuInfo.Edx & 0x80000000) FeatureBits |= KF_3DNOW;
break;
}
}
}
}
#define print_supported(kf_value) ((FeatureBits & kf_value) ? #kf_value : "")
DPRINT1("Supported CPU features : %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s %s\n",
print_supported(KF_V86_VIS),
print_supported(KF_RDTSC),
print_supported(KF_CR4),
print_supported(KF_CMOV),
print_supported(KF_GLOBAL_PAGE),
print_supported(KF_LARGE_PAGE),
print_supported(KF_MTRR),
print_supported(KF_CMPXCHG8B),
print_supported(KF_MMX),
print_supported(KF_WORKING_PTE),
print_supported(KF_PAT),
print_supported(KF_FXSR),
print_supported(KF_FAST_SYSCALL),
print_supported(KF_XMMI),
print_supported(KF_3DNOW),
print_supported(KF_AMDK6MTRR),
print_supported(KF_XMMI64),
print_supported(KF_DTS),
print_supported(KF_NX_BIT),
print_supported(KF_NX_DISABLED),
print_supported(KF_NX_ENABLED));
#undef print_supported
/* Return the Feature Bits */
return FeatureBits;
}
INIT_FUNCTION
VOID
NTAPI
KiGetCacheInformation(VOID)
{
PKIPCR Pcr = (PKIPCR)KeGetPcr();
ULONG Vendor;
CPU_INFO CpuInfo;
ULONG CacheRequests = 0, i;
ULONG CurrentRegister;
UCHAR RegisterByte, Associativity = 0;
ULONG Size, CacheLine = 64, CurrentSize = 0;
BOOLEAN FirstPass = TRUE;
/* Set default L2 size */
Pcr->SecondLevelCacheSize = 0;
/* Get the Vendor ID and make sure we support CPUID */
Vendor = KiGetCpuVendor();
if (!Vendor) return;
/* Check the Vendor ID */
switch (Vendor)
{
/* Handle Intel case */
case CPU_INTEL:
/*Check if we support CPUID 2 */
KiCpuId(&CpuInfo, 0);
if (CpuInfo.Eax >= 2)
{
/* We need to loop for the number of times CPUID will tell us to */
do
{
/* Do the CPUID call */
KiCpuId(&CpuInfo, 2);
/* Check if it was the first call */
if (FirstPass)
{
/*
* The number of times to loop is the first byte. Read
* it and then destroy it so we don't get confused.
*/
CacheRequests = CpuInfo.Eax & 0xFF;
CpuInfo.Eax &= 0xFFFFFF00;
/* Don't go over this again */
FirstPass = FALSE;
}
/* Loop all 4 registers */
for (i = 0; i < 4; i++)
{
/* Get the current register */
CurrentRegister = CpuInfo.AsUINT32[i];
/*
* If the upper bit is set, then this register should
* be skipped.
*/
if (CurrentRegister & 0x80000000) continue;
/* Keep looping for every byte inside this register */
while (CurrentRegister)
{
/* Read a byte, skip a byte. */
RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
CurrentRegister >>= 8;
if (!RegisterByte) continue;
Size = 0;
switch (RegisterByte)
{
case 0x06:
case 0x08:
KePrefetchNTAGranularity = 32;
break;
case 0x09:
KePrefetchNTAGranularity = 64;
break;
case 0x0a:
case 0x0c:
KePrefetchNTAGranularity = 32;
break;
case 0x0d:
case 0x0e:
KePrefetchNTAGranularity = 64;
break;
case 0x1d:
Size = 128 * 1024;
Associativity = 2;
break;
case 0x21:
Size = 256 * 1024;
Associativity = 8;
break;
case 0x24:
Size = 1024 * 1024;
Associativity = 16;
break;
case 0x2c:
case 0x30:
KePrefetchNTAGranularity = 64;
break;
case 0x41:
case 0x42:
case 0x43:
case 0x44:
case 0x45:
Size = (1 << (RegisterByte - 0x41)) * 128 * 1024;
Associativity = 4;
break;
case 0x48:
Size = 3 * 1024 * 1024;
Associativity = 12;
break;
case 0x49:
Size = 4 * 1024 * 1024;
Associativity = 16;
break;
case 0x4e:
Size = 6 * 1024 * 1024;
Associativity = 24;
break;
case 0x60:
case 0x66:
case 0x67:
case 0x68:
KePrefetchNTAGranularity = 64;
break;
case 0x78:
Size = 1024 * 1024;
Associativity = 4;
break;
case 0x79:
case 0x7a:
case 0x7b:
case 0x7c:
case 0x7d:
Size = (1 << (RegisterByte - 0x79)) * 128 * 1024;
Associativity = 8;
break;
case 0x7f:
Size = 512 * 1024;
Associativity = 2;
break;
case 0x80:
Size = 512 * 1024;
Associativity = 8;
break;
case 0x82:
case 0x83:
case 0x84:
case 0x85:
Size = (1 << (RegisterByte - 0x82)) * 256 * 1024;
Associativity = 8;
break;
case 0x86:
Size = 512 * 1024;
Associativity = 4;
break;
case 0x87:
Size = 1024 * 1024;
Associativity = 8;
break;
case 0xf0:
KePrefetchNTAGranularity = 64;
break;
case 0xf1:
KePrefetchNTAGranularity = 128;
break;
}
if (Size && (Size / Associativity) > CurrentSize)
{
/* Set the L2 Cache Size and Associativity */
CurrentSize = Size / Associativity;
Pcr->SecondLevelCacheSize = Size;
Pcr->SecondLevelCacheAssociativity = Associativity;
}
}
}
} while (--CacheRequests);
}
break;
case CPU_AMD:
/* Check if we support CPUID 0x80000005 */
KiCpuId(&CpuInfo, 0x80000000);
if (CpuInfo.Eax >= 0x80000005)
{
/* Get L1 size first */
KiCpuId(&CpuInfo, 0x80000005);
KePrefetchNTAGranularity = CpuInfo.Ecx & 0xFF;
/* Check if we support CPUID 0x80000006 */
KiCpuId(&CpuInfo, 0x80000000);
if (CpuInfo.Eax >= 0x80000006)
{
/* Get 2nd level cache and tlb size */
KiCpuId(&CpuInfo, 0x80000006);
/* Cache line size */
CacheLine = CpuInfo.Ecx & 0xFF;
/* Hardcode associativity */
RegisterByte = (CpuInfo.Ecx >> 12) & 0xFF;
switch (RegisterByte)
{
case 2:
Associativity = 2;
break;
case 4:
Associativity = 4;
break;
case 6:
Associativity = 8;
break;
case 8:
case 15:
Associativity = 16;
break;
default:
Associativity = 1;
break;
}
/* Compute size */
Size = (CpuInfo.Ecx >> 16) << 10;
/* Hack for Model 6, Steping 300 */
if ((KeGetCurrentPrcb()->CpuType == 6) &&
(KeGetCurrentPrcb()->CpuStep == 0x300))
{
/* Stick 64K in there */
Size = 64 * 1024;
}
/* Set the L2 Cache Size and associativity */
Pcr->SecondLevelCacheSize = Size;
Pcr->SecondLevelCacheAssociativity = Associativity;
}
}
break;
case CPU_CYRIX:
case CPU_TRANSMETA:
case CPU_CENTAUR:
case CPU_RISE:
/* FIXME */
break;
}
/* Set the cache line */
if (CacheLine > KeLargestCacheLine) KeLargestCacheLine = CacheLine;
DPRINT1("Prefetch Cache: %lu bytes\tL2 Cache: %lu bytes\tL2 Cache Line: %lu bytes\tL2 Cache Associativity: %lu\n",
KePrefetchNTAGranularity,
Pcr->SecondLevelCacheSize,
KeLargestCacheLine,
Pcr->SecondLevelCacheAssociativity);
}
INIT_FUNCTION
VOID
NTAPI
KiSetCR0Bits(VOID)
{
ULONG Cr0;
/* Save current CR0 */
Cr0 = __readcr0();
/* If this is a 486, enable Write-Protection */
if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
/* Set new Cr0 */
__writecr0(Cr0);
}
INIT_FUNCTION
VOID
NTAPI
KiInitializeTSS2(IN PKTSS Tss,
IN PKGDTENTRY TssEntry OPTIONAL)
{
PUCHAR p;
/* Make sure the GDT Entry is valid */
if (TssEntry)
{
/* Set the Limit */
TssEntry->LimitLow = sizeof(KTSS) - 1;
TssEntry->HighWord.Bits.LimitHi = 0;
}
/* Now clear the I/O Map */
ASSERT(IOPM_COUNT == 1);
RtlFillMemory(Tss->IoMaps[0].IoMap, IOPM_FULL_SIZE, 0xFF);
/* Initialize Interrupt Direction Maps */
p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
RtlZeroMemory(p, IOPM_DIRECTION_MAP_SIZE);
/* Add DPMI support for interrupts */
p[0] = 4;
p[3] = 0x18;
p[4] = 0x18;
/* Initialize the default Interrupt Direction Map */
p = Tss->IntDirectionMap;
RtlZeroMemory(Tss->IntDirectionMap, IOPM_DIRECTION_MAP_SIZE);
/* Add DPMI support */
p[0] = 4;
p[3] = 0x18;
p[4] = 0x18;
}
VOID
NTAPI
KiInitializeTSS(IN PKTSS Tss)
{
/* Set an invalid map base */
Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
/* Disable traps during Task Switches */
Tss->Flags = 0;
/* Set LDT and Ring 0 SS */
Tss->LDT = 0;
Tss->Ss0 = KGDT_R0_DATA;
}
INIT_FUNCTION
VOID
FASTCALL
Ki386InitializeTss(IN PKTSS Tss,
IN PKIDTENTRY Idt,
IN PKGDTENTRY Gdt)
{
PKGDTENTRY TssEntry, TaskGateEntry;
/* Initialize the boot TSS. */
TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
TssEntry->HighWord.Bits.Type = I386_TSS;
TssEntry->HighWord.Bits.Pres = 1;
TssEntry->HighWord.Bits.Dpl = 0;
KiInitializeTSS2(Tss, TssEntry);
KiInitializeTSS(Tss);
/* Load the task register */
Ke386SetTr(KGDT_TSS);
/* Setup the Task Gate for Double Fault Traps */
TaskGateEntry = (PKGDTENTRY)&Idt[8];
TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
TaskGateEntry->HighWord.Bits.Pres = 1;
TaskGateEntry->HighWord.Bits.Dpl = 0;
((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
/* Initialize the TSS used for handling double faults. */
Tss = (PKTSS)KiDoubleFaultTSS;
KiInitializeTSS(Tss);
Tss->CR3 = __readcr3();
Tss->Esp0 = KiDoubleFaultStack;
Tss->Esp = KiDoubleFaultStack;
Tss->Eip = PtrToUlong(KiTrap08);
Tss->Cs = KGDT_R0_CODE;
Tss->Fs = KGDT_R0_PCR;
Tss->Ss = Ke386GetSs();
Tss->Es = KGDT_R3_DATA | RPL_MASK;
Tss->Ds = KGDT_R3_DATA | RPL_MASK;
/* Setup the Double Trap TSS entry in the GDT */
TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
TssEntry->HighWord.Bits.Type = I386_TSS;
TssEntry->HighWord.Bits.Pres = 1;
TssEntry->HighWord.Bits.Dpl = 0;
TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
TssEntry->LimitLow = KTSS_IO_MAPS;
/* Now setup the NMI Task Gate */
TaskGateEntry = (PKGDTENTRY)&Idt[2];
TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
TaskGateEntry->HighWord.Bits.Pres = 1;
TaskGateEntry->HighWord.Bits.Dpl = 0;
((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
/* Initialize the actual TSS */
Tss = (PKTSS)KiNMITSS;
KiInitializeTSS(Tss);
Tss->CR3 = __readcr3();
Tss->Esp0 = KiDoubleFaultStack;
Tss->Esp = KiDoubleFaultStack;
Tss->Eip = PtrToUlong(KiTrap02);
Tss->Cs = KGDT_R0_CODE;
Tss->Fs = KGDT_R0_PCR;
Tss->Ss = Ke386GetSs();
Tss->Es = KGDT_R3_DATA | RPL_MASK;
Tss->Ds = KGDT_R3_DATA | RPL_MASK;
/* And its associated TSS Entry */
TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
TssEntry->HighWord.Bits.Type = I386_TSS;
TssEntry->HighWord.Bits.Pres = 1;
TssEntry->HighWord.Bits.Dpl = 0;
TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
TssEntry->LimitLow = KTSS_IO_MAPS;
}
VOID
NTAPI
KeFlushCurrentTb(VOID)
{
#if !defined(_GLOBAL_PAGES_ARE_AWESOME_)
/* Flush the TLB by resetting CR3 */
__writecr3(__readcr3());
#else
/* Check if global pages are enabled */
if (KeFeatureBits & KF_GLOBAL_PAGE)
{
ULONG Cr4;
/* Disable PGE (Note: may not have been enabled yet) */
Cr4 = __readcr4();
__writecr4(Cr4 & ~CR4_PGE);
/* Flush everything */
__writecr3(__readcr3());
/* Re-enable PGE */
__writecr4(Cr4);
}
else
{
/* No global pages, resetting CR3 is enough */
__writecr3(__readcr3());
}
#endif
}
VOID
NTAPI
KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
{
PKGDTENTRY TssEntry;
//
// Restore the CR registers
//
__writecr0(ProcessorState->SpecialRegisters.Cr0);
Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
__writecr3(ProcessorState->SpecialRegisters.Cr3);
if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
//
// Restore the DR registers
//
__writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
__writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
__writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
__writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
__writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
__writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
//
// Restore GDT and IDT
//
Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
__lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
//
// Clear the busy flag so we don't crash if we reload the same selector
//
TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
ProcessorState->SpecialRegisters.Tr);
TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
//
// Restore TSS and LDT
//
Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
}
VOID
NTAPI
KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
{
/* Save the CR registers */
ProcessorState->SpecialRegisters.Cr0 = __readcr0();
ProcessorState->SpecialRegisters.Cr2 = __readcr2();
ProcessorState->SpecialRegisters.Cr3 = __readcr3();
ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
__readcr4() : 0;
/* Save the DR registers */
ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
__writedr(7, 0);
/* Save GDT, IDT, LDT and TSS */
Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
__sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
ProcessorState->SpecialRegisters.Ldtr = Ke386GetLocalDescriptorTable();
}
INIT_FUNCTION
VOID
NTAPI
KiInitializeMachineType(VOID)
{
/* Set the Machine Type we got from NTLDR */
KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
}
INIT_FUNCTION
ULONG_PTR
NTAPI
KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
{
/* Set CS and ESP */
__writemsr(0x174, KGDT_R0_CODE);
__writemsr(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
/* Set LSTAR */
__writemsr(0x176, (ULONG_PTR)KiFastCallEntry);
return 0;
}
INIT_FUNCTION
VOID
NTAPI
KiRestoreFastSyscallReturnState(VOID)
{
/* Check if the CPU Supports fast system call */
if (KeFeatureBits & KF_FAST_SYSCALL)
{
/* Check if it has been disabled */
if (KiFastSystemCallDisable)
{
/* Disable fast system call */
KeFeatureBits &= ~KF_FAST_SYSCALL;
KiFastCallExitHandler = KiSystemCallTrapReturn;
DPRINT1("Support for SYSENTER disabled.\n");
}
else
{
/* Do an IPI to enable it */
KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
/* It's enabled, so use the proper exit stub */
KiFastCallExitHandler = KiSystemCallSysExitReturn;
DPRINT("Support for SYSENTER detected.\n");
}
}
else
{
/* Use the IRET handler */
KiFastCallExitHandler = KiSystemCallTrapReturn;
DPRINT1("No support for SYSENTER detected.\n");
}
}
INIT_FUNCTION
ULONG_PTR
NTAPI
Ki386EnableDE(IN ULONG_PTR Context)
{
/* Enable DE */
__writecr4(__readcr4() | CR4_DE);
return 0;
}
INIT_FUNCTION
ULONG_PTR
NTAPI
Ki386EnableFxsr(IN ULONG_PTR Context)
{
/* Enable FXSR */
__writecr4(__readcr4() | CR4_FXSR);
return 0;
}
INIT_FUNCTION
ULONG_PTR
NTAPI
Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
{
PKIDTENTRY IdtEntry;
/* Get the IDT Entry for Interrupt 0x13 */
IdtEntry = &((PKIPCR)KeGetPcr())->IDT[0x13];
/* Set it up */
IdtEntry->Selector = KGDT_R0_CODE;
IdtEntry->Offset = ((ULONG_PTR)KiTrap13 & 0xFFFF);
IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap13 >> 16) & 0xFFFF;
((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
/* Enable XMMI exceptions */
__writecr4(__readcr4() | CR4_XMMEXCPT);
return 0;
}
INIT_FUNCTION
VOID
NTAPI
KiI386PentiumLockErrataFixup(VOID)
{
KDESCRIPTOR IdtDescriptor = {0, 0, 0};
PKIDTENTRY NewIdt, NewIdt2;
/* Allocate memory for a new IDT */
NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
/* Put everything after the first 7 entries on a new page */
NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
/* Disable interrupts */
_disable();
/* Get the current IDT and copy it */
__sidt(&IdtDescriptor.Limit);
RtlCopyMemory(NewIdt2,
(PVOID)IdtDescriptor.Base,
IdtDescriptor.Limit + 1);
IdtDescriptor.Base = (ULONG)NewIdt2;
/* Set the new IDT */
__lidt(&IdtDescriptor.Limit);
((PKIPCR)KeGetPcr())->IDT = NewIdt2;
/* Restore interrupts */
_enable();
/* Set the first 7 entries as read-only to produce a fault */
MmSetPageProtect(NULL, NewIdt, PAGE_READONLY);
}
BOOLEAN
NTAPI
KeInvalidateAllCaches(VOID)
{
/* Only supported on Pentium Pro and higher */
if (KeI386CpuType < 6) return FALSE;
/* Invalidate all caches */
__wbinvd();
return TRUE;
}
VOID
FASTCALL
KeZeroPages(IN PVOID Address,
IN ULONG Size)
{
/* Not using XMMI in this routine */
RtlZeroMemory(Address, Size);
}
VOID
NTAPI
KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame,
IN PKEXCEPTION_FRAME ExceptionFrame)
{
PKPRCB Prcb = KeGetCurrentPrcb();
//
// Save full context
//
Prcb->ProcessorState.ContextFrame.ContextFlags = CONTEXT_FULL |
CONTEXT_DEBUG_REGISTERS;
KeTrapFrameToContext(TrapFrame, NULL, &Prcb->ProcessorState.ContextFrame);
//
// Save control registers
//
KiSaveProcessorControlState(&Prcb->ProcessorState);
}
INIT_FUNCTION
BOOLEAN
NTAPI
KiIsNpxPresent(VOID)
{
ULONG Cr0;
USHORT Magic;
/* Set magic */
Magic = 0xFFFF;
/* Read CR0 and mask out FPU flags */
Cr0 = __readcr0() & ~(CR0_MP | CR0_TS | CR0_EM | CR0_ET);
/* Store on FPU stack */
#ifdef _MSC_VER
__asm fninit;
__asm fnstsw Magic;
#else
asm volatile ("fninit;" "fnstsw %0" : "+m"(Magic));
#endif
/* Magic should now be cleared */
if (Magic & 0xFF)
{
/* You don't have an FPU -- enable emulation for now */
__writecr0(Cr0 | CR0_EM | CR0_TS);
return FALSE;
}
/* You have an FPU, enable it */
Cr0 |= CR0_ET;
/* Enable INT 16 on 486 and higher */
if (KeGetCurrentPrcb()->CpuType >= 3) Cr0 |= CR0_NE;
/* Set FPU state */
__writecr0(Cr0 | CR0_EM | CR0_TS);
return TRUE;
}
INIT_FUNCTION
BOOLEAN
NTAPI
KiIsNpxErrataPresent(VOID)
{
static double Value1 = 4195835.0, Value2 = 3145727.0;
INT ErrataPresent;
ULONG Cr0;
/* Disable interrupts */
_disable();
/* Read CR0 and remove FPU flags */
Cr0 = __readcr0();
__writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
/* Initialize FPU state */
Ke386FnInit();
/* Multiply the magic values and divide, we should get the result back */
#ifdef __GNUC__
__asm__ __volatile__
(
"fldl %1\n\t"
"fdivl %2\n\t"
"fmull %2\n\t"
"fldl %1\n\t"
"fsubp\n\t"
"fistpl %0\n\t"
: "=m" (ErrataPresent)
: "m" (Value1),
"m" (Value2)
);
#else
__asm
{
fld Value1
fdiv Value2
fmul Value2
fld Value1
fsubp st(1), st(0)
fistp ErrataPresent
};
#endif
/* Restore CR0 */
__writecr0(Cr0);
/* Enable interrupts */
_enable();
/* Return if there's an errata */
return ErrataPresent != 0;
}
VOID
NTAPI
KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea)
{
ULONG EFlags, Cr0;
PKTHREAD Thread, NpxThread;
PFX_SAVE_AREA FxSaveArea;
/* Save volatiles and disable interrupts */
EFlags = __readeflags();
_disable();
/* Save the PCR and get the current thread */
Thread = KeGetCurrentThread();
/* Check if we're already loaded */
if (Thread->NpxState != NPX_STATE_LOADED)
{
/* If there's nothing to load, quit */
if (!SaveArea)
{
/* Restore interrupt state and return */
__writeeflags(EFlags);
return;
}
/* Need FXSR support for this */
ASSERT(KeI386FxsrPresent == TRUE);
/* Check for sane CR0 */
Cr0 = __readcr0();
if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
{
/* Mask out FPU flags */
__writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
}
/* Get the NPX thread and check its FPU state */
NpxThread = KeGetCurrentPrcb()->NpxThread;
if ((NpxThread) && (NpxThread->NpxState == NPX_STATE_LOADED))
{
/* Get the FX frame and store the state there */
FxSaveArea = KiGetThreadNpxArea(NpxThread);
Ke386FxSave(FxSaveArea);
/* NPX thread has lost its state */
NpxThread->NpxState = NPX_STATE_NOT_LOADED;
}
/* Now load NPX state from the NPX area */
FxSaveArea = KiGetThreadNpxArea(Thread);
Ke386FxStore(FxSaveArea);
}
else
{
/* Check for sane CR0 */
Cr0 = __readcr0();
if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
{
/* Mask out FPU flags */
__writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
}
/* Get FX frame */
FxSaveArea = KiGetThreadNpxArea(Thread);
Thread->NpxState = NPX_STATE_NOT_LOADED;
/* Save state if supported by CPU */
if (KeI386FxsrPresent) Ke386FxSave(FxSaveArea);
}
/* Now save the FN state wherever it was requested */
if (SaveArea) Ke386FnSave(SaveArea);
/* Clear NPX thread */
KeGetCurrentPrcb()->NpxThread = NULL;
/* Add the CR0 from the NPX frame */
Cr0 |= NPX_STATE_NOT_LOADED;
Cr0 |= FxSaveArea->Cr0NpxState;
__writecr0(Cr0);
/* Restore interrupt state */
__writeeflags(EFlags);
}
/* PUBLIC FUNCTIONS **********************************************************/
/*
* @implemented
*/
VOID
NTAPI
KiCoprocessorError(VOID)
{
PFX_SAVE_AREA NpxArea;
/* Get the FPU area */
NpxArea = KiGetThreadNpxArea(KeGetCurrentThread());
/* Set CR0_TS */
NpxArea->Cr0NpxState = CR0_TS;
__writecr0(__readcr0() | CR0_TS);
}
/*
* @implemented
*/
NTSTATUS
NTAPI
KeSaveFloatingPointState(OUT PKFLOATING_SAVE Save)
{
PFNSAVE_FORMAT FpState;
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
DPRINT1("%s is not really implemented\n", __FUNCTION__);
/* check if we are doing software emulation */
if (!KeI386NpxPresent) return STATUS_ILLEGAL_FLOAT_CONTEXT;
FpState = ExAllocatePool(NonPagedPool, sizeof (FNSAVE_FORMAT));
if (!FpState) return STATUS_INSUFFICIENT_RESOURCES;
*((PVOID *) Save) = FpState;
#ifdef __GNUC__
asm volatile("fnsave %0\n\t" : "=m" (*FpState));
#else
__asm
{
mov eax, [FpState]
fnsave [eax]
};
#endif
KeGetCurrentThread()->Header.NpxIrql = KeGetCurrentIrql();
return STATUS_SUCCESS;
}
/*
* @implemented
*/
NTSTATUS
NTAPI
KeRestoreFloatingPointState(IN PKFLOATING_SAVE Save)
{
PFNSAVE_FORMAT FpState = *((PVOID *) Save);
ASSERT(KeGetCurrentThread()->Header.NpxIrql == KeGetCurrentIrql());
DPRINT1("%s is not really implemented\n", __FUNCTION__);
#ifdef __GNUC__
asm volatile("fnclex\n\t");
asm volatile("frstor %0\n\t" : "=m" (*FpState));
#else
__asm
{
fnclex
mov eax, [FpState]
frstor [eax]
};
#endif
ExFreePool(FpState);
return STATUS_SUCCESS;
}
/*
* @implemented
*/
ULONG
NTAPI
KeGetRecommendedSharedDataAlignment(VOID)
{
/* Return the global variable */
return KeLargestCacheLine;
}
VOID
NTAPI
KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
IN PVOID Ignored1,
IN PVOID Ignored2,
IN PVOID Ignored3)
{
/* Signal this packet as done */
KiIpiSignalPacketDone(PacketContext);
/* Flush the TB for the Current CPU */
KeFlushCurrentTb();
}
/*
* @implemented
*/
VOID
NTAPI
KeFlushEntireTb(IN BOOLEAN Invalid,
IN BOOLEAN AllProcessors)
{
KIRQL OldIrql;
#ifdef CONFIG_SMP
KAFFINITY TargetAffinity;
PKPRCB Prcb = KeGetCurrentPrcb();
#endif
/* Raise the IRQL for the TB Flush */
OldIrql = KeRaiseIrqlToSynchLevel();
#ifdef CONFIG_SMP
/* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
/* Get the current processor affinity, and exclude ourselves */
TargetAffinity = KeActiveProcessors;
TargetAffinity &= ~Prcb->SetMember;
/* Make sure this is MP */
if (TargetAffinity)
{
/* Send an IPI TB flush to the other processors */
KiIpiSendPacket(TargetAffinity,
KiFlushTargetEntireTb,
NULL,
0,
NULL);
}
#endif
/* Flush the TB for the Current CPU, and update the flush stamp */
KeFlushCurrentTb();
#ifdef CONFIG_SMP
/* If this is MP, wait for the other processors to finish */
if (TargetAffinity)
{
/* Sanity check */
ASSERT(Prcb == KeGetCurrentPrcb());
/* FIXME: TODO */
ASSERTMSG("Not yet implemented\n", FALSE);
}
#endif
/* Update the flush stamp and return to original IRQL */
InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
KeLowerIrql(OldIrql);
}
/*
* @implemented
*/
VOID
NTAPI
KeSetDmaIoCoherency(IN ULONG Coherency)
{
/* Save the coherency globally */
KiDmaIoCoherency = Coherency;
}
/*
* @implemented
*/
KAFFINITY
NTAPI
KeQueryActiveProcessors(VOID)
{
PAGED_CODE();
/* Simply return the number of active processors */
return KeActiveProcessors;
}
/*
* @implemented
*/
VOID
__cdecl
KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
{
/* Capture the context */
RtlCaptureContext(&State->ContextFrame);
/* Capture the control state */
KiSaveProcessorControlState(State);
}
Reference in a new issue View git blame Copy permalink