mirror of
https://github.com/reactos/reactos.git
synced 2024-08-28 14:18:41 +00:00
![Stanislav Motylkov](/assets/img/avatar_default.png)
KiGetFeatureBits() is now being called in the early boot phase 0 when the Kernel Debugger is not yet initialized, so debug prints are not available here. Move the debug prints into a new function and call it at the right time. CORE-18023
1678 lines
49 KiB
C
1678 lines
49 KiB
C
/*
|
|
* PROJECT: ReactOS Kernel
|
|
* LICENSE: GPL - See COPYING in the top level directory
|
|
* FILE: ntoskrnl/ke/i386/cpu.c
|
|
* PURPOSE: Routines for CPU-level support
|
|
* PROGRAMMERS: Alex Ionescu (alex.ionescu@reactos.org)
|
|
*/
|
|
|
|
/* INCLUDES *****************************************************************/
|
|
|
|
#include <ntoskrnl.h>
|
|
#define NDEBUG
|
|
#include <debug.h>
|
|
|
|
#include <xmmintrin.h>
|
|
|
|
/* GLOBALS *******************************************************************/
|
|
|
|
/* The TSS to use for Double Fault Traps (INT 0x9) */
|
|
UCHAR KiDoubleFaultTSS[KTSS_IO_MAPS];
|
|
|
|
/* The TSS to use for NMI Fault Traps (INT 0x2) */
|
|
UCHAR KiNMITSS[KTSS_IO_MAPS];
|
|
|
|
/* CPU Features and Flags */
|
|
ULONG KeI386CpuType;
|
|
ULONG KeI386CpuStep;
|
|
ULONG KiFastSystemCallDisable = 0;
|
|
ULONG KeI386NpxPresent = TRUE;
|
|
ULONG KiMXCsrMask = 0;
|
|
ULONG MxcsrFeatureMask = 0;
|
|
ULONG KeI386XMMIPresent = 0;
|
|
ULONG KeI386FxsrPresent = 0;
|
|
ULONG KeI386MachineType;
|
|
ULONG Ke386Pae = FALSE;
|
|
ULONG Ke386NoExecute = FALSE;
|
|
ULONG KeLargestCacheLine = 0x40;
|
|
ULONG KeDcacheFlushCount = 0;
|
|
ULONG KeIcacheFlushCount = 0;
|
|
ULONG KiDmaIoCoherency = 0;
|
|
ULONG KePrefetchNTAGranularity = 32;
|
|
BOOLEAN KiI386PentiumLockErrataPresent;
|
|
BOOLEAN KiSMTProcessorsPresent;
|
|
|
|
/* The distance between SYSEXIT and IRETD return modes */
|
|
UCHAR KiSystemCallExitAdjust;
|
|
|
|
/* The offset that was applied -- either 0 or the value above */
|
|
UCHAR KiSystemCallExitAdjusted;
|
|
|
|
/* Whether the adjustment was already done once */
|
|
BOOLEAN KiFastCallCopyDoneOnce;
|
|
|
|
/* Flush data */
|
|
volatile LONG KiTbFlushTimeStamp;
|
|
|
|
/* CPU Signatures */
|
|
static const CHAR CmpIntelID[] = "GenuineIntel";
|
|
static const CHAR CmpAmdID[] = "AuthenticAMD";
|
|
static const CHAR CmpCyrixID[] = "CyrixInstead";
|
|
static const CHAR CmpTransmetaID[] = "GenuineTMx86";
|
|
static const CHAR CmpCentaurID[] = "CentaurHauls";
|
|
static const CHAR CmpRiseID[] = "RiseRiseRise";
|
|
|
|
typedef union _CPU_SIGNATURE
|
|
{
|
|
struct
|
|
{
|
|
ULONG Step : 4;
|
|
ULONG Model : 4;
|
|
ULONG Family : 4;
|
|
ULONG Unused : 4;
|
|
ULONG ExtendedModel : 4;
|
|
ULONG ExtendedFamily : 8;
|
|
ULONG Unused2 : 4;
|
|
};
|
|
ULONG AsULONG;
|
|
} CPU_SIGNATURE;
|
|
|
|
/* FX area alignment size */
|
|
#define FXSAVE_ALIGN 15
|
|
|
|
/* SUPPORT ROUTINES FOR MSVC COMPATIBILITY ***********************************/
|
|
|
|
/* NSC/Cyrix CPU configuration register index */
|
|
#define CX86_CCR1 0xc1
|
|
|
|
/* NSC/Cyrix CPU indexed register access macros */
|
|
static __inline
|
|
UCHAR
|
|
getCx86(UCHAR reg)
|
|
{
|
|
WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22, reg);
|
|
return READ_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23);
|
|
}
|
|
|
|
static __inline
|
|
void
|
|
setCx86(UCHAR reg, UCHAR data)
|
|
{
|
|
WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x22, reg);
|
|
WRITE_PORT_UCHAR((PUCHAR)(ULONG_PTR)0x23, data);
|
|
}
|
|
|
|
/* FUNCTIONS *****************************************************************/
|
|
|
|
CODE_SEG("INIT")
|
|
ULONG
|
|
NTAPI
|
|
KiGetCpuVendor(VOID)
|
|
{
|
|
PKPRCB Prcb = KeGetCurrentPrcb();
|
|
CPU_INFO CpuInfo;
|
|
|
|
/* Get the Vendor ID */
|
|
KiCpuId(&CpuInfo, 0);
|
|
|
|
/* Copy it to the PRCB and null-terminate it */
|
|
*(ULONG*)&Prcb->VendorString[0] = CpuInfo.Ebx;
|
|
*(ULONG*)&Prcb->VendorString[4] = CpuInfo.Edx;
|
|
*(ULONG*)&Prcb->VendorString[8] = CpuInfo.Ecx;
|
|
Prcb->VendorString[12] = 0;
|
|
|
|
/* Now check the CPU Type */
|
|
if (!strcmp(Prcb->VendorString, CmpIntelID))
|
|
{
|
|
return CPU_INTEL;
|
|
}
|
|
else if (!strcmp(Prcb->VendorString, CmpAmdID))
|
|
{
|
|
return CPU_AMD;
|
|
}
|
|
else if (!strcmp(Prcb->VendorString, CmpCyrixID))
|
|
{
|
|
DPRINT1("Cyrix CPU support not fully tested!\n");
|
|
return CPU_CYRIX;
|
|
}
|
|
else if (!strcmp(Prcb->VendorString, CmpTransmetaID))
|
|
{
|
|
DPRINT1("Transmeta CPU support not fully tested!\n");
|
|
return CPU_TRANSMETA;
|
|
}
|
|
else if (!strcmp(Prcb->VendorString, CmpCentaurID))
|
|
{
|
|
DPRINT1("Centaur CPU support not fully tested!\n");
|
|
return CPU_CENTAUR;
|
|
}
|
|
else if (!strcmp(Prcb->VendorString, CmpRiseID))
|
|
{
|
|
DPRINT1("Rise CPU support not fully tested!\n");
|
|
return CPU_RISE;
|
|
}
|
|
|
|
/* Unknown CPU */
|
|
DPRINT1("%s CPU support not fully tested!\n", Prcb->VendorString);
|
|
return CPU_UNKNOWN;
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
NTAPI
|
|
KiSetProcessorType(VOID)
|
|
{
|
|
CPU_INFO CpuInfo;
|
|
CPU_SIGNATURE CpuSignature;
|
|
BOOLEAN ExtendModel;
|
|
ULONG Stepping, Type;
|
|
|
|
/* Do CPUID 1 now */
|
|
KiCpuId(&CpuInfo, 1);
|
|
|
|
/*
|
|
* Get the Stepping and Type. The stepping contains both the
|
|
* Model and the Step, while the Type contains the returned Family.
|
|
*
|
|
* For the stepping, we convert this: zzzzzzxy into this: x0y
|
|
*/
|
|
CpuSignature.AsULONG = CpuInfo.Eax;
|
|
Stepping = CpuSignature.Model;
|
|
ExtendModel = (CpuSignature.Family == 15);
|
|
#if ( (NTDDI_VERSION >= NTDDI_WINXPSP2) && (NTDDI_VERSION < NTDDI_WS03) ) || (NTDDI_VERSION >= NTDDI_WS03SP1)
|
|
if (CpuSignature.Family == 6)
|
|
{
|
|
ULONG Vendor = KiGetCpuVendor();
|
|
ExtendModel |= (Vendor == CPU_INTEL);
|
|
#if (NTDDI_VERSION >= NTDDI_WIN8)
|
|
ExtendModel |= (Vendor == CPU_CENTAUR);
|
|
#endif
|
|
}
|
|
#endif
|
|
if (ExtendModel)
|
|
{
|
|
/* Add ExtendedModel to distinguish from non-extended values. */
|
|
Stepping |= (CpuSignature.ExtendedModel << 4);
|
|
}
|
|
Stepping = (Stepping << 8) | CpuSignature.Step;
|
|
Type = CpuSignature.Family;
|
|
if (CpuSignature.Family == 15)
|
|
{
|
|
/* Add ExtendedFamily to distinguish from non-extended values.
|
|
* It must not be larger than 0xF0 to avoid overflow. */
|
|
Type += min(CpuSignature.ExtendedFamily, 0xF0);
|
|
}
|
|
|
|
/* Save them in the PRCB */
|
|
KeGetCurrentPrcb()->CpuID = TRUE;
|
|
KeGetCurrentPrcb()->CpuType = (UCHAR)Type;
|
|
KeGetCurrentPrcb()->CpuStep = (USHORT)Stepping;
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
ULONG
|
|
NTAPI
|
|
KiGetFeatureBits(VOID)
|
|
{
|
|
PKPRCB Prcb = KeGetCurrentPrcb();
|
|
ULONG Vendor;
|
|
ULONG FeatureBits = KF_WORKING_PTE;
|
|
CPU_INFO CpuInfo, DummyCpuInfo;
|
|
UCHAR Ccr1;
|
|
BOOLEAN ExtendedCPUID = TRUE;
|
|
ULONG CpuFeatures = 0;
|
|
|
|
/* Get the Vendor ID */
|
|
Vendor = KiGetCpuVendor();
|
|
|
|
/* Make sure we got a valid vendor ID at least. */
|
|
if (!Vendor) return FeatureBits;
|
|
|
|
/* Get the CPUID Info. Features are in Reg[3]. */
|
|
KiCpuId(&CpuInfo, 1);
|
|
|
|
/* Set the initial APIC ID */
|
|
Prcb->InitialApicId = (UCHAR)(CpuInfo.Ebx >> 24);
|
|
|
|
switch (Vendor)
|
|
{
|
|
/* Intel CPUs */
|
|
case CPU_INTEL:
|
|
|
|
/* Check if it's a P6 */
|
|
if (Prcb->CpuType == 6)
|
|
{
|
|
/* Perform the special sequence to get the MicroCode Signature */
|
|
__writemsr(0x8B, 0);
|
|
KiCpuId(&DummyCpuInfo, 1);
|
|
Prcb->UpdateSignature.QuadPart = __readmsr(0x8B);
|
|
}
|
|
else if (Prcb->CpuType == 5)
|
|
{
|
|
/* On P5, enable workaround for the LOCK errata. */
|
|
KiI386PentiumLockErrataPresent = TRUE;
|
|
}
|
|
|
|
/* Check for broken P6 with bad SMP PTE implementation */
|
|
if (((CpuInfo.Eax & 0x0FF0) == 0x0610 && (CpuInfo.Eax & 0x000F) <= 0x9) ||
|
|
((CpuInfo.Eax & 0x0FF0) == 0x0630 && (CpuInfo.Eax & 0x000F) <= 0x4))
|
|
{
|
|
/* Remove support for correct PTE support. */
|
|
FeatureBits &= ~KF_WORKING_PTE;
|
|
}
|
|
|
|
/* Check if the CPU is too old to support SYSENTER */
|
|
if ((Prcb->CpuType < 6) ||
|
|
((Prcb->CpuType == 6) && (Prcb->CpuStep < 0x0303)))
|
|
{
|
|
/* Disable it */
|
|
CpuInfo.Edx &= ~0x800;
|
|
}
|
|
|
|
break;
|
|
|
|
/* AMD CPUs */
|
|
case CPU_AMD:
|
|
|
|
/* Check if this is a K5 or K6. (family 5) */
|
|
if ((CpuInfo.Eax & 0x0F00) == 0x0500)
|
|
{
|
|
/* Get the Model Number */
|
|
switch (CpuInfo.Eax & 0x00F0)
|
|
{
|
|
/* Model 1: K5 - 5k86 (initial models) */
|
|
case 0x0010:
|
|
|
|
/* Check if this is Step 0 or 1. They don't support PGE */
|
|
if ((CpuInfo.Eax & 0x000F) > 0x03) break;
|
|
|
|
/* Model 0: K5 - SSA5 */
|
|
case 0x0000:
|
|
|
|
/* Model 0 doesn't support PGE at all. */
|
|
CpuInfo.Edx &= ~0x2000;
|
|
break;
|
|
|
|
/* Model 8: K6-2 */
|
|
case 0x0080:
|
|
|
|
/* K6-2, Step 8 and over have support for MTRR. */
|
|
if ((CpuInfo.Eax & 0x000F) >= 0x8) FeatureBits |= KF_AMDK6MTRR;
|
|
break;
|
|
|
|
/* Model 9: K6-III
|
|
Model D: K6-2+, K6-III+ */
|
|
case 0x0090:
|
|
case 0x00D0:
|
|
|
|
FeatureBits |= KF_AMDK6MTRR;
|
|
break;
|
|
}
|
|
}
|
|
else if((CpuInfo.Eax & 0x0F00) < 0x0500)
|
|
{
|
|
/* Families below 5 don't support PGE, PSE or CMOV at all */
|
|
CpuInfo.Edx &= ~(0x08 | 0x2000 | 0x8000);
|
|
|
|
/* They also don't support advanced CPUID functions. */
|
|
ExtendedCPUID = FALSE;
|
|
}
|
|
|
|
break;
|
|
|
|
/* Cyrix CPUs */
|
|
case CPU_CYRIX:
|
|
|
|
/* Workaround the "COMA" bug on 6x family of Cyrix CPUs */
|
|
if (Prcb->CpuType == 6 &&
|
|
Prcb->CpuStep <= 1)
|
|
{
|
|
/* Get CCR1 value */
|
|
Ccr1 = getCx86(CX86_CCR1);
|
|
|
|
/* Enable the NO_LOCK bit */
|
|
Ccr1 |= 0x10;
|
|
|
|
/* Set the new CCR1 value */
|
|
setCx86(CX86_CCR1, Ccr1);
|
|
}
|
|
|
|
break;
|
|
|
|
/* Transmeta CPUs */
|
|
case CPU_TRANSMETA:
|
|
|
|
/* Enable CMPXCHG8B if the family (>= 5), model and stepping (>= 4.2) support it */
|
|
if ((CpuInfo.Eax & 0x0FFF) >= 0x0542)
|
|
{
|
|
__writemsr(0x80860004, __readmsr(0x80860004) | 0x0100);
|
|
FeatureBits |= KF_CMPXCHG8B;
|
|
}
|
|
|
|
break;
|
|
|
|
/* Centaur, IDT, Rise and VIA CPUs */
|
|
case CPU_CENTAUR:
|
|
case CPU_RISE:
|
|
|
|
/* These CPUs don't report the presence of CMPXCHG8B through CPUID.
|
|
However, this feature exists and operates properly without any additional steps. */
|
|
FeatureBits |= KF_CMPXCHG8B;
|
|
|
|
break;
|
|
}
|
|
|
|
/* Set the current features */
|
|
CpuFeatures = CpuInfo.Edx;
|
|
|
|
/* Convert all CPUID Feature bits into our format */
|
|
if (CpuFeatures & X86_FEATURE_VME) FeatureBits |= KF_V86_VIS | KF_CR4;
|
|
if (CpuFeatures & X86_FEATURE_PSE) FeatureBits |= KF_LARGE_PAGE | KF_CR4;
|
|
if (CpuFeatures & X86_FEATURE_TSC) FeatureBits |= KF_RDTSC;
|
|
if (CpuFeatures & X86_FEATURE_CX8) FeatureBits |= KF_CMPXCHG8B;
|
|
if (CpuFeatures & X86_FEATURE_SYSCALL) FeatureBits |= KF_FAST_SYSCALL;
|
|
if (CpuFeatures & X86_FEATURE_MTTR) FeatureBits |= KF_MTRR;
|
|
if (CpuFeatures & X86_FEATURE_PGE) FeatureBits |= KF_GLOBAL_PAGE | KF_CR4;
|
|
if (CpuFeatures & X86_FEATURE_CMOV) FeatureBits |= KF_CMOV;
|
|
if (CpuFeatures & X86_FEATURE_PAT) FeatureBits |= KF_PAT;
|
|
if (CpuFeatures & X86_FEATURE_DS) FeatureBits |= KF_DTS;
|
|
if (CpuFeatures & X86_FEATURE_MMX) FeatureBits |= KF_MMX;
|
|
if (CpuFeatures & X86_FEATURE_FXSR) FeatureBits |= KF_FXSR;
|
|
if (CpuFeatures & X86_FEATURE_SSE) FeatureBits |= KF_XMMI;
|
|
if (CpuFeatures & X86_FEATURE_SSE2) FeatureBits |= KF_XMMI64;
|
|
|
|
/* Check if the CPU has hyper-threading */
|
|
if (CpuFeatures & X86_FEATURE_HT)
|
|
{
|
|
/* Set the number of logical CPUs */
|
|
Prcb->LogicalProcessorsPerPhysicalProcessor = (UCHAR)(CpuInfo.Ebx >> 16);
|
|
if (Prcb->LogicalProcessorsPerPhysicalProcessor > 1)
|
|
{
|
|
/* We're on dual-core */
|
|
KiSMTProcessorsPresent = TRUE;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
/* We only have a single CPU */
|
|
Prcb->LogicalProcessorsPerPhysicalProcessor = 1;
|
|
}
|
|
|
|
/* Check if CPUID 0x80000000 is supported */
|
|
if (ExtendedCPUID)
|
|
{
|
|
/* Do the call */
|
|
KiCpuId(&CpuInfo, 0x80000000);
|
|
if ((CpuInfo.Eax & 0xffffff00) == 0x80000000)
|
|
{
|
|
/* Check if CPUID 0x80000001 is supported */
|
|
if (CpuInfo.Eax >= 0x80000001)
|
|
{
|
|
/* Check which extended features are available. */
|
|
KiCpuId(&CpuInfo, 0x80000001);
|
|
|
|
/* Check if NX-bit is supported */
|
|
if (CpuInfo.Edx & X86_FEATURE_NX) FeatureBits |= KF_NX_BIT;
|
|
|
|
/* Now handle each features for each CPU Vendor */
|
|
switch (Vendor)
|
|
{
|
|
case CPU_AMD:
|
|
case CPU_CENTAUR:
|
|
if (CpuInfo.Edx & 0x80000000) FeatureBits |= KF_3DNOW;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Return the Feature Bits */
|
|
return FeatureBits;
|
|
}
|
|
|
|
#if DBG
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
KiReportCpuFeatures(VOID)
|
|
{
|
|
ULONG CpuFeatures = 0;
|
|
CPU_INFO CpuInfo;
|
|
|
|
if (KiGetCpuVendor())
|
|
{
|
|
KiCpuId(&CpuInfo, 1);
|
|
CpuFeatures = CpuInfo.Edx;
|
|
}
|
|
|
|
DPRINT1("Supported CPU features: ");
|
|
|
|
#define print_kf_bit(kf_value) if (KeFeatureBits & kf_value) DbgPrint(#kf_value " ")
|
|
print_kf_bit(KF_V86_VIS);
|
|
print_kf_bit(KF_RDTSC);
|
|
print_kf_bit(KF_CR4);
|
|
print_kf_bit(KF_CMOV);
|
|
print_kf_bit(KF_GLOBAL_PAGE);
|
|
print_kf_bit(KF_LARGE_PAGE);
|
|
print_kf_bit(KF_MTRR);
|
|
print_kf_bit(KF_CMPXCHG8B);
|
|
print_kf_bit(KF_MMX);
|
|
print_kf_bit(KF_WORKING_PTE);
|
|
print_kf_bit(KF_PAT);
|
|
print_kf_bit(KF_FXSR);
|
|
print_kf_bit(KF_FAST_SYSCALL);
|
|
print_kf_bit(KF_XMMI);
|
|
print_kf_bit(KF_3DNOW);
|
|
print_kf_bit(KF_AMDK6MTRR);
|
|
print_kf_bit(KF_XMMI64);
|
|
print_kf_bit(KF_DTS);
|
|
print_kf_bit(KF_NX_BIT);
|
|
print_kf_bit(KF_NX_DISABLED);
|
|
print_kf_bit(KF_NX_ENABLED);
|
|
#undef print_kf_bit
|
|
|
|
#define print_cf(cpu_flag) if (CpuFeatures & cpu_flag) DbgPrint(#cpu_flag " ")
|
|
print_cf(X86_FEATURE_PAE);
|
|
print_cf(X86_FEATURE_APIC);
|
|
print_cf(X86_FEATURE_HT);
|
|
#undef print_cf
|
|
|
|
DbgPrint("\n");
|
|
}
|
|
#endif // DBG
|
|
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
NTAPI
|
|
KiGetCacheInformation(VOID)
|
|
{
|
|
PKIPCR Pcr = (PKIPCR)KeGetPcr();
|
|
CPU_INFO CpuInfo;
|
|
ULONG CacheRequests = 0, i;
|
|
ULONG CurrentRegister;
|
|
UCHAR RegisterByte, Associativity = 0;
|
|
ULONG Size, CacheLine = 64, CurrentSize = 0;
|
|
BOOLEAN FirstPass = TRUE;
|
|
|
|
/* Set default L2 size */
|
|
Pcr->SecondLevelCacheSize = 0;
|
|
|
|
/* Check the Vendor ID */
|
|
switch (KiGetCpuVendor())
|
|
{
|
|
/* Handle Intel case */
|
|
case CPU_INTEL:
|
|
|
|
/* Check if we support CPUID 2 */
|
|
KiCpuId(&CpuInfo, 0);
|
|
if (CpuInfo.Eax >= 2)
|
|
{
|
|
/* We need to loop for the number of times CPUID will tell us to */
|
|
do
|
|
{
|
|
/* Do the CPUID call */
|
|
KiCpuId(&CpuInfo, 2);
|
|
|
|
/* Check if it was the first call */
|
|
if (FirstPass)
|
|
{
|
|
/*
|
|
* The number of times to loop is the first byte. Read
|
|
* it and then destroy it so we don't get confused.
|
|
*/
|
|
CacheRequests = CpuInfo.Eax & 0xFF;
|
|
CpuInfo.Eax &= 0xFFFFFF00;
|
|
|
|
/* Don't go over this again */
|
|
FirstPass = FALSE;
|
|
}
|
|
|
|
/* Loop all 4 registers */
|
|
for (i = 0; i < 4; i++)
|
|
{
|
|
/* Get the current register */
|
|
CurrentRegister = CpuInfo.AsUINT32[i];
|
|
|
|
/*
|
|
* If the upper bit is set, then this register should
|
|
* be skipped.
|
|
*/
|
|
if (CurrentRegister & 0x80000000) continue;
|
|
|
|
/* Keep looping for every byte inside this register */
|
|
while (CurrentRegister)
|
|
{
|
|
/* Read a byte, skip a byte. */
|
|
RegisterByte = (UCHAR)(CurrentRegister & 0xFF);
|
|
CurrentRegister >>= 8;
|
|
if (!RegisterByte) continue;
|
|
|
|
Size = 0;
|
|
switch (RegisterByte)
|
|
{
|
|
case 0x06:
|
|
case 0x08:
|
|
KePrefetchNTAGranularity = 32;
|
|
break;
|
|
case 0x09:
|
|
KePrefetchNTAGranularity = 64;
|
|
break;
|
|
case 0x0a:
|
|
case 0x0c:
|
|
KePrefetchNTAGranularity = 32;
|
|
break;
|
|
case 0x0d:
|
|
case 0x0e:
|
|
KePrefetchNTAGranularity = 64;
|
|
break;
|
|
case 0x1d:
|
|
Size = 128 * 1024;
|
|
Associativity = 2;
|
|
break;
|
|
case 0x21:
|
|
Size = 256 * 1024;
|
|
Associativity = 8;
|
|
break;
|
|
case 0x24:
|
|
Size = 1024 * 1024;
|
|
Associativity = 16;
|
|
break;
|
|
case 0x2c:
|
|
case 0x30:
|
|
KePrefetchNTAGranularity = 64;
|
|
break;
|
|
case 0x41:
|
|
case 0x42:
|
|
case 0x43:
|
|
case 0x44:
|
|
case 0x45:
|
|
Size = (1 << (RegisterByte - 0x41)) * 128 * 1024;
|
|
Associativity = 4;
|
|
break;
|
|
case 0x48:
|
|
Size = 3 * 1024 * 1024;
|
|
Associativity = 12;
|
|
break;
|
|
case 0x49:
|
|
Size = 4 * 1024 * 1024;
|
|
Associativity = 16;
|
|
break;
|
|
case 0x4e:
|
|
Size = 6 * 1024 * 1024;
|
|
Associativity = 24;
|
|
break;
|
|
case 0x60:
|
|
case 0x66:
|
|
case 0x67:
|
|
case 0x68:
|
|
KePrefetchNTAGranularity = 64;
|
|
break;
|
|
case 0x78:
|
|
Size = 1024 * 1024;
|
|
Associativity = 4;
|
|
break;
|
|
case 0x79:
|
|
case 0x7a:
|
|
case 0x7b:
|
|
case 0x7c:
|
|
case 0x7d:
|
|
Size = (1 << (RegisterByte - 0x79)) * 128 * 1024;
|
|
Associativity = 8;
|
|
break;
|
|
case 0x7f:
|
|
Size = 512 * 1024;
|
|
Associativity = 2;
|
|
break;
|
|
case 0x80:
|
|
Size = 512 * 1024;
|
|
Associativity = 8;
|
|
break;
|
|
case 0x82:
|
|
case 0x83:
|
|
case 0x84:
|
|
case 0x85:
|
|
Size = (1 << (RegisterByte - 0x82)) * 256 * 1024;
|
|
Associativity = 8;
|
|
break;
|
|
case 0x86:
|
|
Size = 512 * 1024;
|
|
Associativity = 4;
|
|
break;
|
|
case 0x87:
|
|
Size = 1024 * 1024;
|
|
Associativity = 8;
|
|
break;
|
|
case 0xf0:
|
|
KePrefetchNTAGranularity = 64;
|
|
break;
|
|
case 0xf1:
|
|
KePrefetchNTAGranularity = 128;
|
|
break;
|
|
}
|
|
if (Size && (Size / Associativity) > CurrentSize)
|
|
{
|
|
/* Set the L2 Cache Size and Associativity */
|
|
CurrentSize = Size / Associativity;
|
|
Pcr->SecondLevelCacheSize = Size;
|
|
Pcr->SecondLevelCacheAssociativity = Associativity;
|
|
}
|
|
}
|
|
}
|
|
} while (--CacheRequests);
|
|
}
|
|
break;
|
|
|
|
case CPU_AMD:
|
|
|
|
/* Check if we support CPUID 0x80000005 */
|
|
KiCpuId(&CpuInfo, 0x80000000);
|
|
if (CpuInfo.Eax >= 0x80000005)
|
|
{
|
|
/* Get L1 size first */
|
|
KiCpuId(&CpuInfo, 0x80000005);
|
|
KePrefetchNTAGranularity = CpuInfo.Ecx & 0xFF;
|
|
|
|
/* Check if we support CPUID 0x80000006 */
|
|
KiCpuId(&CpuInfo, 0x80000000);
|
|
if (CpuInfo.Eax >= 0x80000006)
|
|
{
|
|
/* Get 2nd level cache and tlb size */
|
|
KiCpuId(&CpuInfo, 0x80000006);
|
|
|
|
/* Cache line size */
|
|
CacheLine = CpuInfo.Ecx & 0xFF;
|
|
|
|
/* Hardcode associativity */
|
|
RegisterByte = (CpuInfo.Ecx >> 12) & 0xFF;
|
|
switch (RegisterByte)
|
|
{
|
|
case 2:
|
|
Associativity = 2;
|
|
break;
|
|
|
|
case 4:
|
|
Associativity = 4;
|
|
break;
|
|
|
|
case 6:
|
|
Associativity = 8;
|
|
break;
|
|
|
|
case 8:
|
|
case 15:
|
|
Associativity = 16;
|
|
break;
|
|
|
|
default:
|
|
Associativity = 1;
|
|
break;
|
|
}
|
|
|
|
/* Compute size */
|
|
Size = (CpuInfo.Ecx >> 16) << 10;
|
|
|
|
/* Hack for Model 6, Steping 300 */
|
|
if ((KeGetCurrentPrcb()->CpuType == 6) &&
|
|
(KeGetCurrentPrcb()->CpuStep == 0x300))
|
|
{
|
|
/* Stick 64K in there */
|
|
Size = 64 * 1024;
|
|
}
|
|
|
|
/* Set the L2 Cache Size and associativity */
|
|
Pcr->SecondLevelCacheSize = Size;
|
|
Pcr->SecondLevelCacheAssociativity = Associativity;
|
|
}
|
|
}
|
|
break;
|
|
|
|
case CPU_CYRIX:
|
|
case CPU_TRANSMETA:
|
|
case CPU_CENTAUR:
|
|
case CPU_RISE:
|
|
|
|
/* FIXME */
|
|
break;
|
|
}
|
|
|
|
/* Set the cache line */
|
|
if (CacheLine > KeLargestCacheLine) KeLargestCacheLine = CacheLine;
|
|
DPRINT1("Prefetch Cache: %lu bytes\tL2 Cache: %lu bytes\tL2 Cache Line: %lu bytes\tL2 Cache Associativity: %lu\n",
|
|
KePrefetchNTAGranularity,
|
|
Pcr->SecondLevelCacheSize,
|
|
KeLargestCacheLine,
|
|
Pcr->SecondLevelCacheAssociativity);
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
NTAPI
|
|
KiSetCR0Bits(VOID)
|
|
{
|
|
ULONG Cr0;
|
|
|
|
/* Save current CR0 */
|
|
Cr0 = __readcr0();
|
|
|
|
/* If this is a 486, enable Write-Protection */
|
|
if (KeGetCurrentPrcb()->CpuType > 3) Cr0 |= CR0_WP;
|
|
|
|
/* Set new Cr0 */
|
|
__writecr0(Cr0);
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
NTAPI
|
|
KiInitializeTSS2(IN PKTSS Tss,
|
|
IN PKGDTENTRY TssEntry OPTIONAL)
|
|
{
|
|
PUCHAR p;
|
|
|
|
/* Make sure the GDT Entry is valid */
|
|
if (TssEntry)
|
|
{
|
|
/* Set the Limit */
|
|
TssEntry->LimitLow = sizeof(KTSS) - 1;
|
|
TssEntry->HighWord.Bits.LimitHi = 0;
|
|
}
|
|
|
|
/* Now clear the I/O Map */
|
|
ASSERT(IOPM_COUNT == 1);
|
|
RtlFillMemory(Tss->IoMaps[0].IoMap, IOPM_FULL_SIZE, 0xFF);
|
|
|
|
/* Initialize Interrupt Direction Maps */
|
|
p = (PUCHAR)(Tss->IoMaps[0].DirectionMap);
|
|
RtlZeroMemory(p, IOPM_DIRECTION_MAP_SIZE);
|
|
|
|
/* Add DPMI support for interrupts */
|
|
p[0] = 4;
|
|
p[3] = 0x18;
|
|
p[4] = 0x18;
|
|
|
|
/* Initialize the default Interrupt Direction Map */
|
|
p = Tss->IntDirectionMap;
|
|
RtlZeroMemory(Tss->IntDirectionMap, IOPM_DIRECTION_MAP_SIZE);
|
|
|
|
/* Add DPMI support */
|
|
p[0] = 4;
|
|
p[3] = 0x18;
|
|
p[4] = 0x18;
|
|
}
|
|
|
|
VOID
|
|
NTAPI
|
|
KiInitializeTSS(IN PKTSS Tss)
|
|
{
|
|
/* Set an invalid map base */
|
|
Tss->IoMapBase = KiComputeIopmOffset(IO_ACCESS_MAP_NONE);
|
|
|
|
/* Disable traps during Task Switches */
|
|
Tss->Flags = 0;
|
|
|
|
/* Set LDT and Ring 0 SS */
|
|
Tss->LDT = 0;
|
|
Tss->Ss0 = KGDT_R0_DATA;
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
FASTCALL
|
|
Ki386InitializeTss(IN PKTSS Tss,
|
|
IN PKIDTENTRY Idt,
|
|
IN PKGDTENTRY Gdt)
|
|
{
|
|
PKGDTENTRY TssEntry, TaskGateEntry;
|
|
|
|
/* Initialize the boot TSS. */
|
|
TssEntry = &Gdt[KGDT_TSS / sizeof(KGDTENTRY)];
|
|
TssEntry->HighWord.Bits.Type = I386_TSS;
|
|
TssEntry->HighWord.Bits.Pres = 1;
|
|
TssEntry->HighWord.Bits.Dpl = 0;
|
|
KiInitializeTSS2(Tss, TssEntry);
|
|
KiInitializeTSS(Tss);
|
|
|
|
/* Load the task register */
|
|
Ke386SetTr(KGDT_TSS);
|
|
|
|
/* Setup the Task Gate for Double Fault Traps */
|
|
TaskGateEntry = (PKGDTENTRY)&Idt[8];
|
|
TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
|
|
TaskGateEntry->HighWord.Bits.Pres = 1;
|
|
TaskGateEntry->HighWord.Bits.Dpl = 0;
|
|
((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_DF_TSS;
|
|
|
|
/* Initialize the TSS used for handling double faults. */
|
|
Tss = (PKTSS)KiDoubleFaultTSS;
|
|
KiInitializeTSS(Tss);
|
|
Tss->CR3 = __readcr3();
|
|
Tss->Esp0 = KiDoubleFaultStack;
|
|
Tss->Esp = KiDoubleFaultStack;
|
|
Tss->Eip = PtrToUlong(KiTrap08);
|
|
Tss->Cs = KGDT_R0_CODE;
|
|
Tss->Fs = KGDT_R0_PCR;
|
|
Tss->Ss = Ke386GetSs();
|
|
Tss->Es = KGDT_R3_DATA | RPL_MASK;
|
|
Tss->Ds = KGDT_R3_DATA | RPL_MASK;
|
|
|
|
/* Setup the Double Trap TSS entry in the GDT */
|
|
TssEntry = &Gdt[KGDT_DF_TSS / sizeof(KGDTENTRY)];
|
|
TssEntry->HighWord.Bits.Type = I386_TSS;
|
|
TssEntry->HighWord.Bits.Pres = 1;
|
|
TssEntry->HighWord.Bits.Dpl = 0;
|
|
TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
|
|
TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
|
|
TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
|
|
TssEntry->LimitLow = KTSS_IO_MAPS;
|
|
|
|
/* Now setup the NMI Task Gate */
|
|
TaskGateEntry = (PKGDTENTRY)&Idt[2];
|
|
TaskGateEntry->HighWord.Bits.Type = I386_TASK_GATE;
|
|
TaskGateEntry->HighWord.Bits.Pres = 1;
|
|
TaskGateEntry->HighWord.Bits.Dpl = 0;
|
|
((PKIDTENTRY)TaskGateEntry)->Selector = KGDT_NMI_TSS;
|
|
|
|
/* Initialize the actual TSS */
|
|
Tss = (PKTSS)KiNMITSS;
|
|
KiInitializeTSS(Tss);
|
|
Tss->CR3 = __readcr3();
|
|
Tss->Esp0 = KiDoubleFaultStack;
|
|
Tss->Esp = KiDoubleFaultStack;
|
|
Tss->Eip = PtrToUlong(KiTrap02);
|
|
Tss->Cs = KGDT_R0_CODE;
|
|
Tss->Fs = KGDT_R0_PCR;
|
|
Tss->Ss = Ke386GetSs();
|
|
Tss->Es = KGDT_R3_DATA | RPL_MASK;
|
|
Tss->Ds = KGDT_R3_DATA | RPL_MASK;
|
|
|
|
/* And its associated TSS Entry */
|
|
TssEntry = &Gdt[KGDT_NMI_TSS / sizeof(KGDTENTRY)];
|
|
TssEntry->HighWord.Bits.Type = I386_TSS;
|
|
TssEntry->HighWord.Bits.Pres = 1;
|
|
TssEntry->HighWord.Bits.Dpl = 0;
|
|
TssEntry->BaseLow = (USHORT)((ULONG_PTR)Tss & 0xFFFF);
|
|
TssEntry->HighWord.Bytes.BaseMid = (UCHAR)((ULONG_PTR)Tss >> 16);
|
|
TssEntry->HighWord.Bytes.BaseHi = (UCHAR)((ULONG_PTR)Tss >> 24);
|
|
TssEntry->LimitLow = KTSS_IO_MAPS;
|
|
}
|
|
|
|
VOID
|
|
NTAPI
|
|
KeFlushCurrentTb(VOID)
|
|
{
|
|
|
|
#if !defined(_GLOBAL_PAGES_ARE_AWESOME_)
|
|
|
|
/* Flush the TLB by resetting CR3 */
|
|
__writecr3(__readcr3());
|
|
|
|
#else
|
|
|
|
/* Check if global pages are enabled */
|
|
if (KeFeatureBits & KF_GLOBAL_PAGE)
|
|
{
|
|
ULONG Cr4;
|
|
|
|
/* Disable PGE (Note: may not have been enabled yet) */
|
|
Cr4 = __readcr4();
|
|
__writecr4(Cr4 & ~CR4_PGE);
|
|
|
|
/* Flush everything */
|
|
__writecr3(__readcr3());
|
|
|
|
/* Re-enable PGE */
|
|
__writecr4(Cr4);
|
|
}
|
|
else
|
|
{
|
|
/* No global pages, resetting CR3 is enough */
|
|
__writecr3(__readcr3());
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
VOID
|
|
NTAPI
|
|
KiRestoreProcessorControlState(PKPROCESSOR_STATE ProcessorState)
|
|
{
|
|
PKGDTENTRY TssEntry;
|
|
|
|
//
|
|
// Restore the CR registers
|
|
//
|
|
__writecr0(ProcessorState->SpecialRegisters.Cr0);
|
|
Ke386SetCr2(ProcessorState->SpecialRegisters.Cr2);
|
|
__writecr3(ProcessorState->SpecialRegisters.Cr3);
|
|
if (KeFeatureBits & KF_CR4) __writecr4(ProcessorState->SpecialRegisters.Cr4);
|
|
|
|
//
|
|
// Restore the DR registers
|
|
//
|
|
__writedr(0, ProcessorState->SpecialRegisters.KernelDr0);
|
|
__writedr(1, ProcessorState->SpecialRegisters.KernelDr1);
|
|
__writedr(2, ProcessorState->SpecialRegisters.KernelDr2);
|
|
__writedr(3, ProcessorState->SpecialRegisters.KernelDr3);
|
|
__writedr(6, ProcessorState->SpecialRegisters.KernelDr6);
|
|
__writedr(7, ProcessorState->SpecialRegisters.KernelDr7);
|
|
|
|
//
|
|
// Restore GDT and IDT
|
|
//
|
|
Ke386SetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
|
|
__lidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
|
|
|
|
//
|
|
// Clear the busy flag so we don't crash if we reload the same selector
|
|
//
|
|
TssEntry = (PKGDTENTRY)(ProcessorState->SpecialRegisters.Gdtr.Base +
|
|
ProcessorState->SpecialRegisters.Tr);
|
|
TssEntry->HighWord.Bytes.Flags1 &= ~0x2;
|
|
|
|
//
|
|
// Restore TSS and LDT
|
|
//
|
|
Ke386SetTr(ProcessorState->SpecialRegisters.Tr);
|
|
Ke386SetLocalDescriptorTable(ProcessorState->SpecialRegisters.Ldtr);
|
|
}
|
|
|
|
VOID
|
|
NTAPI
|
|
KiSaveProcessorControlState(OUT PKPROCESSOR_STATE ProcessorState)
|
|
{
|
|
/* Save the CR registers */
|
|
ProcessorState->SpecialRegisters.Cr0 = __readcr0();
|
|
ProcessorState->SpecialRegisters.Cr2 = __readcr2();
|
|
ProcessorState->SpecialRegisters.Cr3 = __readcr3();
|
|
ProcessorState->SpecialRegisters.Cr4 = (KeFeatureBits & KF_CR4) ?
|
|
__readcr4() : 0;
|
|
|
|
/* Save the DR registers */
|
|
ProcessorState->SpecialRegisters.KernelDr0 = __readdr(0);
|
|
ProcessorState->SpecialRegisters.KernelDr1 = __readdr(1);
|
|
ProcessorState->SpecialRegisters.KernelDr2 = __readdr(2);
|
|
ProcessorState->SpecialRegisters.KernelDr3 = __readdr(3);
|
|
ProcessorState->SpecialRegisters.KernelDr6 = __readdr(6);
|
|
ProcessorState->SpecialRegisters.KernelDr7 = __readdr(7);
|
|
__writedr(7, 0);
|
|
|
|
/* Save GDT, IDT, LDT and TSS */
|
|
Ke386GetGlobalDescriptorTable(&ProcessorState->SpecialRegisters.Gdtr.Limit);
|
|
__sidt(&ProcessorState->SpecialRegisters.Idtr.Limit);
|
|
ProcessorState->SpecialRegisters.Tr = Ke386GetTr();
|
|
Ke386GetLocalDescriptorTable(&ProcessorState->SpecialRegisters.Ldtr);
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
NTAPI
|
|
KiInitializeMachineType(VOID)
|
|
{
|
|
/* Set the Machine Type we got from NTLDR */
|
|
KeI386MachineType = KeLoaderBlock->u.I386.MachineType & 0x000FF;
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
ULONG_PTR
|
|
NTAPI
|
|
KiLoadFastSyscallMachineSpecificRegisters(IN ULONG_PTR Context)
|
|
{
|
|
/* Set CS and ESP */
|
|
__writemsr(0x174, KGDT_R0_CODE);
|
|
__writemsr(0x175, (ULONG_PTR)KeGetCurrentPrcb()->DpcStack);
|
|
|
|
/* Set LSTAR */
|
|
__writemsr(0x176, (ULONG_PTR)KiFastCallEntry);
|
|
return 0;
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
NTAPI
|
|
KiRestoreFastSyscallReturnState(VOID)
|
|
{
|
|
/* Check if the CPU Supports fast system call */
|
|
if (KeFeatureBits & KF_FAST_SYSCALL)
|
|
{
|
|
/* Check if it has been disabled */
|
|
if (KiFastSystemCallDisable)
|
|
{
|
|
/* Disable fast system call */
|
|
KeFeatureBits &= ~KF_FAST_SYSCALL;
|
|
KiFastCallExitHandler = KiSystemCallTrapReturn;
|
|
DPRINT1("Support for SYSENTER disabled.\n");
|
|
}
|
|
else
|
|
{
|
|
/* Do an IPI to enable it */
|
|
KeIpiGenericCall(KiLoadFastSyscallMachineSpecificRegisters, 0);
|
|
|
|
/* It's enabled, so use the proper exit stub */
|
|
KiFastCallExitHandler = KiSystemCallSysExitReturn;
|
|
DPRINT("Support for SYSENTER detected.\n");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
/* Use the IRET handler */
|
|
KiFastCallExitHandler = KiSystemCallTrapReturn;
|
|
DPRINT1("No support for SYSENTER detected.\n");
|
|
}
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
ULONG_PTR
|
|
NTAPI
|
|
Ki386EnableDE(IN ULONG_PTR Context)
|
|
{
|
|
/* Enable DE */
|
|
__writecr4(__readcr4() | CR4_DE);
|
|
return 0;
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
ULONG_PTR
|
|
NTAPI
|
|
Ki386EnableFxsr(IN ULONG_PTR Context)
|
|
{
|
|
/* Enable FXSR */
|
|
__writecr4(__readcr4() | CR4_FXSR);
|
|
return 0;
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
ULONG_PTR
|
|
NTAPI
|
|
Ki386EnableXMMIExceptions(IN ULONG_PTR Context)
|
|
{
|
|
PKIDTENTRY IdtEntry;
|
|
|
|
/* Get the IDT Entry for Interrupt 0x13 */
|
|
IdtEntry = &((PKIPCR)KeGetPcr())->IDT[0x13];
|
|
|
|
/* Set it up */
|
|
IdtEntry->Selector = KGDT_R0_CODE;
|
|
IdtEntry->Offset = ((ULONG_PTR)KiTrap13 & 0xFFFF);
|
|
IdtEntry->ExtendedOffset = ((ULONG_PTR)KiTrap13 >> 16) & 0xFFFF;
|
|
((PKIDT_ACCESS)&IdtEntry->Access)->Dpl = 0;
|
|
((PKIDT_ACCESS)&IdtEntry->Access)->Present = 1;
|
|
((PKIDT_ACCESS)&IdtEntry->Access)->SegmentType = I386_INTERRUPT_GATE;
|
|
|
|
/* Enable XMMI exceptions */
|
|
__writecr4(__readcr4() | CR4_XMMEXCPT);
|
|
return 0;
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
VOID
|
|
NTAPI
|
|
KiI386PentiumLockErrataFixup(VOID)
|
|
{
|
|
KDESCRIPTOR IdtDescriptor = {0, 0, 0};
|
|
PKIDTENTRY NewIdt, NewIdt2;
|
|
PMMPTE PointerPte;
|
|
|
|
/* Allocate memory for a new IDT */
|
|
NewIdt = ExAllocatePool(NonPagedPool, 2 * PAGE_SIZE);
|
|
|
|
/* Put everything after the first 7 entries on a new page */
|
|
NewIdt2 = (PVOID)((ULONG_PTR)NewIdt + PAGE_SIZE - (7 * sizeof(KIDTENTRY)));
|
|
|
|
/* Disable interrupts */
|
|
_disable();
|
|
|
|
/* Get the current IDT and copy it */
|
|
__sidt(&IdtDescriptor.Limit);
|
|
RtlCopyMemory(NewIdt2,
|
|
(PVOID)IdtDescriptor.Base,
|
|
IdtDescriptor.Limit + 1);
|
|
IdtDescriptor.Base = (ULONG)NewIdt2;
|
|
|
|
/* Set the new IDT */
|
|
__lidt(&IdtDescriptor.Limit);
|
|
((PKIPCR)KeGetPcr())->IDT = NewIdt2;
|
|
|
|
/* Restore interrupts */
|
|
_enable();
|
|
|
|
/* Set the first 7 entries as read-only to produce a fault */
|
|
PointerPte = MiAddressToPte(NewIdt);
|
|
ASSERT(PointerPte->u.Hard.Write == 1);
|
|
PointerPte->u.Hard.Write = 0;
|
|
KeInvalidateTlbEntry(NewIdt);
|
|
}
|
|
|
|
BOOLEAN
|
|
NTAPI
|
|
KeInvalidateAllCaches(VOID)
|
|
{
|
|
/* Only supported on Pentium Pro and higher */
|
|
if (KeI386CpuType < 6) return FALSE;
|
|
|
|
/* Invalidate all caches */
|
|
__wbinvd();
|
|
return TRUE;
|
|
}
|
|
|
|
VOID
|
|
NTAPI
|
|
KiSaveProcessorState(IN PKTRAP_FRAME TrapFrame,
|
|
IN PKEXCEPTION_FRAME ExceptionFrame)
|
|
{
|
|
PKPRCB Prcb = KeGetCurrentPrcb();
|
|
|
|
//
|
|
// Save full context
|
|
//
|
|
Prcb->ProcessorState.ContextFrame.ContextFlags = CONTEXT_FULL |
|
|
CONTEXT_DEBUG_REGISTERS;
|
|
KeTrapFrameToContext(TrapFrame, NULL, &Prcb->ProcessorState.ContextFrame);
|
|
|
|
//
|
|
// Save control registers
|
|
//
|
|
KiSaveProcessorControlState(&Prcb->ProcessorState);
|
|
}
|
|
|
|
CODE_SEG("INIT")
|
|
BOOLEAN
|
|
NTAPI
|
|
KiIsNpxErrataPresent(VOID)
|
|
{
|
|
static double Value1 = 4195835.0, Value2 = 3145727.0;
|
|
INT ErrataPresent;
|
|
ULONG Cr0;
|
|
|
|
/* Interrupts have to be disabled here. */
|
|
ASSERT(!(__readeflags() & EFLAGS_INTERRUPT_MASK));
|
|
|
|
/* Read CR0 and remove FPU flags */
|
|
Cr0 = __readcr0();
|
|
__writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
|
|
|
|
/* Initialize FPU state */
|
|
Ke386FnInit();
|
|
|
|
/* Multiply the magic values and divide, we should get the result back */
|
|
#ifdef __GNUC__
|
|
__asm__ __volatile__
|
|
(
|
|
"fldl %1\n\t"
|
|
"fdivl %2\n\t"
|
|
"fmull %2\n\t"
|
|
"fldl %1\n\t"
|
|
"fsubp\n\t"
|
|
"fistpl %0\n\t"
|
|
: "=m" (ErrataPresent)
|
|
: "m" (Value1),
|
|
"m" (Value2)
|
|
);
|
|
#else
|
|
__asm
|
|
{
|
|
fld Value1
|
|
fdiv Value2
|
|
fmul Value2
|
|
fld Value1
|
|
fsubp st(1), st(0)
|
|
fistp ErrataPresent
|
|
};
|
|
#endif
|
|
|
|
/* Restore CR0 */
|
|
__writecr0(Cr0);
|
|
|
|
/* Return if there's an errata */
|
|
return ErrataPresent != 0;
|
|
}
|
|
|
|
VOID
|
|
NTAPI
|
|
KiFlushNPXState(IN PFLOATING_SAVE_AREA SaveArea)
|
|
{
|
|
ULONG EFlags, Cr0;
|
|
PKTHREAD Thread, NpxThread;
|
|
PFX_SAVE_AREA FxSaveArea;
|
|
|
|
/* Save volatiles and disable interrupts */
|
|
EFlags = __readeflags();
|
|
_disable();
|
|
|
|
/* Save the PCR and get the current thread */
|
|
Thread = KeGetCurrentThread();
|
|
|
|
/* Check if we're already loaded */
|
|
if (Thread->NpxState != NPX_STATE_LOADED)
|
|
{
|
|
/* If there's nothing to load, quit */
|
|
if (!SaveArea)
|
|
{
|
|
/* Restore interrupt state and return */
|
|
__writeeflags(EFlags);
|
|
return;
|
|
}
|
|
|
|
/* Need FXSR support for this */
|
|
ASSERT(KeI386FxsrPresent == TRUE);
|
|
|
|
/* Check for sane CR0 */
|
|
Cr0 = __readcr0();
|
|
if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
|
|
{
|
|
/* Mask out FPU flags */
|
|
__writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
|
|
}
|
|
|
|
/* Get the NPX thread and check its FPU state */
|
|
NpxThread = KeGetCurrentPrcb()->NpxThread;
|
|
if ((NpxThread) && (NpxThread->NpxState == NPX_STATE_LOADED))
|
|
{
|
|
/* Get the FX frame and store the state there */
|
|
FxSaveArea = KiGetThreadNpxArea(NpxThread);
|
|
Ke386FxSave(FxSaveArea);
|
|
|
|
/* NPX thread has lost its state */
|
|
NpxThread->NpxState = NPX_STATE_NOT_LOADED;
|
|
}
|
|
|
|
/* Now load NPX state from the NPX area */
|
|
FxSaveArea = KiGetThreadNpxArea(Thread);
|
|
Ke386FxStore(FxSaveArea);
|
|
}
|
|
else
|
|
{
|
|
/* Check for sane CR0 */
|
|
Cr0 = __readcr0();
|
|
if (Cr0 & (CR0_MP | CR0_TS | CR0_EM))
|
|
{
|
|
/* Mask out FPU flags */
|
|
__writecr0(Cr0 & ~(CR0_MP | CR0_TS | CR0_EM));
|
|
}
|
|
|
|
/* Get FX frame */
|
|
FxSaveArea = KiGetThreadNpxArea(Thread);
|
|
Thread->NpxState = NPX_STATE_NOT_LOADED;
|
|
|
|
/* Save state if supported by CPU */
|
|
if (KeI386FxsrPresent) Ke386FxSave(FxSaveArea);
|
|
}
|
|
|
|
/* Now save the FN state wherever it was requested */
|
|
if (SaveArea) Ke386FnSave(SaveArea);
|
|
|
|
/* Clear NPX thread */
|
|
KeGetCurrentPrcb()->NpxThread = NULL;
|
|
|
|
/* Add the CR0 from the NPX frame */
|
|
Cr0 |= NPX_STATE_NOT_LOADED;
|
|
Cr0 |= FxSaveArea->Cr0NpxState;
|
|
__writecr0(Cr0);
|
|
|
|
/* Restore interrupt state */
|
|
__writeeflags(EFlags);
|
|
}
|
|
|
|
/* PUBLIC FUNCTIONS **********************************************************/
|
|
|
|
/*
|
|
* @implemented
|
|
*/
|
|
VOID
|
|
NTAPI
|
|
KiCoprocessorError(VOID)
|
|
{
|
|
PFX_SAVE_AREA NpxArea;
|
|
|
|
/* Get the FPU area */
|
|
NpxArea = KiGetThreadNpxArea(KeGetCurrentThread());
|
|
|
|
/* Set CR0_TS */
|
|
NpxArea->Cr0NpxState = CR0_TS;
|
|
__writecr0(__readcr0() | CR0_TS);
|
|
}
|
|
|
|
/**
|
|
* @brief
|
|
* Saves the current floating point unit state
|
|
* context of the current calling thread.
|
|
*
|
|
* @param[out] Save
|
|
* The saved floating point context given to the
|
|
* caller at the end of function's operations.
|
|
* The structure whose data contents are opaque
|
|
* to the calling thread.
|
|
*
|
|
* @return
|
|
* Returns STATUS_SUCCESS if the function has
|
|
* successfully completed its operations.
|
|
* STATUS_INSUFFICIENT_RESOURCES is returned
|
|
* if the function couldn't allocate memory
|
|
* for FPU state information.
|
|
*
|
|
* @remarks
|
|
* The function performs a FPU state save
|
|
* in two ways. A normal FPU save (FNSAVE)
|
|
* is performed if the system doesn't have
|
|
* SSE/SSE2, otherwise the function performs
|
|
* a save of FPU, MMX and SSE states save (FXSAVE).
|
|
*/
|
|
#if defined(__clang__)
|
|
__attribute__((__target__("sse")))
|
|
#endif
|
|
NTSTATUS
|
|
NTAPI
|
|
KeSaveFloatingPointState(
|
|
_Out_ PKFLOATING_SAVE Save)
|
|
{
|
|
PFLOATING_SAVE_CONTEXT FsContext;
|
|
PFX_SAVE_AREA FxSaveAreaFrame;
|
|
PKPRCB CurrentPrcb;
|
|
|
|
/* Sanity checks */
|
|
ASSERT(Save);
|
|
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
|
|
ASSERT(KeI386NpxPresent);
|
|
|
|
/* Initialize the floating point context */
|
|
FsContext = ExAllocatePoolWithTag(NonPagedPool,
|
|
sizeof(FLOATING_SAVE_CONTEXT),
|
|
TAG_FLOATING_POINT_CONTEXT);
|
|
if (!FsContext)
|
|
{
|
|
/* Bail out if we failed */
|
|
return STATUS_INSUFFICIENT_RESOURCES;
|
|
}
|
|
|
|
/*
|
|
* Allocate some memory pool for the buffer. The size
|
|
* of this allocated buffer is the FX area plus the
|
|
* alignment requirement needed for FXSAVE as a 16-byte
|
|
* aligned pointer is compulsory in order to save the
|
|
* FPU state.
|
|
*/
|
|
FsContext->Buffer = ExAllocatePoolWithTag(NonPagedPool,
|
|
sizeof(FX_SAVE_AREA) + FXSAVE_ALIGN,
|
|
TAG_FLOATING_POINT_FX);
|
|
if (!FsContext->Buffer)
|
|
{
|
|
/* Bail out if we failed */
|
|
ExFreePoolWithTag(FsContext, TAG_FLOATING_POINT_CONTEXT);
|
|
return STATUS_INSUFFICIENT_RESOURCES;
|
|
}
|
|
|
|
/*
|
|
* Now cache the allocated buffer into the save area
|
|
* and align the said area to a 16-byte boundary. Why
|
|
* do we have to do this is because of ExAllocate function.
|
|
* We gave the necessary alignment requirement in the pool
|
|
* allocation size although the function will always return
|
|
* a 8-byte aligned pointer. Aligning the given pointer directly
|
|
* can cause issues when freeing it from memory afterwards. With
|
|
* that said, we have to cache the buffer to the area so that we
|
|
* do not touch or mess the allocated buffer any further.
|
|
*/
|
|
FsContext->PfxSaveArea = ALIGN_UP_POINTER_BY(FsContext->Buffer, 16);
|
|
|
|
/* Disable interrupts and get the current processor control region */
|
|
_disable();
|
|
CurrentPrcb = KeGetCurrentPrcb();
|
|
|
|
/* Store the current thread to context */
|
|
FsContext->CurrentThread = KeGetCurrentThread();
|
|
|
|
/*
|
|
* Save the previous NPX thread state registers (aka Numeric
|
|
* Processor eXtension) into the current context so that
|
|
* we are informing the scheduler the current FPU state
|
|
* belongs to this thread.
|
|
*/
|
|
if (FsContext->CurrentThread != CurrentPrcb->NpxThread)
|
|
{
|
|
if ((CurrentPrcb->NpxThread != NULL) &&
|
|
(CurrentPrcb->NpxThread->NpxState == NPX_STATE_LOADED))
|
|
{
|
|
/* Get the FX frame */
|
|
FxSaveAreaFrame = KiGetThreadNpxArea(CurrentPrcb->NpxThread);
|
|
|
|
/* Save the FPU state */
|
|
Ke386SaveFpuState(FxSaveAreaFrame);
|
|
|
|
/* NPX thread has lost its state */
|
|
CurrentPrcb->NpxThread->NpxState = NPX_STATE_NOT_LOADED;
|
|
FxSaveAreaFrame->NpxSavedCpu = 0;
|
|
}
|
|
|
|
/* The new NPX thread is the current thread */
|
|
CurrentPrcb->NpxThread = FsContext->CurrentThread;
|
|
}
|
|
|
|
/* Perform the save */
|
|
Ke386SaveFpuState(FsContext->PfxSaveArea);
|
|
|
|
/* Store the NPX IRQL */
|
|
FsContext->OldNpxIrql = FsContext->CurrentThread->Header.NpxIrql;
|
|
|
|
/* Set the current IRQL to NPX */
|
|
FsContext->CurrentThread->Header.NpxIrql = KeGetCurrentIrql();
|
|
|
|
/* Initialize the FPU */
|
|
Ke386FnInit();
|
|
|
|
/* Enable interrupts back */
|
|
_enable();
|
|
|
|
/* Give the saved FPU context to the caller */
|
|
*((PVOID *) Save) = FsContext;
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
/**
|
|
* @brief
|
|
* Restores the original FPU state context that has
|
|
* been saved by a API call of KeSaveFloatingPointState.
|
|
* Callers are expected to restore the floating point
|
|
* state by calling this function when they've finished
|
|
* doing FPU operations.
|
|
*
|
|
* @param[in] Save
|
|
* The saved floating point context that is to be given
|
|
* to the function to restore the FPU state.
|
|
*
|
|
* @return
|
|
* Returns STATUS_SUCCESS indicating the function
|
|
* has fully completed its operations.
|
|
*/
|
|
#if defined(__clang__)
|
|
__attribute__((__target__("sse")))
|
|
#endif
|
|
NTSTATUS
|
|
NTAPI
|
|
KeRestoreFloatingPointState(
|
|
_In_ PKFLOATING_SAVE Save)
|
|
{
|
|
PFLOATING_SAVE_CONTEXT FsContext;
|
|
|
|
/* Sanity checks */
|
|
ASSERT(Save);
|
|
ASSERT(KeGetCurrentIrql() <= DISPATCH_LEVEL);
|
|
ASSERT(KeI386NpxPresent);
|
|
|
|
/* Cache the saved FS context */
|
|
FsContext = *((PVOID *) Save);
|
|
|
|
/*
|
|
* We have to restore the regular saved FPU
|
|
* state. For this we must first do some
|
|
* validation checks so that we are sure
|
|
* ourselves the state context is saved
|
|
* properly. Check if we are in the same
|
|
* calling thread.
|
|
*/
|
|
if (FsContext->CurrentThread != KeGetCurrentThread())
|
|
{
|
|
/*
|
|
* This isn't the thread that saved the
|
|
* FPU state context, crash the system!
|
|
*/
|
|
KeBugCheckEx(INVALID_FLOATING_POINT_STATE,
|
|
0x2,
|
|
(ULONG_PTR)FsContext->CurrentThread,
|
|
(ULONG_PTR)KeGetCurrentThread(),
|
|
0);
|
|
}
|
|
|
|
/* Are we under the same NPX interrupt level? */
|
|
if (FsContext->CurrentThread->Header.NpxIrql != KeGetCurrentIrql())
|
|
{
|
|
/* The interrupt level has changed, crash the system! */
|
|
KeBugCheckEx(INVALID_FLOATING_POINT_STATE,
|
|
0x1,
|
|
(ULONG_PTR)FsContext->CurrentThread->Header.NpxIrql,
|
|
(ULONG_PTR)KeGetCurrentIrql(),
|
|
0);
|
|
}
|
|
|
|
/* Disable interrupts */
|
|
_disable();
|
|
|
|
/*
|
|
* The saved FPU state context is valid,
|
|
* it's time to restore the state. First,
|
|
* clear FPU exceptions now.
|
|
*/
|
|
Ke386ClearFpExceptions();
|
|
|
|
/* Restore the state */
|
|
Ke386RestoreFpuState(FsContext->PfxSaveArea);
|
|
|
|
/* Give the saved NPX IRQL back to the NPX thread */
|
|
FsContext->CurrentThread->Header.NpxIrql = FsContext->OldNpxIrql;
|
|
|
|
/* Enable interrupts back */
|
|
_enable();
|
|
|
|
/* We're done, free the allocated area and context */
|
|
ExFreePoolWithTag(FsContext->Buffer, TAG_FLOATING_POINT_FX);
|
|
ExFreePoolWithTag(FsContext, TAG_FLOATING_POINT_CONTEXT);
|
|
|
|
return STATUS_SUCCESS;
|
|
}
|
|
|
|
/*
|
|
* @implemented
|
|
*/
|
|
ULONG
|
|
NTAPI
|
|
KeGetRecommendedSharedDataAlignment(VOID)
|
|
{
|
|
/* Return the global variable */
|
|
return KeLargestCacheLine;
|
|
}
|
|
|
|
VOID
|
|
NTAPI
|
|
KiFlushTargetEntireTb(IN PKIPI_CONTEXT PacketContext,
|
|
IN PVOID Ignored1,
|
|
IN PVOID Ignored2,
|
|
IN PVOID Ignored3)
|
|
{
|
|
/* Signal this packet as done */
|
|
KiIpiSignalPacketDone(PacketContext);
|
|
|
|
/* Flush the TB for the Current CPU */
|
|
KeFlushCurrentTb();
|
|
}
|
|
|
|
/*
|
|
* @implemented
|
|
*/
|
|
VOID
|
|
NTAPI
|
|
KeFlushEntireTb(IN BOOLEAN Invalid,
|
|
IN BOOLEAN AllProcessors)
|
|
{
|
|
KIRQL OldIrql;
|
|
#ifdef CONFIG_SMP
|
|
KAFFINITY TargetAffinity;
|
|
PKPRCB Prcb = KeGetCurrentPrcb();
|
|
#endif
|
|
|
|
/* Raise the IRQL for the TB Flush */
|
|
OldIrql = KeRaiseIrqlToSynchLevel();
|
|
|
|
#ifdef CONFIG_SMP
|
|
/* FIXME: Use KiTbFlushTimeStamp to synchronize TB flush */
|
|
|
|
/* Get the current processor affinity, and exclude ourselves */
|
|
TargetAffinity = KeActiveProcessors;
|
|
TargetAffinity &= ~Prcb->SetMember;
|
|
|
|
/* Make sure this is MP */
|
|
if (TargetAffinity)
|
|
{
|
|
/* Send an IPI TB flush to the other processors */
|
|
KiIpiSendPacket(TargetAffinity,
|
|
KiFlushTargetEntireTb,
|
|
NULL,
|
|
0,
|
|
NULL);
|
|
}
|
|
#endif
|
|
|
|
/* Flush the TB for the Current CPU, and update the flush stamp */
|
|
KeFlushCurrentTb();
|
|
|
|
#ifdef CONFIG_SMP
|
|
/* If this is MP, wait for the other processors to finish */
|
|
if (TargetAffinity)
|
|
{
|
|
/* Sanity check */
|
|
ASSERT(Prcb == KeGetCurrentPrcb());
|
|
|
|
/* FIXME: TODO */
|
|
ASSERTMSG("Not yet implemented\n", FALSE);
|
|
}
|
|
#endif
|
|
|
|
/* Update the flush stamp and return to original IRQL */
|
|
InterlockedExchangeAdd(&KiTbFlushTimeStamp, 1);
|
|
KeLowerIrql(OldIrql);
|
|
}
|
|
|
|
/*
|
|
* @implemented
|
|
*/
|
|
VOID
|
|
NTAPI
|
|
KeSetDmaIoCoherency(IN ULONG Coherency)
|
|
{
|
|
/* Save the coherency globally */
|
|
KiDmaIoCoherency = Coherency;
|
|
}
|
|
|
|
/*
|
|
* @implemented
|
|
*/
|
|
KAFFINITY
|
|
NTAPI
|
|
KeQueryActiveProcessors(VOID)
|
|
{
|
|
PAGED_CODE();
|
|
|
|
/* Simply return the number of active processors */
|
|
return KeActiveProcessors;
|
|
}
|
|
|
|
/*
|
|
* @implemented
|
|
*/
|
|
VOID
|
|
__cdecl
|
|
KeSaveStateForHibernate(IN PKPROCESSOR_STATE State)
|
|
{
|
|
/* Capture the context */
|
|
RtlCaptureContext(&State->ContextFrame);
|
|
|
|
/* Capture the control state */
|
|
KiSaveProcessorControlState(State);
|
|
}
|