Trap Handlers in C Patch 9 of 12:

[NTOS]: Fix a logic bug in KiExitTrap: Always only restore segments if we came from user-mode (since they might be bogus on a kernel transition as they're not always saved), even if the caller wants segment restore.
    [NTOS]: Small perf boot: do a JMP, not a CALL into C handling code.
    [NTOS]: Make KiGetTickCount/KiCallbackReturn handled in C (as stubs).
    [NTOS]: Implement KeSynchronizeExecution in C. Move Kei386SpinOnSpinLock to C stub.
    [NTOS]: Implement overall architecture for handling hardware interrupts in C. Not used yet, since it needs C code in HAL.

svn path=/trunk/; revision=45045

reactos/ntoskrnl/include/internal/i386/asmmacro.S

@@ -238,7 +238,7 @@ _&Name:
     pushad
     sub esp, KTRAP_FRAME_LENGTH - KTRAP_FRAME_PREVIOUS_MODE
     mov ecx, esp
-    call @&Name&Handler@4
+    jmp @&Name&Handler@4
 .endfunc
 .endm
 

reactos/ntoskrnl/include/internal/ke.h

@@ -938,6 +938,7 @@ KiEndUnexpectedRange(
     VOID
 );
 
+#ifndef HAL_INTERRUPT_SUPPORT_IN_C
 VOID
 NTAPI
 KiInterruptDispatch(
@@ -949,6 +950,21 @@ NTAPI
 KiChainedDispatch(
     VOID
 );
+#else
+VOID
+FASTCALL
+KiInterruptDispatch(
+    IN PKTRAP_FRAME TrapFrame,
+    IN PKINTERRUPT Interrupt
+);
+
+VOID
+FASTCALL
+KiChainedDispatch(
+    IN PKTRAP_FRAME TrapFrame,
+    IN PKINTERRUPT Interrupt
+);
+#endif
 
 VOID
 NTAPI

reactos/ntoskrnl/ke/i386/irqobj.c

@@ -48,9 +48,9 @@ KiGetVectorDispatch(IN ULONG Vector,
                                    KiUnexpectedEntrySize);
 
     /* Setup the handlers */
-    Dispatch->InterruptDispatch = KiInterruptDispatch;
+    Dispatch->InterruptDispatch = (PVOID)KiInterruptDispatch;
     Dispatch->FloatingDispatch = NULL; // Floating Interrupts are not supported
-    Dispatch->ChainedDispatch = KiChainedDispatch;
+    Dispatch->ChainedDispatch = (PVOID)KiChainedDispatch;
     Dispatch->FlatDispatch = NULL;
 
     /* Get the current handler */
@@ -97,7 +97,9 @@ KiConnectVectorToInterrupt(IN PKINTERRUPT Interrupt,
 {
     DISPATCH_INFO Dispatch;
     PKINTERRUPT_ROUTINE Handler;
+#ifndef HAL_INTERRUPT_SUPPORT_IN_C
     PULONG Patch = &Interrupt->DispatchCode[0];
+#endif
 
     /* Get vector data */
     KiGetVectorDispatch(Interrupt->Vector, &Dispatch);
@@ -119,6 +121,8 @@ KiConnectVectorToInterrupt(IN PKINTERRUPT Interrupt,
         /* Set the handler */
         Interrupt->DispatchAddress = Handler;
 
+        /* Read note in trap.s -- patching not needed since JMP is static */
+#ifndef HAL_INTERRUPT_SUPPORT_IN_C
         /* Jump to the last 4 bytes */
         Patch = (PULONG)((ULONG_PTR)Patch +
                          ((ULONG_PTR)&KiInterruptTemplateDispatch -
@@ -126,6 +130,7 @@ KiConnectVectorToInterrupt(IN PKINTERRUPT Interrupt,
 
         /* Apply the patch */
         *Patch = (ULONG)((ULONG_PTR)Handler - ((ULONG_PTR)Patch + 4));
+#endif
 
         /* Now set the final handler address */
         ASSERT(Dispatch.FlatDispatch == NULL);
@@ -195,8 +200,10 @@ KeInitializeInterrupt(IN PKINTERRUPT Interrupt,
     }
 
     /* Sanity check */
+#ifndef HAL_INTERRUPT_SUPPORT_IN_C
     ASSERT((ULONG_PTR)&KiChainedDispatch2ndLvl -
            (ULONG_PTR)KiInterruptTemplate <= (KINTERRUPT_DISPATCH_CODES * 4));
+#endif
 
     /* Jump to the last 4 bytes */
     Patch = (PULONG)((ULONG_PTR)Patch +
@@ -390,4 +397,35 @@ KeDisconnectInterrupt(IN PKINTERRUPT Interrupt)
     return State;
 }
 
+/*
+ * @implemented
+ */
+BOOLEAN
+NTAPI
+KeSynchronizeExecution(IN OUT PKINTERRUPT Interrupt,
+                       IN PKSYNCHRONIZE_ROUTINE SynchronizeRoutine,
+                       IN PVOID SynchronizeContext OPTIONAL)
+{
+    NTSTATUS Status;
+    KIRQL OldIrql;
+    
+    /* Raise IRQL */
+    OldIrql = KfRaiseIrql(Interrupt->SynchronizeIrql);
+    
+    /* Acquire interrupt spinlock */
+    KeAcquireSpinLockAtDpcLevel(Interrupt->ActualLock);
+    
+    /* Call the routine */
+    Status = SynchronizeRoutine(SynchronizeContext);
+    
+    /* Release lock */
+    KeReleaseSpinLockFromDpcLevel(Interrupt->ActualLock);
+    
+    /* Lower IRQL */
+    KfLowerIrql(OldIrql);
+    
+    /* Return status */
+    return Status;
+}
+
 /* EOF */

reactos/ntoskrnl/ke/i386/trap.s

@@ -54,7 +54,6 @@ idt _KiTrap0F,         INT_32_DPL0  /* INT 2F: RESERVED                     */
 GENERATE_IDT_STUBS                  /* INT 30-FF: UNEXPECTED INTERRUPTS     */
 
 /* Trap handlers referenced from C code                                     */
-.globl _KiTrap2
 .globl _KiTrap8
 .globl _KiTrap19
 
@@ -69,7 +68,6 @@ GENERATE_IDT_STUBS                  /* INT 30-FF: UNEXPECTED INTERRUPTS     */
 /* And special system-defined software traps:                               */
 .globl _NtRaiseException@12
 .globl _NtContinue@8
-.globl _KiCoprocessorError@0
 .globl _KiDispatchInterrupt@0
 
 /* Interrupt template entrypoints                                           */
@@ -77,10 +75,12 @@ GENERATE_IDT_STUBS                  /* INT 30-FF: UNEXPECTED INTERRUPTS     */
 .globl _KiInterruptTemplateObject
 .globl _KiInterruptTemplateDispatch
 
+#ifndef HAL_INTERRUPT_SUPPORT_IN_C
 /* Chained and Normal generic interrupt handlers for 1st and 2nd level entry*/
 .globl _KiChainedDispatch2ndLvl@0
 .globl _KiInterruptDispatch@0
 .globl _KiChainedDispatch@0
+#endif
 
 /* We implement the following trap exit points:                             */
 .globl _KiServiceExit               /* Exit from syscall                    */
@@ -143,10 +143,6 @@ _KiTrapIoTable:
 /* SOFTWARE INTERRUPT SERVICES ***********************************************/
 .text
 
-_KiGetTickCount:
-_KiCallbackReturn:
-    /* FIXME: TODO */
-    UNHANDLED_PATH "TickCount/Callback Interrupts\n"
 
 .func KiSystemService
 TRAP_FIXUPS kss_a, kss_t, DoNotFixupV86, DoNotFixupAbios
@@ -473,6 +469,8 @@ AbiosExit:
     /* FIXME: TODO */
     UNHANDLED_PATH "ABIOS Exit"
 
+GENERATE_TRAP_HANDLER KiGetTickCount, 1
+GENERATE_TRAP_HANDLER KiCallbackReturn, 1        
 GENERATE_TRAP_HANDLER KiRaiseAssertion, 1
 GENERATE_TRAP_HANDLER KiDebugService, 1
 
@@ -768,6 +766,52 @@ QuantumEnd:
     ret
 .endfunc
 
+/*
+ * This is how the new-style interrupt template will look like.
+ *
+ * We setup the stack for a trap frame in the KINTERRUPT DispatchCode itself and
+ * then mov the stack address in ECX, since the handlers are FASTCALL. We also
+ * need to know the address of the KINTERRUPT. To do this, we maintain the old
+ * dynamic patching technique (EDX instead of EDI, however) and let the C API
+ * up in KeInitializeInterrupt replace the 0 with the address. Since this is in
+ * EDX, it becomes the second parameter for our FASTCALL function.
+ *
+ * Finally, we jump directly to the C interrupt handler, which will choose the
+ * appropriate dispatcher (chained, single, flat, floating) that was setup. The
+ * dispatchers themselves are also C FASTCALL functions. This double-indirection
+ * maintains the NT model should anything depend on it.
+ *
+ * Note that since we always jump to the C handler which then jumps to the C
+ * dispatcher, the first JMP in the template object is NOT patched anymore since
+ * it's static. Also, keep in mind this code is dynamically copied into nonpaged
+ * pool! It runs off the KINTERRUPT directly, so you can't just JMP to the code
+ * since JMPs are relative, and the location of the JMP below is dynamic. So we
+ * use EDI to store the absolute offset, and jump to that instead.
+ *
+ */
+#ifdef HAL_INTERRUPT_SUPPORT_IN_C
+.func KiInterruptTemplate
+_KiInterruptTemplate:
+    push 0
+    pushad
+    sub esp, KTRAP_FRAME_LENGTH - KTRAP_FRAME_PREVIOUS_MODE
+    mov ecx, esp
+
+_KiInterruptTemplate2ndDispatch:
+    /* Dummy code, will be replaced by the address of the KINTERRUPT */
+    mov edx, 0
+
+_KiInterruptTemplateObject:
+    /* Jump to C code */
+    mov edi, offset @KiInterruptHandler@8
+    jmp edi
+
+_KiInterruptTemplateDispatch:
+    /* Marks the end of the template so that the jump above can be edited */
+.endfunc
+
+#else
+
 .func KiInterruptTemplate
 _KiInterruptTemplate:
 
@@ -982,74 +1026,4 @@ IsrTimeout:
     /* Cleanup verification */
     VERIFY_INT_END kid, 0
 .endfunc
-
-.globl _KeSynchronizeExecution@12
-.func KeSynchronizeExecution@12
-_KeSynchronizeExecution@12:
-
-    /* Save EBX and put the interrupt object in it */
-    push ebx
-    mov ebx, [esp+8]
-
-    /* Go to DIRQL */
-    mov cl, [ebx+KINTERRUPT_SYNCHRONIZE_IRQL]
-    call @KfRaiseIrql@4
-    push eax
-
-#ifdef CONFIG_SMP
-    /* Acquire the interrupt spinlock FIXME: Write this in assembly */
-    mov ecx, [ebx+KINTERRUPT_ACTUAL_LOCK]
-    call @KefAcquireSpinLockAtDpcLevel@4
 #endif
-
-    /* Call the routine */
-    push [esp+20]
-    call [esp+20]
-
-#ifdef CONFIG_SMP
-    /* Release the interrupt spinlock FIXME: Write this in assembly */
-    push eax
-    mov ecx, [ebx+KINTERRUPT_ACTUAL_LOCK]
-    call @KefReleaseSpinLockFromDpcLevel@4
-    pop eax
-#endif
-
-    /* Lower IRQL */
-    mov ebx, eax
-    pop ecx
-    call @KfLowerIrql@4
-
-    /* Return status */
-    mov eax, ebx
-    pop ebx
-    ret 12
-.endfunc
-
-/*++
- * Kii386SpinOnSpinLock 
- *
- *     FILLMEIN
- *
- * Params:
- *     SpinLock - FILLMEIN
- *
- *     Flags - FILLMEIN
- *
- * Returns:
- *     None.
- *
- * Remarks:
- *     FILLMEIN
- *
- *--*/
-.globl _Kii386SpinOnSpinLock@8
-.func Kii386SpinOnSpinLock@8
-_Kii386SpinOnSpinLock@8:
-
-#ifdef CONFIG_SMP
-    /* FIXME: TODO */
-    int 3
-#endif
-
-    ret 8
-.endfunc

reactos/ntoskrnl/ke/i386/traphdlr.c

@@ -89,6 +89,9 @@ KiExitTrap(IN PKTRAP_FRAME TrapFrame,
         while (TRUE);
     }
     
+    /* Check if this is a user trap */
+    if (KiUserTrap(TrapFrame))
+    {
         /* Check if segments should be restored */
         if (!SkipBits.SkipSegments)
         {
@@ -98,8 +101,7 @@ KiExitTrap(IN PKTRAP_FRAME TrapFrame,
             Ke386SetDs(TrapFrame->SegDs);
             Ke386SetFs(TrapFrame->SegFs);
         }
-    else if (KiUserTrap(TrapFrame))
+        
-    {
         /* Always restore FS since it goes from KPCR to TEB */
         Ke386SetFs(TrapFrame->SegFs);
     }
@@ -225,6 +227,58 @@ KiEnterV86Trap(IN PKTRAP_FRAME TrapFrame)
     }
 }
 
+VOID
+FASTCALL
+KiEnterInterruptTrap(IN PKTRAP_FRAME TrapFrame)
+{
+    /* Save registers */
+    KiTrapFrameFromPushaStack(TrapFrame);
+    
+    /* Set bogus previous mode */
+    TrapFrame->PreviousPreviousMode = -1;
+    
+    /* Check for V86 mode */
+    if (TrapFrame->EFlags & EFLAGS_V86_MASK)
+    {
+        UNIMPLEMENTED;
+        while (TRUE);
+    }
+    
+    /* Check if this wasn't kernel code */
+    if (TrapFrame->SegCs != KGDT_R0_CODE)
+    {
+        /* Save segments and then switch to correct ones */
+        TrapFrame->SegFs = Ke386GetFs();
+        TrapFrame->SegGs = Ke386GetGs();
+        TrapFrame->SegDs = Ke386GetDs();
+        TrapFrame->SegEs = Ke386GetEs();
+        Ke386SetFs(KGDT_R0_PCR);
+        Ke386SetDs(KGDT_R3_DATA | RPL_MASK);
+        Ke386SetEs(KGDT_R3_DATA | RPL_MASK);
+    }
+    
+    /* Save exception list and terminate it */
+    TrapFrame->ExceptionList = KeGetPcr()->Tib.ExceptionList;
+    KeGetPcr()->Tib.ExceptionList = EXCEPTION_CHAIN_END;
+    
+    /* FIXME: This doesn't support 16-bit ABIOS interrupts */
+    TrapFrame->ErrCode = 0;
+    
+    /* Clear direction flag */
+    Ke386ClearDirectionFlag();
+    
+    /* Flush DR7 and check for debugging */
+    TrapFrame->Dr7 = 0;
+    if (KeGetCurrentThread()->DispatcherHeader.DebugActive & 0xFF)
+    {
+        UNIMPLEMENTED;
+        while (TRUE);
+    }
+    
+    /* Set debug header */
+    KiFillTrapFrameDebug(TrapFrame);
+}
+
 VOID
 FASTCALL
 KiEnterTrap(IN PKTRAP_FRAME TrapFrame)
@@ -1499,6 +1553,24 @@ KiTrap19Handler(IN PKTRAP_FRAME TrapFrame)
     KeBugCheckWithTf(TRAP_CAUSE_UNKNOWN, 13, 0, 0, 1, TrapFrame);
 }
 
+/* SOFTWARE SERVICES **********************************************************/
+
+VOID
+FASTCALL
+KiGetTickCountHandler(IN PKTRAP_FRAME TrapFrame)
+{
+    UNIMPLEMENTED;
+    while (TRUE);
+}
+
+VOID
+FASTCALL
+KiCallbackReturnHandler(IN PKTRAP_FRAME TrapFrame)
+{
+    UNIMPLEMENTED;
+    while (TRUE);
+}
+
 VOID
 FASTCALL
 KiRaiseAssertionHandler(IN PKTRAP_FRAME TrapFrame)
@@ -1529,4 +1601,104 @@ KiDebugServiceHandler(IN PKTRAP_FRAME TrapFrame)
     KiDebugHandler(TrapFrame, TrapFrame->Eax, TrapFrame->Ecx, TrapFrame->Edx);
 }
 
+/* HARDWARE INTERRUPTS ********************************************************/
+
+/*
+ * This code can only be used once the HAL handles system interrupt code in C.
+ *
+ * This is because the HAL, when ending a system interrupt, might see pending
+ * DPC or APC interrupts, and attempt to piggyback on the interrupt context in
+ * order to deliver them. Once they have been devlivered, it will then "end" the
+ * interrupt context by doing a call to the ASM EOI Handler which naturally will
+ * throw up on our C-style KTRAP_FRAME.
+ *
+ * Once it works, expect a noticeable speed boost during hardware interrupts.
+ */
+#ifdef HAL_INTERRUPT_SUPPORT_IN_C
+
+typedef
+FASTCALL
+VOID
+(PKI_INTERRUPT_DISPATCH)(
+    IN PKTRAP_FRAME TrapFrame,
+    IN PKINTERRUPT Interrupt
+);
+
+VOID
+FORCEINLINE
+KiExitInterrupt(IN PKTRAP_FRAME TrapFrame,
+                IN KIRQL OldIrql,
+                IN BOOLEAN Spurious)
+{
+    if (Spurious) KiEoiHelper(TrapFrame);
+    
+    _disable();
+    
+    DPRINT1("Calling HAL to restore IRQL to: %d\n", OldIrql);
+    HalEndSystemInterrupt(OldIrql, 0);
+    
+    DPRINT1("Exiting trap\n");
+    KiEoiHelper(TrapFrame);
+}
+
+VOID
+FASTCALL
+KiInterruptDispatch(IN PKTRAP_FRAME TrapFrame,
+                    IN PKINTERRUPT Interrupt)
+{       
+    KIRQL OldIrql;
+
+    KeGetCurrentPrcb()->InterruptCount++;
+    
+    DPRINT1("Calling HAL with %lx %lx\n", Interrupt->SynchronizeIrql, Interrupt->Vector);
+    if (HalBeginSystemInterrupt(Interrupt->SynchronizeIrql,
+                                Interrupt->Vector,
+                                &OldIrql))
+    {
+        /* Acquire interrupt lock */
+        KxAcquireSpinLock(Interrupt->ActualLock);
+        
+        /* Call the ISR */
+        DPRINT1("Calling ISR: %p with context: %p\n", Interrupt->ServiceRoutine, Interrupt->ServiceContext);
+        Interrupt->ServiceRoutine(Interrupt, Interrupt->ServiceContext);
+        
+        /* Release interrupt lock */
+        KxReleaseSpinLock(Interrupt->ActualLock);
+        
+        /* Now call the epilogue code */
+        DPRINT1("Exiting interrupt\n");
+        KiExitInterrupt(TrapFrame, OldIrql, FALSE);
+    }
+    else
+    {
+        /* Now call the epilogue code */
+        DPRINT1("Exiting Spurious interrupt\n");
+        KiExitInterrupt(TrapFrame, OldIrql, TRUE);
+    }
+}
+
+VOID
+FASTCALL
+KiChainedDispatch(IN PKTRAP_FRAME TrapFrame,
+                  IN PKINTERRUPT Interrupt)
+{   
+    KeGetCurrentPrcb()->InterruptCount++;
+    
+    UNIMPLEMENTED;
+    while (TRUE);
+}
+
+VOID
+FASTCALL
+KiInterruptHandler(IN PKTRAP_FRAME TrapFrame,
+                   IN PKINTERRUPT Interrupt)
+{   
+    /* Enter interrupt frame */
+    KiEnterInterruptTrap(TrapFrame);
+
+    /* Call the correct dispatcher */
+    ((PKI_INTERRUPT_DISPATCH*)Interrupt->DispatchAddress)(TrapFrame, Interrupt);
+}
+#endif
+
 /* EOF */

reactos/ntoskrnl/ke/spinlock.c

@@ -454,3 +454,17 @@ KeTestSpinLock(IN PKSPIN_LOCK SpinLock)
     /* Spinlock appears to be free */
     return TRUE;
 }
+
+#ifdef _M_IX86
+/*
+ * @unimplemented
+ */
+VOID
+NTAPI
+Kii386SpinOnSpinLock(IN PKSPIN_LOCK SpinLock,
+                     IN ULONG Flags)
+{
+    UNIMPLEMENTED;
+    while (TRUE);
+}
+#endif