[ACLAYERS] Implement the ForceDXSetupSuccess shim

This shim is used on older installers that try to install DirectX,
but encounter errors doing so with the bundled setup.
CORE-15814

dll/appcompat/shims/layer/CMakeLists.txt

@@ -5,6 +5,7 @@ spec2def(aclayers.dll layer.spec)
 
 list(APPEND SOURCE
     dispmode.c
+    forcedxsetupsuccess.c
     versionlie.c
     vmhorizon.c
     main.c

dll/appcompat/shims/layer/forcedxsetupsuccess.c

@@ -0,0 +1,268 @@
+/*
+ * PROJECT:     ReactOS 'Layers' Shim library
+ * LICENSE:     GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later)
+ * PURPOSE:     ForceDxSetupSuccess shim
+ * COPYRIGHT:   Copyright 2019 Mark Jansen (mark.jansen@reactos.org)
+ */
+
+#define WIN32_NO_STATUS
+#include <windef.h>
+#include <winbase.h>
+#include <shimlib.h>
+#include "ntndk.h"
+
+typedef HMODULE(WINAPI* LOADLIBRARYAPROC)(LPCSTR lpLibFileName);
+typedef HMODULE(WINAPI* LOADLIBRARYWPROC)(LPCWSTR lpLibFileName);
+typedef HMODULE(WINAPI* LOADLIBRARYEXAPROC)(LPCSTR lpLibFileName, HANDLE hFile, DWORD dwFlags);
+typedef HMODULE(WINAPI* LOADLIBRARYEXWPROC)(LPCWSTR lpLibFileName, HANDLE hFile, DWORD dwFlags);
+typedef FARPROC(WINAPI* GETPROCADDRESSPROC)(HMODULE hModule, LPCSTR lpProcName);
+typedef BOOL   (WINAPI* FREELIBRARYPROC)(HINSTANCE hLibModule);
+
+
+#define SHIM_NS         ForceDxSetupSuccess
+#include <setup_shim.inl>
+
+
+#define DSETUPERR_SUCCESS 0
+
+INT WINAPI DirectXSetup(HWND hWnd, LPSTR lpszRootPath, DWORD dwFlags)
+{
+    SHIM_MSG("Returning DSETUPERR_SUCCESS\n");
+    return DSETUPERR_SUCCESS;
+}
+
+INT WINAPI DirectXSetupA(HWND hWnd, LPSTR lpszRootPath, DWORD dwFlags)
+{
+    SHIM_MSG("Returning DSETUPERR_SUCCESS\n");
+    return DSETUPERR_SUCCESS;
+}
+
+INT WINAPI DirectXSetupW(HWND hWnd, LPWSTR lpszRootPath, DWORD dwFlags)
+{
+    SHIM_MSG("Returning DSETUPERR_SUCCESS\n");
+    return DSETUPERR_SUCCESS;
+}
+
+INT WINAPI DirectXSetupGetVersion(DWORD *lpdwVersion, DWORD *lpdwMinorVersion)
+{
+    if (lpdwVersion)
+        *lpdwVersion = MAKELONG(7, 4);     // DirectX 7.0
+    if (lpdwMinorVersion)
+        *lpdwMinorVersion = MAKELONG(1792, 0);
+
+    return TRUE;
+}
+
+
+static
+BOOLEAN
+IsCharInAnsiString(
+    IN CHAR Char,
+    IN const STRING* MatchString)
+{
+    USHORT i;
+
+    for (i = 0; i < MatchString->Length; i++)
+    {
+        if (Char == MatchString->Buffer[i])
+            return TRUE;
+    }
+
+    return FALSE;
+}
+
+
+NTSTATUS
+NTAPI
+FindCharInAnsiString(
+    IN const STRING* SearchString,
+    IN const STRING* MatchString,
+    OUT PUSHORT Position)
+{
+    BOOLEAN Found;
+    ULONG i, Length;
+
+    *Position = 0;
+
+    /* Search */
+    Length = SearchString->Length;
+    for (i = Length - 1; (LONG)i >= 0; i--)
+    {
+        Found = IsCharInAnsiString(SearchString->Buffer[i], MatchString);
+        if (Found)
+        {
+            *Position = i;
+            return STATUS_SUCCESS;
+        }
+    }
+
+    return STATUS_NOT_FOUND;
+}
+
+
+static BOOL IsDxSetupA(const PSTRING LibraryPath)
+{
+    static const STRING DxSetupDlls[] = {
+        RTL_CONSTANT_STRING("dsetup.dll"),
+        RTL_CONSTANT_STRING("dsetup32.dll"),
+        RTL_CONSTANT_STRING("dsetup"),
+        RTL_CONSTANT_STRING("dsetup32"),
+    };
+    static const STRING PathDividerFind = RTL_CONSTANT_STRING("\\/");
+    STRING LibraryName;
+    USHORT PathDivider;
+    DWORD n;
+
+    if (!NT_SUCCESS(FindCharInAnsiString(LibraryPath, &PathDividerFind, &PathDivider)))
+        PathDivider = 0;
+
+    if (PathDivider)
+        PathDivider += sizeof(CHAR);
+
+    LibraryName.Buffer = LibraryPath->Buffer + PathDivider;
+    LibraryName.Length = LibraryPath->Length - PathDivider;
+    LibraryName.MaximumLength = LibraryPath->MaximumLength - PathDivider;
+
+    for (n = 0; n < ARRAYSIZE(DxSetupDlls); ++n)
+    {
+        if (RtlEqualString(&LibraryName, DxSetupDlls + n, TRUE))
+        {
+            SHIM_MSG("Found %Z\n", DxSetupDlls + n);
+            return TRUE;
+        }
+    }
+    return FALSE;
+}
+
+static BOOL IsDxSetupW(PCUNICODE_STRING LibraryPath)
+{
+    static const UNICODE_STRING DxSetupDlls[] = {
+        RTL_CONSTANT_STRING(L"dsetup.dll"),
+        RTL_CONSTANT_STRING(L"dsetup32.dll"),
+        RTL_CONSTANT_STRING(L"dsetup"),
+        RTL_CONSTANT_STRING(L"dsetup32"),
+    };
+    static const UNICODE_STRING PathDividerFind = RTL_CONSTANT_STRING(L"\\/");
+    UNICODE_STRING LibraryName;
+    USHORT PathDivider;
+    DWORD n;
+
+    if (!NT_SUCCESS(RtlFindCharInUnicodeString(RTL_FIND_CHAR_IN_UNICODE_STRING_START_AT_END, LibraryPath, &PathDividerFind, &PathDivider)))
+        PathDivider = 0;
+
+    if (PathDivider)
+        PathDivider += sizeof(WCHAR);
+
+    LibraryName.Buffer = LibraryPath->Buffer + PathDivider / sizeof(WCHAR);
+    LibraryName.Length = LibraryPath->Length - PathDivider;
+    LibraryName.MaximumLength = LibraryPath->MaximumLength - PathDivider;
+
+    for (n = 0; n < ARRAYSIZE(DxSetupDlls); ++n)
+    {
+        if (RtlEqualUnicodeString(&LibraryName, DxSetupDlls + n, TRUE))
+        {
+            SHIM_MSG("Found %wZ\n", DxSetupDlls + n);
+            return TRUE;
+        }
+    }
+    return FALSE;
+}
+
+HMODULE WINAPI SHIM_OBJ_NAME(APIHook_LoadLibraryA)(LPCSTR lpLibFileName)
+{
+    STRING Library;
+
+    RtlInitAnsiString(&Library, lpLibFileName);
+    if (IsDxSetupA(&Library))
+        return ShimLib_Instance();
+
+    return CALL_SHIM(0, LOADLIBRARYAPROC)(lpLibFileName);
+}
+
+HMODULE WINAPI SHIM_OBJ_NAME(APIHook_LoadLibraryW)(LPCWSTR lpLibFileName)
+{
+    UNICODE_STRING Library;
+
+    RtlInitUnicodeString(&Library, lpLibFileName);
+    if (IsDxSetupW(&Library))
+        return ShimLib_Instance();
+
+    return CALL_SHIM(1, LOADLIBRARYWPROC)(lpLibFileName);
+}
+
+HMODULE WINAPI SHIM_OBJ_NAME(APIHook_LoadLibraryExA)(LPCSTR lpLibFileName, HANDLE hFile, DWORD dwFlags)
+{
+    STRING Library;
+
+    RtlInitAnsiString(&Library, lpLibFileName);
+    if (IsDxSetupA(&Library))
+        return ShimLib_Instance();
+
+    return CALL_SHIM(2, LOADLIBRARYEXAPROC)(lpLibFileName, hFile, dwFlags);
+}
+
+HMODULE WINAPI SHIM_OBJ_NAME(APIHook_LoadLibraryExW)(LPCWSTR lpLibFileName, HANDLE hFile, DWORD dwFlags)
+{
+    UNICODE_STRING Library;
+
+    RtlInitUnicodeString(&Library, lpLibFileName);
+    if (IsDxSetupW(&Library))
+        return ShimLib_Instance();
+
+    return CALL_SHIM(3, LOADLIBRARYEXWPROC)(lpLibFileName, hFile, dwFlags);
+}
+
+FARPROC WINAPI SHIM_OBJ_NAME(APIHook_GetProcAddress)(HMODULE hModule, LPCSTR lpProcName)
+{
+    static const STRING DxSetupFunctions[] = {
+        RTL_CONSTANT_STRING("DirectXSetup"),
+        RTL_CONSTANT_STRING("DirectXSetupA"),
+        RTL_CONSTANT_STRING("DirectXSetupW"),
+        RTL_CONSTANT_STRING("DirectXSetupGetVersion"),
+    };
+    static const FARPROC DxSetupRedirections[] = {
+        DirectXSetup,
+        DirectXSetupA,
+        DirectXSetupW,
+        DirectXSetupGetVersion,
+    };
+    DWORD n;
+
+    if (hModule == ShimLib_Instance() && ((ULONG_PTR)lpProcName > MAXUSHORT))
+    {
+        STRING ProcName;
+        RtlInitAnsiString(&ProcName, lpProcName);
+        for (n = 0; n < ARRAYSIZE(DxSetupFunctions); ++n)
+        {
+            if (RtlEqualString(&ProcName, DxSetupFunctions + n, TRUE))
+            {
+                SHIM_MSG("Intercepted %Z\n", DxSetupFunctions + n);
+                return DxSetupRedirections[n];
+            }
+        }
+    }
+    return CALL_SHIM(4, GETPROCADDRESSPROC)(hModule, lpProcName);
+}
+
+BOOL WINAPI SHIM_OBJ_NAME(APIHook_FreeLibrary)(HINSTANCE hLibModule)
+{
+    if (hLibModule == ShimLib_Instance())
+    {
+        SHIM_MSG("Intercepted\n");
+        return TRUE;
+    }
+
+    return CALL_SHIM(5, FREELIBRARYPROC)(hLibModule);
+}
+
+
+#define SHIM_NUM_HOOKS  6
+#define SHIM_SETUP_HOOKS \
+    SHIM_HOOK(0, "KERNEL32.DLL", "LoadLibraryA", SHIM_OBJ_NAME(APIHook_LoadLibraryA)) \
+    SHIM_HOOK(1, "KERNEL32.DLL", "LoadLibraryW", SHIM_OBJ_NAME(APIHook_LoadLibraryW)) \
+    SHIM_HOOK(2, "KERNEL32.DLL", "LoadLibraryExA", SHIM_OBJ_NAME(APIHook_LoadLibraryExA)) \
+    SHIM_HOOK(3, "KERNEL32.DLL", "LoadLibraryExW", SHIM_OBJ_NAME(APIHook_LoadLibraryExW)) \
+    SHIM_HOOK(4, "KERNEL32.DLL", "GetProcAddress", SHIM_OBJ_NAME(APIHook_GetProcAddress)) \
+    SHIM_HOOK(5, "KERNEL32.DLL", "FreeLibrary", SHIM_OBJ_NAME(APIHook_FreeLibrary))
+
+#include <implement_shim.inl>

dll/appcompat/shims/shimlib/shimlib.c

@@ -23,18 +23,20 @@ typedef struct UsedShim
 
 
 ULONG g_ShimEngDebugLevel = 0xffffffff;
+static HINSTANCE g_ShimLib_hInstance;
 static HANDLE g_ShimLib_Heap;
 static PSLIST_HEADER g_UsedShims;
 
 void ShimLib_Init(HINSTANCE hInstance)
 {
+    g_ShimLib_hInstance = hInstance;
     g_ShimLib_Heap = HeapCreate(0, 0x10000, 0);
 
     g_UsedShims = (PSLIST_HEADER)ShimLib_ShimMalloc(sizeof(SLIST_HEADER));
     RtlInitializeSListHead(g_UsedShims);
 }
 
-void ShimLib_Deinit()
+void ShimLib_Deinit(VOID)
 {
     // Is this a good idea?
     HeapDestroy(g_ShimLib_Heap);
@@ -50,6 +52,11 @@ void ShimLib_ShimFree(PVOID pData)
     HeapFree(g_ShimLib_Heap, 0, pData);
 }
 
+HINSTANCE ShimLib_Instance(VOID)
+{
+    return g_ShimLib_hInstance;
+}
+
 PCSTR ShimLib_StringNDuplicateA(PCSTR szString, SIZE_T stringLengthIncludingNullTerm)
 {
     PSTR NewString = ShimLib_ShimMalloc(stringLengthIncludingNullTerm);

dll/appcompat/shims/shimlib/shimlib.h

@@ -28,7 +28,7 @@ VOID ShimLib_ShimFree(PVOID pData);
 PCSTR ShimLib_StringDuplicateA(PCSTR szString);
 PCSTR ShimLib_StringNDuplicateA(PCSTR szString, SIZE_T stringLength);
 BOOL ShimLib_StrAEqualsW(PCSTR szString, PCWSTR wszString);
-
+HINSTANCE ShimLib_Instance(VOID);
 
 /* Forward events to generic handlers */
 VOID ShimLib_Init(HINSTANCE hInstance);

media/sdb/sysmain.xml

@@ -215,6 +215,10 @@
             <SHIM NAME="DisableThemes">
                 <DLLFILE>acgenral.dll</DLLFILE>
             </SHIM>
+            <SHIM NAME="ForceDXSetupSuccess">
+                <DLLFILE>aclayers.dll</DLLFILE>
+                <DESCRIPTION>Some application using an older version of DirectX may encounter problems when calling LoadLibrary or GetProcAddress to use DSETUP.DLL or DSETUP32.DLL</DESCRIPTION>
+            </SHIM>
             <SHIM NAME="IgnoreFreeLibrary">
                 <DLLFILE>acgenral.dll</DLLFILE>
             </SHIM>
@@ -239,10 +243,12 @@
 
         <LAYER NAME="WIN95">
             <SHIM_REF NAME="Win95VersionLie" />
+            <SHIM_REF NAME="ForceDXSetupSuccess" />
             <!-- TODO: Add more fixes! -->
         </LAYER>
         <LAYER NAME="WIN98">
             <SHIM_REF NAME="Win98VersionLie" />
+            <SHIM_REF NAME="ForceDXSetupSuccess" />
             <!-- TODO: Add more fixes! -->
         </LAYER>
         <LAYER NAME="NT4SP5">
@@ -363,6 +369,9 @@
         <LAYER NAME="DisableThemes">
             <SHIM_REF NAME="DisableThemes" />
         </LAYER>
+        <LAYER NAME="ForceDXSetupSuccess">
+            <SHIM_REF NAME="ForceDXSetupSuccess" />
+        </LAYER>
         <LAYER NAME="VMHorizonSetup">
             <!-- ProductId: {7051C96D-AA61-4D83-AF37-646E82D616ED} -->
             <SHIM_REF NAME="VMHorizonSetup" />

modules/rostests/apitests/appshim/CMakeLists.txt

@@ -3,6 +3,7 @@ add_definitions(-D__ROS_LONG64__)
 
 list(APPEND SOURCE
     dispmode.c
+    forcedxsetup.c
     genral_hooks.c
     ignorefreelib.c
     layer_hooks.c