[SHELL32] ShellExecute: Re-work Part 1 (#6871)

The goal is a correct implementation of shell32!ShellExecute.
JIRA issue: CORE-17482
- Add SHELL_InRunDllProcess helper function.
- Add structure size check to ShellExecuteExA function.
- Add code to ShellExecuteExW function.
- Add "MaximizeApps" registry value check.

dll/win32/shell32/shlexec.cpp

@@ -3,7 +3,7 @@
  *
  * Copyright 1998 Marcus Meissner
  * Copyright 2002 Eric Pouech
- * Copyright 2018-2019 Katayama Hirofumi MZ
+ * Copyright 2018-2024 Katayama Hirofumi MZ
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -32,6 +32,20 @@ EXTERN_C BOOL PathIsExeW(LPCWSTR lpszPath);
 typedef UINT_PTR (*SHELL_ExecuteW32)(const WCHAR *lpCmd, WCHAR *env, BOOL shWait,
                 const SHELLEXECUTEINFOW *sei, LPSHELLEXECUTEINFOW sei_out);
 
+// Is the current process a rundll32.exe?
+static BOOL SHELL_InRunDllProcess(VOID)
+{
+    WCHAR szModule[MAX_PATH];
+    static INT s_bInDllProcess = -1;
+
+    if (s_bInDllProcess != -1)
+        return s_bInDllProcess;
+
+    s_bInDllProcess = GetModuleFileNameW(NULL, szModule, _countof(szModule)) &&
+                      (StrStrIW(PathFindFileNameW(szModule), L"rundll") != NULL);
+    return s_bInDllProcess;
+}
+
 static void ParseNoTildeEffect(PWSTR &res, LPCWSTR &args, DWORD &len, DWORD &used, int argNum)
 {
     bool firstCharQuote = false;
@@ -2331,9 +2345,24 @@ HINSTANCE WINAPI ShellExecuteA(HWND hWnd, LPCSTR lpVerb, LPCSTR lpFile,
     return sei.hInstApp;
 }
 
+static DWORD
+ShellExecute_Normal(_Inout_ LPSHELLEXECUTEINFOW sei)
+{
+    // FIXME
+    return SHELL_execute(sei, SHELL_ExecuteW) ? ERROR_SUCCESS : ERROR_FILE_NOT_FOUND;
+}
+
+static VOID
+ShellExecute_ShowError(
+    _In_ const SHELLEXECUTEINFOW *ExecInfo,
+    _In_opt_ LPCWSTR pszCaption,
+    _In_ DWORD dwError)
+{
+    // FIXME: Show error message
+}
+
 /*************************************************************************
  * ShellExecuteExA                [SHELL32.292]
- *
  */
 BOOL
 WINAPI
@@ -2346,8 +2375,17 @@ ShellExecuteExA(LPSHELLEXECUTEINFOA sei)
 
     TRACE("%p\n", sei);
 
+    if (sei->cbSize != sizeof(SHELLEXECUTEINFOA))
+    {
+        sei->hInstApp = (HINSTANCE)ERROR_ACCESS_DENIED;
+        SetLastError(ERROR_ACCESS_DENIED);
+        return FALSE;
+    }
+
     memcpy(&seiW, sei, sizeof(SHELLEXECUTEINFOW));
 
+    seiW.cbSize = sizeof(SHELLEXECUTEINFOW);
+
     if (sei->lpVerb)
         seiW.lpVerb = __SHCloneStrAtoW(&wVerb, sei->lpVerb);
 
@@ -2365,7 +2403,7 @@ ShellExecuteExA(LPSHELLEXECUTEINFOA sei)
     else
         seiW.lpClass = NULL;
 
-    ret = SHELL_execute(&seiW, SHELL_ExecuteW);
+    ret = ShellExecuteExW(&seiW);
 
     sei->hInstApp = seiW.hInstApp;
 
@@ -2383,14 +2421,61 @@ ShellExecuteExA(LPSHELLEXECUTEINFOA sei)
 
 /*************************************************************************
  * ShellExecuteExW                [SHELL32.293]
- *
  */
 BOOL
 WINAPI
 DECLSPEC_HOTPATCH
 ShellExecuteExW(LPSHELLEXECUTEINFOW sei)
 {
-    return SHELL_execute(sei, SHELL_ExecuteW);
+    HRESULT hrCoInit;
+    DWORD dwError;
+    ULONG fOldMask;
+
+    hrCoInit = SHCoInitializeAnyApartment();
+
+    if (sei->cbSize != sizeof(SHELLEXECUTEINFOW))
+    {
+        dwError = ERROR_ACCESS_DENIED;
+        sei->hInstApp = (HINSTANCE)ERROR_ACCESS_DENIED;
+    }
+    else
+    {
+        if (SHRegGetBoolUSValueW(L"Software\\Microsoft\\Windows\\CurrentVersion\\Explorer",
+                                 L"MaximizeApps", FALSE, FALSE))
+        {
+            switch (sei->nShow)
+            {
+                case SW_SHOW:
+                case SW_SHOWDEFAULT:
+                case SW_SHOWNORMAL:
+                case SW_RESTORE:
+                    sei->nShow = SW_SHOWMAXIMIZED;
+                    break;
+                default:
+                    break;
+            }
+        }
+
+        fOldMask = sei->fMask;
+
+        if (!(fOldMask & SEE_MASK_NOASYNC) && SHELL_InRunDllProcess())
+            sei->fMask |= SEE_MASK_WAITFORINPUTIDLE | SEE_MASK_NOASYNC;
+
+        dwError = ShellExecute_Normal(sei);
+
+        if (dwError && dwError != ERROR_DLL_NOT_FOUND && dwError != ERROR_CANCELLED)
+            ShellExecute_ShowError(sei, NULL, dwError);
+
+        sei->fMask = fOldMask;
+    }
+
+    if (SUCCEEDED(hrCoInit))
+        CoUninitialize();
+
+    if (dwError)
+        SetLastError(dwError);
+
+    return dwError == ERROR_SUCCESS;
 }
 
 /*************************************************************************