[OSK] Fix an instance race condition

On-Screen Keyboard provides a mechanism to launch the application only once, to avoid multiple entry point instances. Such mechanism is based upon mutex objects, although it could happen that the program may end up creating two or more mutexes (a race condition).

CORE-15877

base/applications/osk/CMakeLists.txt

@@ -3,5 +3,5 @@ file(GLOB osk_rc_deps res/*.*)
 add_rc_deps(rsrc.rc ${osk_rc_deps})
 add_executable(osk main.c settings.c rsrc.rc)
 set_module_type(osk win32gui UNICODE)
-add_importlibs(osk comdlg32 shell32 user32 gdi32 advapi32 comctl32 msvcrt kernel32 winmm)
+add_importlibs(osk comdlg32 winmm shell32 user32 gdi32 advapi32 comctl32 msvcrt kernel32 ntdll)
 add_cd_file(TARGET osk DESTINATION reactos/system32 FOR all)

base/applications/osk/main.c

@@ -585,12 +585,36 @@ int WINAPI wWinMain(HINSTANCE hInstance,
                     int show)
 {
     HANDLE hMutex;
+    DWORD dwError;
     INT LayoutResource;
 
     UNREFERENCED_PARAMETER(prev);
     UNREFERENCED_PARAMETER(cmdline);
     UNREFERENCED_PARAMETER(show);
 
+    /*
+        Obtain a mutex for the program. This will ensure that
+        the program is launched only once.
+    */
+    hMutex = CreateMutexW(NULL, FALSE, L"OSKRunning");
+
+    if (hMutex)
+    {
+        /* Check if there's already a mutex for the program */
+        dwError = GetLastError();
+
+        if (dwError == ERROR_ALREADY_EXISTS)
+        {
+            /*
+                A mutex with the object name has been created previously.
+                Therefore, another instance is already running.
+            */
+            DPRINT("wWinMain(): Failed to create a mutex! The program instance is already running.\n");
+            CloseHandle(hMutex);
+            return 0;
+        }
+    }
+
     ZeroMemory(&Globals, sizeof(Globals));
     Globals.hInstance = hInstance;
 
@@ -613,31 +637,16 @@ int WINAPI wWinMain(HINSTANCE hInstance,
         LayoutResource = MAIN_DIALOG_STANDARD_KB;
     }
 
-    /* Rry to open a mutex for a single instance */
-    hMutex = OpenMutexW(MUTEX_ALL_ACCESS, FALSE, L"osk");
+    /* Create the modal box based on the configuration registry */
+    DialogBoxW(hInstance,
+               MAKEINTRESOURCEW(LayoutResource),
+               GetDesktopWindow(),
+               OSK_DlgProc);
 
-    if (!hMutex)
+    /* Delete the mutex */
+    if (hMutex)
     {
-        /* Mutex doesn't exist. This is the first instance so create the mutex. */
-        hMutex = CreateMutexW(NULL, FALSE, L"osk");
-
-        /* Create the modal box based on the configuration registry */
-        DialogBoxW(hInstance,
-                   MAKEINTRESOURCEW(LayoutResource),
-                   GetDesktopWindow(),
-                   OSK_DlgProc);
-
-        /* Delete the mutex */
-        if (hMutex) CloseHandle(hMutex);
-    }
-    else
-    {
-        /* Programme already launched */
-
-        /* Delete the mutex */
         CloseHandle(hMutex);
-
-        ExitProcess(0);
     }
 
     return 0;

base/applications/osk/osk.h

@@ -14,6 +14,7 @@
 #include <stdio.h>
 
 #include <windows.h>
+#include <debug.h>
 
 #include "main.h"