diff --git a/dll/win32/winmm/joystick.c b/dll/win32/winmm/joystick.c
index ed6054bfbf7..92bd48d3995 100644
--- a/dll/win32/winmm/joystick.c
+++ b/dll/win32/winmm/joystick.c
@@ -55,12 +55,21 @@ static	WINE_JOYSTICK	JOY_Sticks[MAXJOYSTICK];
  */
 static	BOOL JOY_LoadDriver(DWORD dwJoyID)
 {
-    if (dwJoyID >= MAXJOYSTICK)
+    static BOOL winejoystick_missing = FALSE;
+
+    if (dwJoyID >= MAXJOYSTICK || winejoystick_missing)
 	return FALSE;
     if (JOY_Sticks[dwJoyID].hDriver)
 	return TRUE;
 
     JOY_Sticks[dwJoyID].hDriver = OpenDriverA("winejoystick.drv", 0, dwJoyID);
+
+    if (!JOY_Sticks[dwJoyID].hDriver)
+    {
+        /* The default driver is missing, don't attempt to load it again */
+        winejoystick_missing = TRUE;
+    }
+
     return (JOY_Sticks[dwJoyID].hDriver != 0);
 }