diff --git a/src/video/x11/SDL_x11video.h b/src/video/x11/SDL_x11video.h
index a853e4290..c6ab6dc17 100644
--- a/src/video/x11/SDL_x11video.h
+++ b/src/video/x11/SDL_x11video.h
@@ -124,6 +124,8 @@ typedef struct SDL_VideoData
     SDL_Scancode key_layout[256];
     SDL_bool selection_waiting;
 
+    SDL_bool broken_pointer_grab;  /* true if XGrabPointer seems unreliable. */
+
     Uint32 last_mode_change_deadline;
 
     SDL_bool global_mouse_changed;
diff --git a/src/video/x11/SDL_x11window.c b/src/video/x11/SDL_x11window.c
index 0464728e3..10d233fe3 100644
--- a/src/video/x11/SDL_x11window.c
+++ b/src/video/x11/SDL_x11window.c
@@ -41,6 +41,7 @@
 #include "SDL_timer.h"
 #include "SDL_syswm.h"
 #include "SDL_assert.h"
+#include "SDL_log.h"
 
 #define _NET_WM_STATE_REMOVE    0l
 #define _NET_WM_STATE_ADD       1l
@@ -1483,14 +1484,24 @@ X11_SetWindowGrab(_THIS, SDL_Window * window, SDL_bool grabbed)
 
     if (oldstyle_fullscreen || grabbed) {
         /* Try to grab the mouse */
-        for (;;) {
-            int result =
-                X11_XGrabPointer(display, data->xwindow, True, 0, GrabModeAsync,
-                             GrabModeAsync, data->xwindow, None, CurrentTime);
-            if (result == GrabSuccess) {
-                break;
+        if (!data->videodata->broken_pointer_grab) {
+            int attempts;
+            int result;
+
+            /* Try for up to ~250ms to grab. If it still fails, stop trying. */
+            for (attempts = 0; attempts < 5; attempts++) {
+                result = X11_XGrabPointer(display, data->xwindow, True, 0, GrabModeAsync,
+                                 GrabModeAsync, data->xwindow, None, CurrentTime);
+                if (result == GrabSuccess) {
+                    break;
+                }
+                SDL_Delay(50);
+            }
+
+            if (result != GrabSuccess) {
+                SDL_LogWarn(SDL_LOG_CATEGORY_VIDEO, "The X server refused to let us grab the mouse. You might experience input bugs.");
+                data->videodata->broken_pointer_grab = SDL_TRUE;  /* don't try again. */
             }
-            SDL_Delay(50);
         }
 
         /* Raise the window if we grab the mouse */