diff --git a/src/input_common/drivers/sdl_driver.cpp b/src/input_common/drivers/sdl_driver.cpp
index 0b24f1858..d5af6c09b 100644
--- a/src/input_common/drivers/sdl_driver.cpp
+++ b/src/input_common/drivers/sdl_driver.cpp
@@ -220,24 +220,6 @@ public:
         return "Unknown";
     }
 
-    bool IsYAxis(u8 index) {
-        if (!sdl_controller) {
-            return false;
-        }
-
-        const auto& binding_left_y =
-            SDL_GameControllerGetBindForAxis(sdl_controller.get(), SDL_CONTROLLER_AXIS_LEFTY);
-        const auto& binding_right_y =
-            SDL_GameControllerGetBindForAxis(sdl_controller.get(), SDL_CONTROLLER_AXIS_RIGHTY);
-        if (index == binding_left_y.value.axis) {
-            return true;
-        }
-        if (index == binding_right_y.value.axis) {
-            return true;
-        }
-        return false;
-    }
-
 private:
     std::string guid;
     int port;
@@ -376,11 +358,6 @@ void SDLDriver::HandleGameControllerEvent(const SDL_Event& event) {
     case SDL_JOYAXISMOTION: {
         if (const auto joystick = GetSDLJoystickBySDLID(event.jaxis.which)) {
             const PadIdentifier identifier = joystick->GetPadIdentifier();
-            // Vertical axis is inverted on nintendo compared to SDL
-            if (joystick->IsYAxis(event.jaxis.axis)) {
-                SetAxis(identifier, event.jaxis.axis, -event.jaxis.value / 32767.0f);
-                break;
-            }
             SetAxis(identifier, event.jaxis.axis, event.jaxis.value / 32767.0f);
         }
         break;
diff --git a/src/input_common/input_poller.cpp b/src/input_common/input_poller.cpp
index 92cf690cd..7e4eafded 100644
--- a/src/input_common/input_poller.cpp
+++ b/src/input_common/input_poller.cpp
@@ -146,7 +146,8 @@ public:
                             Common::Input::AnalogProperties properties_y_,
                             InputEngine* input_engine_)
         : identifier(identifier_), axis_x(axis_x_), axis_y(axis_y_), properties_x(properties_x_),
-          properties_y(properties_y_), input_engine(input_engine_) {
+          properties_y(properties_y_),
+          input_engine(input_engine_), invert_axis_y{input_engine_->GetEngineName() == "sdl"} {
         UpdateCallback engine_callback{[this]() { OnChange(); }};
         const InputIdentifier x_input_identifier{
             .identifier = identifier,
@@ -181,6 +182,11 @@ public:
             .raw_value = input_engine->GetAxis(identifier, axis_y),
             .properties = properties_y,
         };
+        // This is a workaround too keep compatibility with old yuzu versions. Vertical axis is
+        // inverted on SDL compared to Nintendo
+        if (invert_axis_y) {
+            status.y.raw_value = -status.y.raw_value;
+        }
         return status;
     }
 
@@ -220,6 +226,7 @@ private:
     float last_axis_x_value;
     float last_axis_y_value;
     InputEngine* input_engine;
+    const bool invert_axis_y;
 };
 
 class InputFromTouch final : public Common::Input::InputDevice {