diff --git a/src/input_common/CMakeLists.txt b/src/input_common/CMakeLists.txt
index 1c7db28c03..5b4e032bd9 100644
--- a/src/input_common/CMakeLists.txt
+++ b/src/input_common/CMakeLists.txt
@@ -7,15 +7,18 @@ add_library(input_common STATIC
     main.h
     motion_emu.cpp
     motion_emu.h
-
-    $<$<BOOL:${SDL2_FOUND}>:sdl/sdl.cpp sdl/sdl.h>
+    sdl/sdl.cpp
+    sdl/sdl.h
 )
 
-create_target_directory_groups(input_common)
-
-target_link_libraries(input_common PUBLIC core PRIVATE common)
-
 if(SDL2_FOUND)
+    target_sources(input_common PRIVATE
+        sdl/sdl_impl.cpp
+        sdl/sdl_impl.h
+    )
     target_link_libraries(input_common PRIVATE SDL2)
     target_compile_definitions(input_common PRIVATE HAVE_SDL2)
 endif()
+
+create_target_directory_groups(input_common)
+target_link_libraries(input_common PUBLIC core PRIVATE common)
diff --git a/src/input_common/main.cpp b/src/input_common/main.cpp
index 37f5728532..8e66c1b150 100644
--- a/src/input_common/main.cpp
+++ b/src/input_common/main.cpp
@@ -17,10 +17,7 @@ namespace InputCommon {
 
 static std::shared_ptr<Keyboard> keyboard;
 static std::shared_ptr<MotionEmu> motion_emu;
-
-#ifdef HAVE_SDL2
-static std::thread poll_thread;
-#endif
+static std::unique_ptr<SDL::State> sdl;
 
 void Init() {
     keyboard = std::make_shared<Keyboard>();
@@ -30,15 +27,7 @@ void Init() {
     motion_emu = std::make_shared<MotionEmu>();
     Input::RegisterFactory<Input::MotionDevice>("motion_emu", motion_emu);
 
-#ifdef HAVE_SDL2
-    SDL::Init();
-#endif
-}
-
-void StartJoystickEventHandler() {
-#ifdef HAVE_SDL2
-    poll_thread = std::thread(SDL::PollLoop);
-#endif
+    sdl = SDL::Init();
 }
 
 void Shutdown() {
@@ -47,11 +36,7 @@ void Shutdown() {
     Input::UnregisterFactory<Input::AnalogDevice>("analog_from_button");
     Input::UnregisterFactory<Input::MotionDevice>("motion_emu");
     motion_emu.reset();
-
-#ifdef HAVE_SDL2
-    SDL::Shutdown();
-    poll_thread.join();
-#endif
+    sdl.reset();
 }
 
 Keyboard* GetKeyboard() {
@@ -88,7 +73,7 @@ namespace Polling {
 
 std::vector<std::unique_ptr<DevicePoller>> GetPollers(DeviceType type) {
 #ifdef HAVE_SDL2
-    return SDL::Polling::GetPollers(type);
+    return sdl->GetPollers(type);
 #else
     return {};
 #endif
diff --git a/src/input_common/main.h b/src/input_common/main.h
index 9eb13106e5..77a0ce90b2 100644
--- a/src/input_common/main.h
+++ b/src/input_common/main.h
@@ -20,8 +20,6 @@ void Init();
 /// Deregisters all built-in input device factories and shuts them down.
 void Shutdown();
 
-void StartJoystickEventHandler();
-
 class Keyboard;
 
 /// Gets the keyboard button device factory.
diff --git a/src/input_common/sdl/sdl.cpp b/src/input_common/sdl/sdl.cpp
index faf3c1fa30..644db3448f 100644
--- a/src/input_common/sdl/sdl.cpp
+++ b/src/input_common/sdl/sdl.cpp
@@ -1,631 +1,19 @@
-// Copyright 2017 Citra Emulator Project
+// Copyright 2018 Citra Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
-#include <algorithm>
-#include <atomic>
-#include <cmath>
-#include <functional>
-#include <iterator>
-#include <mutex>
-#include <string>
-#include <thread>
-#include <tuple>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-#include <SDL.h>
-#include "common/assert.h"
-#include "common/logging/log.h"
-#include "common/math_util.h"
-#include "common/param_package.h"
-#include "common/threadsafe_queue.h"
-#include "input_common/main.h"
 #include "input_common/sdl/sdl.h"
+#ifdef HAVE_SDL2
+#include "input_common/sdl/sdl_impl.h"
+#endif
 
-namespace InputCommon {
+namespace InputCommon::SDL {
 
-namespace SDL {
-
-class SDLJoystick;
-class SDLButtonFactory;
-class SDLAnalogFactory;
-
-/// Map of GUID of a list of corresponding virtual Joysticks
-static std::unordered_map<std::string, std::vector<std::shared_ptr<SDLJoystick>>> joystick_map;
-static std::mutex joystick_map_mutex;
-
-static std::shared_ptr<SDLButtonFactory> button_factory;
-static std::shared_ptr<SDLAnalogFactory> analog_factory;
-
-/// Used by the Pollers during config
-static std::atomic<bool> polling;
-static Common::SPSCQueue<SDL_Event> event_queue;
-
-static std::atomic<bool> initialized = false;
-
-static std::string GetGUID(SDL_Joystick* joystick) {
-    SDL_JoystickGUID guid = SDL_JoystickGetGUID(joystick);
-    char guid_str[33];
-    SDL_JoystickGetGUIDString(guid, guid_str, sizeof(guid_str));
-    return guid_str;
+std::unique_ptr<State> Init() {
+#ifdef HAVE_SDL2
+    return std::make_unique<SDLState>();
+#else
+    return std::make_unique<NullState>();
+#endif
 }
-
-class SDLJoystick {
-public:
-    SDLJoystick(std::string guid_, int port_, SDL_Joystick* joystick,
-                decltype(&SDL_JoystickClose) deleter = &SDL_JoystickClose)
-        : guid{std::move(guid_)}, port{port_}, sdl_joystick{joystick, deleter} {}
-
-    void SetButton(int button, bool value) {
-        std::lock_guard<std::mutex> lock(mutex);
-        state.buttons[button] = value;
-    }
-
-    bool GetButton(int button) const {
-        std::lock_guard<std::mutex> lock(mutex);
-        return state.buttons.at(button);
-    }
-
-    void SetAxis(int axis, Sint16 value) {
-        std::lock_guard<std::mutex> lock(mutex);
-        state.axes[axis] = value;
-    }
-
-    float GetAxis(int axis) const {
-        std::lock_guard<std::mutex> lock(mutex);
-        return state.axes.at(axis) / 32767.0f;
-    }
-
-    std::tuple<float, float> GetAnalog(int axis_x, int axis_y) const {
-        float x = GetAxis(axis_x);
-        float y = GetAxis(axis_y);
-        y = -y; // 3DS uses an y-axis inverse from SDL
-
-        // Make sure the coordinates are in the unit circle,
-        // otherwise normalize it.
-        float r = x * x + y * y;
-        if (r > 1.0f) {
-            r = std::sqrt(r);
-            x /= r;
-            y /= r;
-        }
-
-        return std::make_tuple(x, y);
-    }
-
-    void SetHat(int hat, Uint8 direction) {
-        std::lock_guard<std::mutex> lock(mutex);
-        state.hats[hat] = direction;
-    }
-
-    bool GetHatDirection(int hat, Uint8 direction) const {
-        std::lock_guard<std::mutex> lock(mutex);
-        return (state.hats.at(hat) & direction) != 0;
-    }
-    /**
-     * The guid of the joystick
-     */
-    const std::string& GetGUID() const {
-        return guid;
-    }
-
-    /**
-     * The number of joystick from the same type that were connected before this joystick
-     */
-    int GetPort() const {
-        return port;
-    }
-
-    SDL_Joystick* GetSDLJoystick() const {
-        return sdl_joystick.get();
-    }
-
-    void SetSDLJoystick(SDL_Joystick* joystick,
-                        decltype(&SDL_JoystickClose) deleter = &SDL_JoystickClose) {
-        sdl_joystick =
-            std::unique_ptr<SDL_Joystick, decltype(&SDL_JoystickClose)>(joystick, deleter);
-    }
-
-private:
-    struct State {
-        std::unordered_map<int, bool> buttons;
-        std::unordered_map<int, Sint16> axes;
-        std::unordered_map<int, Uint8> hats;
-    } state;
-    std::string guid;
-    int port;
-    std::unique_ptr<SDL_Joystick, decltype(&SDL_JoystickClose)> sdl_joystick;
-    mutable std::mutex mutex;
-};
-
-/**
- * Get the nth joystick with the corresponding GUID
- */
-static std::shared_ptr<SDLJoystick> GetSDLJoystickByGUID(const std::string& guid, int port) {
-    std::lock_guard<std::mutex> lock(joystick_map_mutex);
-    const auto it = joystick_map.find(guid);
-    if (it != joystick_map.end()) {
-        while (it->second.size() <= port) {
-            auto joystick = std::make_shared<SDLJoystick>(guid, it->second.size(), nullptr,
-                                                          [](SDL_Joystick*) {});
-            it->second.emplace_back(std::move(joystick));
-        }
-        return it->second[port];
-    }
-    auto joystick = std::make_shared<SDLJoystick>(guid, 0, nullptr, [](SDL_Joystick*) {});
-    return joystick_map[guid].emplace_back(std::move(joystick));
-}
-
-/**
- * Check how many identical joysticks (by guid) were connected before the one with sdl_id and so tie
- * it to a SDLJoystick with the same guid and that port
- */
-static std::shared_ptr<SDLJoystick> GetSDLJoystickBySDLID(SDL_JoystickID sdl_id) {
-    std::lock_guard<std::mutex> lock(joystick_map_mutex);
-    auto sdl_joystick = SDL_JoystickFromInstanceID(sdl_id);
-    const std::string guid = GetGUID(sdl_joystick);
-    auto map_it = joystick_map.find(guid);
-    if (map_it != joystick_map.end()) {
-        auto vec_it = std::find_if(map_it->second.begin(), map_it->second.end(),
-                                   [&sdl_joystick](const std::shared_ptr<SDLJoystick>& joystick) {
-                                       return sdl_joystick == joystick->GetSDLJoystick();
-                                   });
-        if (vec_it != map_it->second.end()) {
-            // This is the common case: There is already an existing SDL_Joystick maped to a
-            // SDLJoystick. return the SDLJoystick
-            return *vec_it;
-        }
-        // Search for a SDLJoystick without a mapped SDL_Joystick...
-        auto nullptr_it = std::find_if(map_it->second.begin(), map_it->second.end(),
-                                       [](const std::shared_ptr<SDLJoystick>& joystick) {
-                                           return !joystick->GetSDLJoystick();
-                                       });
-        if (nullptr_it != map_it->second.end()) {
-            // ... and map it
-            (*nullptr_it)->SetSDLJoystick(sdl_joystick);
-            return *nullptr_it;
-        }
-        // There is no SDLJoystick without a mapped SDL_Joystick
-        // Create a new SDLJoystick
-        auto joystick = std::make_shared<SDLJoystick>(guid, map_it->second.size(), sdl_joystick);
-        return map_it->second.emplace_back(std::move(joystick));
-    }
-    auto joystick = std::make_shared<SDLJoystick>(guid, 0, sdl_joystick);
-    return joystick_map[guid].emplace_back(std::move(joystick));
-}
-
-void InitJoystick(int joystick_index) {
-    std::lock_guard<std::mutex> lock(joystick_map_mutex);
-    SDL_Joystick* sdl_joystick = SDL_JoystickOpen(joystick_index);
-    if (!sdl_joystick) {
-        LOG_ERROR(Input, "failed to open joystick {}", joystick_index);
-        return;
-    }
-    std::string guid = GetGUID(sdl_joystick);
-    if (joystick_map.find(guid) == joystick_map.end()) {
-        auto joystick = std::make_shared<SDLJoystick>(guid, 0, sdl_joystick);
-        joystick_map[guid].emplace_back(std::move(joystick));
-        return;
-    }
-    auto& joystick_guid_list = joystick_map[guid];
-    const auto it = std::find_if(
-        joystick_guid_list.begin(), joystick_guid_list.end(),
-        [](const std::shared_ptr<SDLJoystick>& joystick) { return !joystick->GetSDLJoystick(); });
-    if (it != joystick_guid_list.end()) {
-        (*it)->SetSDLJoystick(sdl_joystick);
-        return;
-    }
-    auto joystick = std::make_shared<SDLJoystick>(guid, joystick_guid_list.size(), sdl_joystick);
-    joystick_guid_list.emplace_back(std::move(joystick));
-}
-
-void CloseJoystick(SDL_Joystick* sdl_joystick) {
-    std::lock_guard<std::mutex> lock(joystick_map_mutex);
-    std::string guid = GetGUID(sdl_joystick);
-    // This call to guid is save since the joystick is guranteed to be in that map
-    auto& joystick_guid_list = joystick_map[guid];
-    const auto joystick_it =
-        std::find_if(joystick_guid_list.begin(), joystick_guid_list.end(),
-                     [&sdl_joystick](const std::shared_ptr<SDLJoystick>& joystick) {
-                         return joystick->GetSDLJoystick() == sdl_joystick;
-                     });
-    (*joystick_it)->SetSDLJoystick(nullptr, [](SDL_Joystick*) {});
-}
-
-void HandleGameControllerEvent(const SDL_Event& event) {
-    switch (event.type) {
-    case SDL_JOYBUTTONUP: {
-        auto joystick = GetSDLJoystickBySDLID(event.jbutton.which);
-        if (joystick) {
-            joystick->SetButton(event.jbutton.button, false);
-        }
-        break;
-    }
-    case SDL_JOYBUTTONDOWN: {
-        auto joystick = GetSDLJoystickBySDLID(event.jbutton.which);
-        if (joystick) {
-            joystick->SetButton(event.jbutton.button, true);
-        }
-        break;
-    }
-    case SDL_JOYHATMOTION: {
-        auto joystick = GetSDLJoystickBySDLID(event.jhat.which);
-        if (joystick) {
-            joystick->SetHat(event.jhat.hat, event.jhat.value);
-        }
-        break;
-    }
-    case SDL_JOYAXISMOTION: {
-        auto joystick = GetSDLJoystickBySDLID(event.jaxis.which);
-        if (joystick) {
-            joystick->SetAxis(event.jaxis.axis, event.jaxis.value);
-        }
-        break;
-    }
-    case SDL_JOYDEVICEREMOVED:
-        LOG_DEBUG(Input, "Controller removed with Instance_ID {}", event.jdevice.which);
-        CloseJoystick(SDL_JoystickFromInstanceID(event.jdevice.which));
-        break;
-    case SDL_JOYDEVICEADDED:
-        LOG_DEBUG(Input, "Controller connected with device index {}", event.jdevice.which);
-        InitJoystick(event.jdevice.which);
-        break;
-    }
-}
-
-void CloseSDLJoysticks() {
-    std::lock_guard<std::mutex> lock(joystick_map_mutex);
-    joystick_map.clear();
-}
-
-void PollLoop() {
-    if (SDL_Init(SDL_INIT_JOYSTICK) < 0) {
-        LOG_CRITICAL(Input, "SDL_Init(SDL_INIT_JOYSTICK) failed with: {}", SDL_GetError());
-        return;
-    }
-
-    SDL_Event event;
-    while (initialized) {
-        // Wait for 10 ms or until an event happens
-        if (SDL_WaitEventTimeout(&event, 10)) {
-            // Don't handle the event if we are configuring
-            if (polling) {
-                event_queue.Push(event);
-            } else {
-                HandleGameControllerEvent(event);
-            }
-        }
-    }
-    CloseSDLJoysticks();
-    SDL_QuitSubSystem(SDL_INIT_JOYSTICK);
-}
-
-class SDLButton final : public Input::ButtonDevice {
-public:
-    explicit SDLButton(std::shared_ptr<SDLJoystick> joystick_, int button_)
-        : joystick(std::move(joystick_)), button(button_) {}
-
-    bool GetStatus() const override {
-        return joystick->GetButton(button);
-    }
-
-private:
-    std::shared_ptr<SDLJoystick> joystick;
-    int button;
-};
-
-class SDLDirectionButton final : public Input::ButtonDevice {
-public:
-    explicit SDLDirectionButton(std::shared_ptr<SDLJoystick> joystick_, int hat_, Uint8 direction_)
-        : joystick(std::move(joystick_)), hat(hat_), direction(direction_) {}
-
-    bool GetStatus() const override {
-        return joystick->GetHatDirection(hat, direction);
-    }
-
-private:
-    std::shared_ptr<SDLJoystick> joystick;
-    int hat;
-    Uint8 direction;
-};
-
-class SDLAxisButton final : public Input::ButtonDevice {
-public:
-    explicit SDLAxisButton(std::shared_ptr<SDLJoystick> joystick_, int axis_, float threshold_,
-                           bool trigger_if_greater_)
-        : joystick(std::move(joystick_)), axis(axis_), threshold(threshold_),
-          trigger_if_greater(trigger_if_greater_) {}
-
-    bool GetStatus() const override {
-        float axis_value = joystick->GetAxis(axis);
-        if (trigger_if_greater)
-            return axis_value > threshold;
-        return axis_value < threshold;
-    }
-
-private:
-    std::shared_ptr<SDLJoystick> joystick;
-    int axis;
-    float threshold;
-    bool trigger_if_greater;
-};
-
-class SDLAnalog final : public Input::AnalogDevice {
-public:
-    SDLAnalog(std::shared_ptr<SDLJoystick> joystick_, int axis_x_, int axis_y_)
-        : joystick(std::move(joystick_)), axis_x(axis_x_), axis_y(axis_y_) {}
-
-    std::tuple<float, float> GetStatus() const override {
-        return joystick->GetAnalog(axis_x, axis_y);
-    }
-
-private:
-    std::shared_ptr<SDLJoystick> joystick;
-    int axis_x;
-    int axis_y;
-};
-
-/// A button device factory that creates button devices from SDL joystick
-class SDLButtonFactory final : public Input::Factory<Input::ButtonDevice> {
-public:
-    /**
-     * Creates a button device from a joystick button
-     * @param params contains parameters for creating the device:
-     *     - "guid": the guid of the joystick to bind
-     *     - "port": the nth joystick of the same type to bind
-     *     - "button"(optional): the index of the button to bind
-     *     - "hat"(optional): the index of the hat to bind as direction buttons
-     *     - "axis"(optional): the index of the axis to bind
-     *     - "direction"(only used for hat): the direction name of the hat to bind. Can be "up",
-     *         "down", "left" or "right"
-     *     - "threshold"(only used for axis): a float value in (-1.0, 1.0) which the button is
-     *         triggered if the axis value crosses
-     *     - "direction"(only used for axis): "+" means the button is triggered when the axis
-     * value is greater than the threshold; "-" means the button is triggered when the axis
-     * value is smaller than the threshold
-     */
-    std::unique_ptr<Input::ButtonDevice> Create(const Common::ParamPackage& params) override {
-        const std::string guid = params.Get("guid", "0");
-        const int port = params.Get("port", 0);
-
-        auto joystick = GetSDLJoystickByGUID(guid, port);
-
-        if (params.Has("hat")) {
-            const int hat = params.Get("hat", 0);
-            const std::string direction_name = params.Get("direction", "");
-            Uint8 direction;
-            if (direction_name == "up") {
-                direction = SDL_HAT_UP;
-            } else if (direction_name == "down") {
-                direction = SDL_HAT_DOWN;
-            } else if (direction_name == "left") {
-                direction = SDL_HAT_LEFT;
-            } else if (direction_name == "right") {
-                direction = SDL_HAT_RIGHT;
-            } else {
-                direction = 0;
-            }
-            // This is necessary so accessing GetHat with hat won't crash
-            joystick->SetHat(hat, SDL_HAT_CENTERED);
-            return std::make_unique<SDLDirectionButton>(joystick, hat, direction);
-        }
-
-        if (params.Has("axis")) {
-            const int axis = params.Get("axis", 0);
-            const float threshold = params.Get("threshold", 0.5f);
-            const std::string direction_name = params.Get("direction", "");
-            bool trigger_if_greater;
-            if (direction_name == "+") {
-                trigger_if_greater = true;
-            } else if (direction_name == "-") {
-                trigger_if_greater = false;
-            } else {
-                trigger_if_greater = true;
-                LOG_ERROR(Input, "Unknown direction '{}'", direction_name);
-            }
-            // This is necessary so accessing GetAxis with axis won't crash
-            joystick->SetAxis(axis, 0);
-            return std::make_unique<SDLAxisButton>(joystick, axis, threshold, trigger_if_greater);
-        }
-
-        const int button = params.Get("button", 0);
-        // This is necessary so accessing GetButton with button won't crash
-        joystick->SetButton(button, false);
-        return std::make_unique<SDLButton>(joystick, button);
-    }
-};
-
-/// An analog device factory that creates analog devices from SDL joystick
-class SDLAnalogFactory final : public Input::Factory<Input::AnalogDevice> {
-public:
-    /**
-     * Creates analog device from joystick axes
-     * @param params contains parameters for creating the device:
-     *     - "guid": the guid of the joystick to bind
-     *     - "port": the nth joystick of the same type
-     *     - "axis_x": the index of the axis to be bind as x-axis
-     *     - "axis_y": the index of the axis to be bind as y-axis
-     */
-    std::unique_ptr<Input::AnalogDevice> Create(const Common::ParamPackage& params) override {
-        const std::string guid = params.Get("guid", "0");
-        const int port = params.Get("port", 0);
-        const int axis_x = params.Get("axis_x", 0);
-        const int axis_y = params.Get("axis_y", 1);
-
-        auto joystick = GetSDLJoystickByGUID(guid, port);
-
-        // This is necessary so accessing GetAxis with axis_x and axis_y won't crash
-        joystick->SetAxis(axis_x, 0);
-        joystick->SetAxis(axis_y, 0);
-        return std::make_unique<SDLAnalog>(joystick, axis_x, axis_y);
-    }
-};
-
-void Init() {
-    using namespace Input;
-    RegisterFactory<ButtonDevice>("sdl", std::make_shared<SDLButtonFactory>());
-    RegisterFactory<AnalogDevice>("sdl", std::make_shared<SDLAnalogFactory>());
-    polling = false;
-    initialized = true;
-}
-
-void Shutdown() {
-    if (initialized) {
-        using namespace Input;
-        UnregisterFactory<ButtonDevice>("sdl");
-        UnregisterFactory<AnalogDevice>("sdl");
-        initialized = false;
-    }
-}
-
-Common::ParamPackage SDLEventToButtonParamPackage(const SDL_Event& event) {
-    Common::ParamPackage params({{"engine", "sdl"}});
-    switch (event.type) {
-    case SDL_JOYAXISMOTION: {
-        auto joystick = GetSDLJoystickBySDLID(event.jaxis.which);
-        params.Set("port", joystick->GetPort());
-        params.Set("guid", joystick->GetGUID());
-        params.Set("axis", event.jaxis.axis);
-        if (event.jaxis.value > 0) {
-            params.Set("direction", "+");
-            params.Set("threshold", "0.5");
-        } else {
-            params.Set("direction", "-");
-            params.Set("threshold", "-0.5");
-        }
-        break;
-    }
-    case SDL_JOYBUTTONUP: {
-        auto joystick = GetSDLJoystickBySDLID(event.jbutton.which);
-        params.Set("port", joystick->GetPort());
-        params.Set("guid", joystick->GetGUID());
-        params.Set("button", event.jbutton.button);
-        break;
-    }
-    case SDL_JOYHATMOTION: {
-        auto joystick = GetSDLJoystickBySDLID(event.jhat.which);
-        params.Set("port", joystick->GetPort());
-        params.Set("guid", joystick->GetGUID());
-        params.Set("hat", event.jhat.hat);
-        switch (event.jhat.value) {
-        case SDL_HAT_UP:
-            params.Set("direction", "up");
-            break;
-        case SDL_HAT_DOWN:
-            params.Set("direction", "down");
-            break;
-        case SDL_HAT_LEFT:
-            params.Set("direction", "left");
-            break;
-        case SDL_HAT_RIGHT:
-            params.Set("direction", "right");
-            break;
-        default:
-            return {};
-        }
-        break;
-    }
-    }
-    return params;
-}
-
-namespace Polling {
-
-class SDLPoller : public InputCommon::Polling::DevicePoller {
-public:
-    void Start() override {
-        event_queue.Clear();
-        polling = true;
-    }
-
-    void Stop() override {
-        polling = false;
-    }
-};
-
-class SDLButtonPoller final : public SDLPoller {
-public:
-    Common::ParamPackage GetNextInput() override {
-        SDL_Event event;
-        while (event_queue.Pop(event)) {
-            switch (event.type) {
-            case SDL_JOYAXISMOTION:
-                if (std::abs(event.jaxis.value / 32767.0) < 0.5) {
-                    break;
-                }
-            case SDL_JOYBUTTONUP:
-            case SDL_JOYHATMOTION:
-                return SDLEventToButtonParamPackage(event);
-            }
-        }
-        return {};
-    }
-};
-
-class SDLAnalogPoller final : public SDLPoller {
-public:
-    void Start() override {
-        SDLPoller::Start();
-
-        // Reset stored axes
-        analog_xaxis = -1;
-        analog_yaxis = -1;
-        analog_axes_joystick = -1;
-    }
-
-    Common::ParamPackage GetNextInput() override {
-        SDL_Event event;
-        while (event_queue.Pop(event)) {
-            if (event.type != SDL_JOYAXISMOTION || std::abs(event.jaxis.value / 32767.0) < 0.5) {
-                continue;
-            }
-            // An analog device needs two axes, so we need to store the axis for later and wait for
-            // a second SDL event. The axes also must be from the same joystick.
-            int axis = event.jaxis.axis;
-            if (analog_xaxis == -1) {
-                analog_xaxis = axis;
-                analog_axes_joystick = event.jaxis.which;
-            } else if (analog_yaxis == -1 && analog_xaxis != axis &&
-                       analog_axes_joystick == event.jaxis.which) {
-                analog_yaxis = axis;
-            }
-        }
-        Common::ParamPackage params;
-        if (analog_xaxis != -1 && analog_yaxis != -1) {
-            auto joystick = GetSDLJoystickBySDLID(event.jaxis.which);
-            params.Set("engine", "sdl");
-            params.Set("port", joystick->GetPort());
-            params.Set("guid", joystick->GetGUID());
-            params.Set("axis_x", analog_xaxis);
-            params.Set("axis_y", analog_yaxis);
-            analog_xaxis = -1;
-            analog_yaxis = -1;
-            analog_axes_joystick = -1;
-            return params;
-        }
-        return params;
-    }
-
-private:
-    int analog_xaxis = -1;
-    int analog_yaxis = -1;
-    SDL_JoystickID analog_axes_joystick = -1;
-};
-
-std::vector<std::unique_ptr<InputCommon::Polling::DevicePoller>> GetPollers(
-    InputCommon::Polling::DeviceType type) {
-    std::vector<std::unique_ptr<InputCommon::Polling::DevicePoller>> pollers;
-    switch (type) {
-    case InputCommon::Polling::DeviceType::Analog:
-        pollers.push_back(std::make_unique<SDLAnalogPoller>());
-        break;
-    case InputCommon::Polling::DeviceType::Button:
-        pollers.push_back(std::make_unique<SDLButtonPoller>());
-        break;
-    }
-    return pollers;
-}
-} // namespace Polling
-} // namespace SDL
-} // namespace InputCommon
+} // namespace InputCommon::SDL
diff --git a/src/input_common/sdl/sdl.h b/src/input_common/sdl/sdl.h
index 0206860d32..02a8d2e2c9 100644
--- a/src/input_common/sdl/sdl.h
+++ b/src/input_common/sdl/sdl.h
@@ -1,4 +1,4 @@
-// Copyright 2017 Citra Emulator Project
+// Copyright 2018 Citra Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
@@ -7,45 +7,36 @@
 #include <memory>
 #include <vector>
 #include "core/frontend/input.h"
+#include "input_common/main.h"
 
 union SDL_Event;
+
 namespace Common {
 class ParamPackage;
-}
-namespace InputCommon {
-namespace Polling {
+} // namespace Common
+
+namespace InputCommon::Polling {
 class DevicePoller;
 enum class DeviceType;
-} // namespace Polling
-} // namespace InputCommon
+} // namespace InputCommon::Polling
 
-namespace InputCommon {
-namespace SDL {
+namespace InputCommon::SDL {
 
-/// Initializes and registers SDL device factories
-void Init();
+class State {
+public:
+    /// Unresisters SDL device factories and shut them down.
+    virtual ~State() = default;
 
-/// Unresisters SDL device factories and shut them down.
-void Shutdown();
+    virtual std::vector<std::unique_ptr<InputCommon::Polling::DevicePoller>> GetPollers(
+        InputCommon::Polling::DeviceType type) = 0;
+};
 
-/// Needs to be called before SDL_QuitSubSystem.
-void CloseSDLJoysticks();
+class NullState : public State {
+public:
+    std::vector<std::unique_ptr<InputCommon::Polling::DevicePoller>> GetPollers(
+        InputCommon::Polling::DeviceType type) override {}
+};
 
-/// Handle SDL_Events for joysticks from SDL_PollEvent
-void HandleGameControllerEvent(const SDL_Event& event);
+std::unique_ptr<State> Init();
 
-/// A Loop that calls HandleGameControllerEvent until Shutdown is called
-void PollLoop();
-
-/// Creates a ParamPackage from an SDL_Event that can directly be used to create a ButtonDevice
-Common::ParamPackage SDLEventToButtonParamPackage(const SDL_Event& event);
-
-namespace Polling {
-
-/// Get all DevicePoller that use the SDL backend for a specific device type
-std::vector<std::unique_ptr<InputCommon::Polling::DevicePoller>> GetPollers(
-    InputCommon::Polling::DeviceType type);
-
-} // namespace Polling
-} // namespace SDL
-} // namespace InputCommon
+} // namespace InputCommon::SDL
diff --git a/src/input_common/sdl/sdl_impl.cpp b/src/input_common/sdl/sdl_impl.cpp
new file mode 100644
index 0000000000..934339d3bc
--- /dev/null
+++ b/src/input_common/sdl/sdl_impl.cpp
@@ -0,0 +1,669 @@
+// Copyright 2018 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include <algorithm>
+#include <atomic>
+#include <cmath>
+#include <functional>
+#include <iterator>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <tuple>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+#include <SDL.h>
+#include "common/assert.h"
+#include "common/logging/log.h"
+#include "common/math_util.h"
+#include "common/param_package.h"
+#include "common/threadsafe_queue.h"
+#include "core/frontend/input.h"
+#include "input_common/sdl/sdl_impl.h"
+
+namespace InputCommon {
+
+namespace SDL {
+
+static std::string GetGUID(SDL_Joystick* joystick) {
+    SDL_JoystickGUID guid = SDL_JoystickGetGUID(joystick);
+    char guid_str[33];
+    SDL_JoystickGetGUIDString(guid, guid_str, sizeof(guid_str));
+    return guid_str;
+}
+
+/// Creates a ParamPackage from an SDL_Event that can directly be used to create a ButtonDevice
+static Common::ParamPackage SDLEventToButtonParamPackage(SDLState& state, const SDL_Event& event);
+
+static int SDLEventWatcher(void* userdata, SDL_Event* event) {
+    SDLState* sdl_state = reinterpret_cast<SDLState*>(userdata);
+    // Don't handle the event if we are configuring
+    if (sdl_state->polling) {
+        sdl_state->event_queue.Push(*event);
+    } else {
+        sdl_state->HandleGameControllerEvent(*event);
+    }
+    return 0;
+}
+
+class SDLJoystick {
+public:
+    SDLJoystick(std::string guid_, int port_, SDL_Joystick* joystick,
+                decltype(&SDL_JoystickClose) deleter = &SDL_JoystickClose)
+        : guid{std::move(guid_)}, port{port_}, sdl_joystick{joystick, deleter} {}
+
+    void SetButton(int button, bool value) {
+        std::lock_guard<std::mutex> lock(mutex);
+        state.buttons[button] = value;
+    }
+
+    bool GetButton(int button) const {
+        std::lock_guard<std::mutex> lock(mutex);
+        return state.buttons.at(button);
+    }
+
+    void SetAxis(int axis, Sint16 value) {
+        std::lock_guard<std::mutex> lock(mutex);
+        state.axes[axis] = value;
+    }
+
+    float GetAxis(int axis) const {
+        std::lock_guard<std::mutex> lock(mutex);
+        return state.axes.at(axis) / 32767.0f;
+    }
+
+    std::tuple<float, float> GetAnalog(int axis_x, int axis_y) const {
+        float x = GetAxis(axis_x);
+        float y = GetAxis(axis_y);
+        y = -y; // 3DS uses an y-axis inverse from SDL
+
+        // Make sure the coordinates are in the unit circle,
+        // otherwise normalize it.
+        float r = x * x + y * y;
+        if (r > 1.0f) {
+            r = std::sqrt(r);
+            x /= r;
+            y /= r;
+        }
+
+        return std::make_tuple(x, y);
+    }
+
+    void SetHat(int hat, Uint8 direction) {
+        std::lock_guard<std::mutex> lock(mutex);
+        state.hats[hat] = direction;
+    }
+
+    bool GetHatDirection(int hat, Uint8 direction) const {
+        std::lock_guard<std::mutex> lock(mutex);
+        return (state.hats.at(hat) & direction) != 0;
+    }
+    /**
+     * The guid of the joystick
+     */
+    const std::string& GetGUID() const {
+        return guid;
+    }
+
+    /**
+     * The number of joystick from the same type that were connected before this joystick
+     */
+    int GetPort() const {
+        return port;
+    }
+
+    SDL_Joystick* GetSDLJoystick() const {
+        return sdl_joystick.get();
+    }
+
+    void SetSDLJoystick(SDL_Joystick* joystick,
+                        decltype(&SDL_JoystickClose) deleter = &SDL_JoystickClose) {
+        sdl_joystick =
+            std::unique_ptr<SDL_Joystick, decltype(&SDL_JoystickClose)>(joystick, deleter);
+    }
+
+private:
+    struct State {
+        std::unordered_map<int, bool> buttons;
+        std::unordered_map<int, Sint16> axes;
+        std::unordered_map<int, Uint8> hats;
+    } state;
+    std::string guid;
+    int port;
+    std::unique_ptr<SDL_Joystick, decltype(&SDL_JoystickClose)> sdl_joystick;
+    mutable std::mutex mutex;
+};
+
+/**
+ * Get the nth joystick with the corresponding GUID
+ */
+std::shared_ptr<SDLJoystick> SDLState::GetSDLJoystickByGUID(const std::string& guid, int port) {
+    std::lock_guard<std::mutex> lock(joystick_map_mutex);
+    const auto it = joystick_map.find(guid);
+    if (it != joystick_map.end()) {
+        while (it->second.size() <= port) {
+            auto joystick = std::make_shared<SDLJoystick>(guid, it->second.size(), nullptr,
+                                                          [](SDL_Joystick*) {});
+            it->second.emplace_back(std::move(joystick));
+        }
+        return it->second[port];
+    }
+    auto joystick = std::make_shared<SDLJoystick>(guid, 0, nullptr, [](SDL_Joystick*) {});
+    return joystick_map[guid].emplace_back(std::move(joystick));
+}
+
+/**
+ * Check how many identical joysticks (by guid) were connected before the one with sdl_id and so tie
+ * it to a SDLJoystick with the same guid and that port
+ */
+std::shared_ptr<SDLJoystick> SDLState::GetSDLJoystickBySDLID(SDL_JoystickID sdl_id) {
+    auto sdl_joystick = SDL_JoystickFromInstanceID(sdl_id);
+    const std::string guid = GetGUID(sdl_joystick);
+    std::lock_guard<std::mutex> lock(joystick_map_mutex);
+    auto map_it = joystick_map.find(guid);
+    if (map_it != joystick_map.end()) {
+        auto vec_it = std::find_if(map_it->second.begin(), map_it->second.end(),
+                                   [&sdl_joystick](const std::shared_ptr<SDLJoystick>& joystick) {
+                                       return sdl_joystick == joystick->GetSDLJoystick();
+                                   });
+        if (vec_it != map_it->second.end()) {
+            // This is the common case: There is already an existing SDL_Joystick maped to a
+            // SDLJoystick. return the SDLJoystick
+            return *vec_it;
+        }
+        // Search for a SDLJoystick without a mapped SDL_Joystick...
+        auto nullptr_it = std::find_if(map_it->second.begin(), map_it->second.end(),
+                                       [](const std::shared_ptr<SDLJoystick>& joystick) {
+                                           return !joystick->GetSDLJoystick();
+                                       });
+        if (nullptr_it != map_it->second.end()) {
+            // ... and map it
+            (*nullptr_it)->SetSDLJoystick(sdl_joystick);
+            return *nullptr_it;
+        }
+        // There is no SDLJoystick without a mapped SDL_Joystick
+        // Create a new SDLJoystick
+        auto joystick = std::make_shared<SDLJoystick>(guid, map_it->second.size(), sdl_joystick);
+        return map_it->second.emplace_back(std::move(joystick));
+    }
+    auto joystick = std::make_shared<SDLJoystick>(guid, 0, sdl_joystick);
+    return joystick_map[guid].emplace_back(std::move(joystick));
+}
+
+void SDLState::InitJoystick(int joystick_index) {
+    SDL_Joystick* sdl_joystick = SDL_JoystickOpen(joystick_index);
+    if (!sdl_joystick) {
+        LOG_ERROR(Input, "failed to open joystick {}", joystick_index);
+        return;
+    }
+    std::string guid = GetGUID(sdl_joystick);
+    std::lock_guard<std::mutex> lock(joystick_map_mutex);
+    if (joystick_map.find(guid) == joystick_map.end()) {
+        auto joystick = std::make_shared<SDLJoystick>(guid, 0, sdl_joystick);
+        joystick_map[guid].emplace_back(std::move(joystick));
+        return;
+    }
+    auto& joystick_guid_list = joystick_map[guid];
+    const auto it = std::find_if(
+        joystick_guid_list.begin(), joystick_guid_list.end(),
+        [](const std::shared_ptr<SDLJoystick>& joystick) { return !joystick->GetSDLJoystick(); });
+    if (it != joystick_guid_list.end()) {
+        (*it)->SetSDLJoystick(sdl_joystick);
+        return;
+    }
+    auto joystick = std::make_shared<SDLJoystick>(guid, joystick_guid_list.size(), sdl_joystick);
+    joystick_guid_list.emplace_back(std::move(joystick));
+}
+
+void SDLState::CloseJoystick(SDL_Joystick* sdl_joystick) {
+    std::string guid = GetGUID(sdl_joystick);
+    std::shared_ptr<SDLJoystick> joystick;
+    {
+        std::lock_guard<std::mutex> lock(joystick_map_mutex);
+        // This call to guid is safe since the joystick is guaranteed to be in the map
+        auto& joystick_guid_list = joystick_map[guid];
+        const auto joystick_it =
+            std::find_if(joystick_guid_list.begin(), joystick_guid_list.end(),
+                         [&sdl_joystick](const std::shared_ptr<SDLJoystick>& joystick) {
+                             return joystick->GetSDLJoystick() == sdl_joystick;
+                         });
+        joystick = *joystick_it;
+    }
+    // Destruct SDL_Joystick outside the lock guard because SDL can internally call event calback
+    // which locks the mutex again
+    joystick->SetSDLJoystick(nullptr, [](SDL_Joystick*) {});
+}
+
+void SDLState::HandleGameControllerEvent(const SDL_Event& event) {
+    switch (event.type) {
+    case SDL_JOYBUTTONUP: {
+        if (auto joystick = GetSDLJoystickBySDLID(event.jbutton.which)) {
+            joystick->SetButton(event.jbutton.button, false);
+        }
+        break;
+    }
+    case SDL_JOYBUTTONDOWN: {
+        if (auto joystick = GetSDLJoystickBySDLID(event.jbutton.which)) {
+            joystick->SetButton(event.jbutton.button, true);
+        }
+        break;
+    }
+    case SDL_JOYHATMOTION: {
+        if (auto joystick = GetSDLJoystickBySDLID(event.jhat.which)) {
+            joystick->SetHat(event.jhat.hat, event.jhat.value);
+        }
+        break;
+    }
+    case SDL_JOYAXISMOTION: {
+        if (auto joystick = GetSDLJoystickBySDLID(event.jaxis.which)) {
+            joystick->SetAxis(event.jaxis.axis, event.jaxis.value);
+        }
+        break;
+    }
+    case SDL_JOYDEVICEREMOVED:
+        LOG_DEBUG(Input, "Controller removed with Instance_ID {}", event.jdevice.which);
+        CloseJoystick(SDL_JoystickFromInstanceID(event.jdevice.which));
+        break;
+    case SDL_JOYDEVICEADDED:
+        LOG_DEBUG(Input, "Controller connected with device index {}", event.jdevice.which);
+        InitJoystick(event.jdevice.which);
+        break;
+    }
+}
+
+void SDLState::CloseJoysticks() {
+    std::lock_guard<std::mutex> lock(joystick_map_mutex);
+    joystick_map.clear();
+}
+
+class SDLButton final : public Input::ButtonDevice {
+public:
+    explicit SDLButton(std::shared_ptr<SDLJoystick> joystick_, int button_)
+        : joystick(std::move(joystick_)), button(button_) {}
+
+    bool GetStatus() const override {
+        return joystick->GetButton(button);
+    }
+
+private:
+    std::shared_ptr<SDLJoystick> joystick;
+    int button;
+};
+
+class SDLDirectionButton final : public Input::ButtonDevice {
+public:
+    explicit SDLDirectionButton(std::shared_ptr<SDLJoystick> joystick_, int hat_, Uint8 direction_)
+        : joystick(std::move(joystick_)), hat(hat_), direction(direction_) {}
+
+    bool GetStatus() const override {
+        return joystick->GetHatDirection(hat, direction);
+    }
+
+private:
+    std::shared_ptr<SDLJoystick> joystick;
+    int hat;
+    Uint8 direction;
+};
+
+class SDLAxisButton final : public Input::ButtonDevice {
+public:
+    explicit SDLAxisButton(std::shared_ptr<SDLJoystick> joystick_, int axis_, float threshold_,
+                           bool trigger_if_greater_)
+        : joystick(std::move(joystick_)), axis(axis_), threshold(threshold_),
+          trigger_if_greater(trigger_if_greater_) {}
+
+    bool GetStatus() const override {
+        float axis_value = joystick->GetAxis(axis);
+        if (trigger_if_greater)
+            return axis_value > threshold;
+        return axis_value < threshold;
+    }
+
+private:
+    std::shared_ptr<SDLJoystick> joystick;
+    int axis;
+    float threshold;
+    bool trigger_if_greater;
+};
+
+class SDLAnalog final : public Input::AnalogDevice {
+public:
+    SDLAnalog(std::shared_ptr<SDLJoystick> joystick_, int axis_x_, int axis_y_, float deadzone_)
+        : joystick(std::move(joystick_)), axis_x(axis_x_), axis_y(axis_y_), deadzone(deadzone_) {}
+
+    std::tuple<float, float> GetStatus() const override {
+        const auto [x, y] = joystick->GetAnalog(axis_x, axis_y);
+        const float r = std::sqrt((x * x) + (y * y));
+        if (r > deadzone) {
+            return std::make_tuple(x / r * (r - deadzone) / (1 - deadzone),
+                                   y / r * (r - deadzone) / (1 - deadzone));
+        }
+        return std::make_tuple<float, float>(0.0f, 0.0f);
+    }
+
+private:
+    std::shared_ptr<SDLJoystick> joystick;
+    const int axis_x;
+    const int axis_y;
+    const float deadzone;
+};
+
+/// A button device factory that creates button devices from SDL joystick
+class SDLButtonFactory final : public Input::Factory<Input::ButtonDevice> {
+public:
+    explicit SDLButtonFactory(SDLState& state_) : state(state_) {}
+
+    /**
+     * Creates a button device from a joystick button
+     * @param params contains parameters for creating the device:
+     *     - "guid": the guid of the joystick to bind
+     *     - "port": the nth joystick of the same type to bind
+     *     - "button"(optional): the index of the button to bind
+     *     - "hat"(optional): the index of the hat to bind as direction buttons
+     *     - "axis"(optional): the index of the axis to bind
+     *     - "direction"(only used for hat): the direction name of the hat to bind. Can be "up",
+     *         "down", "left" or "right"
+     *     - "threshold"(only used for axis): a float value in (-1.0, 1.0) which the button is
+     *         triggered if the axis value crosses
+     *     - "direction"(only used for axis): "+" means the button is triggered when the axis
+     * value is greater than the threshold; "-" means the button is triggered when the axis
+     * value is smaller than the threshold
+     */
+    std::unique_ptr<Input::ButtonDevice> Create(const Common::ParamPackage& params) override {
+        const std::string guid = params.Get("guid", "0");
+        const int port = params.Get("port", 0);
+
+        auto joystick = state.GetSDLJoystickByGUID(guid, port);
+
+        if (params.Has("hat")) {
+            const int hat = params.Get("hat", 0);
+            const std::string direction_name = params.Get("direction", "");
+            Uint8 direction;
+            if (direction_name == "up") {
+                direction = SDL_HAT_UP;
+            } else if (direction_name == "down") {
+                direction = SDL_HAT_DOWN;
+            } else if (direction_name == "left") {
+                direction = SDL_HAT_LEFT;
+            } else if (direction_name == "right") {
+                direction = SDL_HAT_RIGHT;
+            } else {
+                direction = 0;
+            }
+            // This is necessary so accessing GetHat with hat won't crash
+            joystick->SetHat(hat, SDL_HAT_CENTERED);
+            return std::make_unique<SDLDirectionButton>(joystick, hat, direction);
+        }
+
+        if (params.Has("axis")) {
+            const int axis = params.Get("axis", 0);
+            const float threshold = params.Get("threshold", 0.5f);
+            const std::string direction_name = params.Get("direction", "");
+            bool trigger_if_greater;
+            if (direction_name == "+") {
+                trigger_if_greater = true;
+            } else if (direction_name == "-") {
+                trigger_if_greater = false;
+            } else {
+                trigger_if_greater = true;
+                LOG_ERROR(Input, "Unknown direction {}", direction_name);
+            }
+            // This is necessary so accessing GetAxis with axis won't crash
+            joystick->SetAxis(axis, 0);
+            return std::make_unique<SDLAxisButton>(joystick, axis, threshold, trigger_if_greater);
+        }
+
+        const int button = params.Get("button", 0);
+        // This is necessary so accessing GetButton with button won't crash
+        joystick->SetButton(button, false);
+        return std::make_unique<SDLButton>(joystick, button);
+    }
+
+private:
+    SDLState& state;
+};
+
+/// An analog device factory that creates analog devices from SDL joystick
+class SDLAnalogFactory final : public Input::Factory<Input::AnalogDevice> {
+public:
+    explicit SDLAnalogFactory(SDLState& state_) : state(state_) {}
+    /**
+     * Creates analog device from joystick axes
+     * @param params contains parameters for creating the device:
+     *     - "guid": the guid of the joystick to bind
+     *     - "port": the nth joystick of the same type
+     *     - "axis_x": the index of the axis to be bind as x-axis
+     *     - "axis_y": the index of the axis to be bind as y-axis
+     */
+    std::unique_ptr<Input::AnalogDevice> Create(const Common::ParamPackage& params) override {
+        const std::string guid = params.Get("guid", "0");
+        const int port = params.Get("port", 0);
+        const int axis_x = params.Get("axis_x", 0);
+        const int axis_y = params.Get("axis_y", 1);
+        float deadzone = std::clamp(params.Get("deadzone", 0.0f), 0.0f, .99f);
+
+        auto joystick = state.GetSDLJoystickByGUID(guid, port);
+
+        // This is necessary so accessing GetAxis with axis_x and axis_y won't crash
+        joystick->SetAxis(axis_x, 0);
+        joystick->SetAxis(axis_y, 0);
+        return std::make_unique<SDLAnalog>(joystick, axis_x, axis_y, deadzone);
+    }
+
+private:
+    SDLState& state;
+};
+
+SDLState::SDLState() {
+    using namespace Input;
+    RegisterFactory<ButtonDevice>("sdl", std::make_shared<SDLButtonFactory>(*this));
+    RegisterFactory<AnalogDevice>("sdl", std::make_shared<SDLAnalogFactory>(*this));
+
+    // If the frontend is going to manage the event loop, then we dont start one here
+    start_thread = !SDL_WasInit(SDL_INIT_JOYSTICK);
+    if (start_thread && SDL_Init(SDL_INIT_JOYSTICK) < 0) {
+        LOG_CRITICAL(Input, "SDL_Init(SDL_INIT_JOYSTICK) failed with: {}", SDL_GetError());
+        return;
+    }
+    if (SDL_SetHint(SDL_HINT_JOYSTICK_ALLOW_BACKGROUND_EVENTS, "1") == SDL_FALSE) {
+        LOG_ERROR(Input, "Failed to set Hint for background events", SDL_GetError());
+    }
+
+    SDL_AddEventWatch(&SDLEventWatcher, this);
+
+    initialized = true;
+    if (start_thread) {
+        poll_thread = std::thread([&] {
+            using namespace std::chrono_literals;
+            SDL_Event event;
+            while (initialized) {
+                SDL_PumpEvents();
+                std::this_thread::sleep_for(std::chrono::duration(10ms));
+            }
+        });
+    }
+    // Because the events for joystick connection happens before we have our event watcher added, we
+    // can just open all the joysticks right here
+    for (int i = 0; i < SDL_NumJoysticks(); ++i) {
+        InitJoystick(i);
+    }
+}
+
+SDLState::~SDLState() {
+    using namespace Input;
+    UnregisterFactory<ButtonDevice>("sdl");
+    UnregisterFactory<AnalogDevice>("sdl");
+
+    CloseJoysticks();
+    SDL_DelEventWatch(&SDLEventWatcher, this);
+
+    initialized = false;
+    if (start_thread) {
+        poll_thread.join();
+        SDL_QuitSubSystem(SDL_INIT_JOYSTICK);
+    }
+}
+
+Common::ParamPackage SDLEventToButtonParamPackage(SDLState& state, const SDL_Event& event) {
+    Common::ParamPackage params({{"engine", "sdl"}});
+
+    switch (event.type) {
+    case SDL_JOYAXISMOTION: {
+        auto joystick = state.GetSDLJoystickBySDLID(event.jaxis.which);
+        params.Set("port", joystick->GetPort());
+        params.Set("guid", joystick->GetGUID());
+        params.Set("axis", event.jaxis.axis);
+        if (event.jaxis.value > 0) {
+            params.Set("direction", "+");
+            params.Set("threshold", "0.5");
+        } else {
+            params.Set("direction", "-");
+            params.Set("threshold", "-0.5");
+        }
+        break;
+    }
+    case SDL_JOYBUTTONUP: {
+        auto joystick = state.GetSDLJoystickBySDLID(event.jbutton.which);
+        params.Set("port", joystick->GetPort());
+        params.Set("guid", joystick->GetGUID());
+        params.Set("button", event.jbutton.button);
+        break;
+    }
+    case SDL_JOYHATMOTION: {
+        auto joystick = state.GetSDLJoystickBySDLID(event.jhat.which);
+        params.Set("port", joystick->GetPort());
+        params.Set("guid", joystick->GetGUID());
+        params.Set("hat", event.jhat.hat);
+        switch (event.jhat.value) {
+        case SDL_HAT_UP:
+            params.Set("direction", "up");
+            break;
+        case SDL_HAT_DOWN:
+            params.Set("direction", "down");
+            break;
+        case SDL_HAT_LEFT:
+            params.Set("direction", "left");
+            break;
+        case SDL_HAT_RIGHT:
+            params.Set("direction", "right");
+            break;
+        default:
+            return {};
+        }
+        break;
+    }
+    }
+    return params;
+}
+
+namespace Polling {
+
+class SDLPoller : public InputCommon::Polling::DevicePoller {
+public:
+    explicit SDLPoller(SDLState& state_) : state(state_) {}
+
+    void Start() override {
+        state.event_queue.Clear();
+        state.polling = true;
+    }
+
+    void Stop() override {
+        state.polling = false;
+    }
+
+protected:
+    SDLState& state;
+};
+
+class SDLButtonPoller final : public SDLPoller {
+public:
+    explicit SDLButtonPoller(SDLState& state_) : SDLPoller(state_) {}
+
+    Common::ParamPackage GetNextInput() override {
+        SDL_Event event;
+        while (state.event_queue.Pop(event)) {
+            switch (event.type) {
+            case SDL_JOYAXISMOTION:
+                if (std::abs(event.jaxis.value / 32767.0) < 0.5) {
+                    break;
+                }
+            case SDL_JOYBUTTONUP:
+            case SDL_JOYHATMOTION:
+                return SDLEventToButtonParamPackage(state, event);
+            }
+        }
+        return {};
+    }
+};
+
+class SDLAnalogPoller final : public SDLPoller {
+public:
+    explicit SDLAnalogPoller(SDLState& state_) : SDLPoller(state_) {}
+
+    void Start() override {
+        SDLPoller::Start();
+
+        // Reset stored axes
+        analog_xaxis = -1;
+        analog_yaxis = -1;
+        analog_axes_joystick = -1;
+    }
+
+    Common::ParamPackage GetNextInput() override {
+        SDL_Event event;
+        while (state.event_queue.Pop(event)) {
+            if (event.type != SDL_JOYAXISMOTION || std::abs(event.jaxis.value / 32767.0) < 0.5) {
+                continue;
+            }
+            // An analog device needs two axes, so we need to store the axis for later and wait for
+            // a second SDL event. The axes also must be from the same joystick.
+            int axis = event.jaxis.axis;
+            if (analog_xaxis == -1) {
+                analog_xaxis = axis;
+                analog_axes_joystick = event.jaxis.which;
+            } else if (analog_yaxis == -1 && analog_xaxis != axis &&
+                       analog_axes_joystick == event.jaxis.which) {
+                analog_yaxis = axis;
+            }
+        }
+        Common::ParamPackage params;
+        if (analog_xaxis != -1 && analog_yaxis != -1) {
+            auto joystick = state.GetSDLJoystickBySDLID(event.jaxis.which);
+            params.Set("engine", "sdl");
+            params.Set("port", joystick->GetPort());
+            params.Set("guid", joystick->GetGUID());
+            params.Set("axis_x", analog_xaxis);
+            params.Set("axis_y", analog_yaxis);
+            analog_xaxis = -1;
+            analog_yaxis = -1;
+            analog_axes_joystick = -1;
+            return params;
+        }
+        return params;
+    }
+
+private:
+    int analog_xaxis = -1;
+    int analog_yaxis = -1;
+    SDL_JoystickID analog_axes_joystick = -1;
+};
+} // namespace Polling
+
+std::vector<std::unique_ptr<InputCommon::Polling::DevicePoller>> SDLState::GetPollers(
+    InputCommon::Polling::DeviceType type) {
+    std::vector<std::unique_ptr<InputCommon::Polling::DevicePoller>> pollers;
+    switch (type) {
+    case InputCommon::Polling::DeviceType::Analog:
+        pollers.emplace_back(std::make_unique<Polling::SDLAnalogPoller>(*this));
+        break;
+    case InputCommon::Polling::DeviceType::Button:
+        pollers.emplace_back(std::make_unique<Polling::SDLButtonPoller>(*this));
+        break;
+        return pollers;
+    }
+}
+
+} // namespace SDL
+} // namespace InputCommon
diff --git a/src/input_common/sdl/sdl_impl.h b/src/input_common/sdl/sdl_impl.h
new file mode 100644
index 0000000000..fec82fbe63
--- /dev/null
+++ b/src/input_common/sdl/sdl_impl.h
@@ -0,0 +1,64 @@
+// Copyright 2018 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <atomic>
+#include <memory>
+#include <thread>
+#include "common/threadsafe_queue.h"
+#include "input_common/sdl/sdl.h"
+
+union SDL_Event;
+using SDL_Joystick = struct _SDL_Joystick;
+using SDL_JoystickID = s32;
+
+namespace InputCommon::SDL {
+
+class SDLJoystick;
+class SDLButtonFactory;
+class SDLAnalogFactory;
+
+class SDLState : public State {
+public:
+    /// Initializes and registers SDL device factories
+    SDLState();
+
+    /// Unresisters SDL device factories and shut them down.
+    ~SDLState() override;
+
+    /// Handle SDL_Events for joysticks from SDL_PollEvent
+    void HandleGameControllerEvent(const SDL_Event& event);
+
+    std::shared_ptr<SDLJoystick> GetSDLJoystickBySDLID(SDL_JoystickID sdl_id);
+    std::shared_ptr<SDLJoystick> GetSDLJoystickByGUID(const std::string& guid, int port);
+
+    /// Get all DevicePoller that use the SDL backend for a specific device type
+    std::vector<std::unique_ptr<InputCommon::Polling::DevicePoller>> GetPollers(
+        InputCommon::Polling::DeviceType type) override;
+
+    /// Used by the Pollers during config
+    std::atomic<bool> polling = false;
+    Common::SPSCQueue<SDL_Event> event_queue;
+
+private:
+    void InitJoystick(int joystick_index);
+    void CloseJoystick(SDL_Joystick* sdl_joystick);
+
+    /// Needs to be called before SDL_QuitSubSystem.
+    void CloseJoysticks();
+
+    /// Map of GUID of a list of corresponding virtual Joysticks
+    std::unordered_map<std::string, std::vector<std::shared_ptr<SDLJoystick>>> joystick_map;
+    std::mutex joystick_map_mutex;
+
+    std::shared_ptr<SDLButtonFactory> button_factory;
+    std::shared_ptr<SDLAnalogFactory> analog_factory;
+
+    bool start_thread = false;
+    std::atomic<bool> initialized = false;
+
+    std::thread poll_thread;
+};
+} // namespace InputCommon::SDL
diff --git a/src/yuzu/bootmanager.cpp b/src/yuzu/bootmanager.cpp
index 3b070bfbb8..d2c97b1f8b 100644
--- a/src/yuzu/bootmanager.cpp
+++ b/src/yuzu/bootmanager.cpp
@@ -123,7 +123,6 @@ GRenderWindow::GRenderWindow(QWidget* parent, EmuThread* emu_thread)
     setAttribute(Qt::WA_AcceptTouchEvents);
 
     InputCommon::Init();
-    InputCommon::StartJoystickEventHandler();
     connect(this, &GRenderWindow::FirstFrameDisplayed, static_cast<GMainWindow*>(parent),
             &GMainWindow::OnLoadComplete);
 }
diff --git a/src/yuzu_cmd/emu_window/emu_window_sdl2.cpp b/src/yuzu_cmd/emu_window/emu_window_sdl2.cpp
index 7df8eff539..de7a26e149 100644
--- a/src/yuzu_cmd/emu_window/emu_window_sdl2.cpp
+++ b/src/yuzu_cmd/emu_window/emu_window_sdl2.cpp
@@ -135,16 +135,16 @@ bool EmuWindow_SDL2::SupportsRequiredGLExtensions() {
 }
 
 EmuWindow_SDL2::EmuWindow_SDL2(bool fullscreen) {
-    InputCommon::Init();
-
-    SDL_SetMainReady();
-
     // Initialize the window
     if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_JOYSTICK) < 0) {
         LOG_CRITICAL(Frontend, "Failed to initialize SDL2! Exiting...");
         exit(1);
     }
 
+    InputCommon::Init();
+
+    SDL_SetMainReady();
+
     SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 4);
     SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 3);
     SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_CORE);
@@ -201,11 +201,9 @@ EmuWindow_SDL2::EmuWindow_SDL2(bool fullscreen) {
 }
 
 EmuWindow_SDL2::~EmuWindow_SDL2() {
-    InputCommon::SDL::CloseSDLJoysticks();
+    InputCommon::Shutdown();
     SDL_GL_DeleteContext(gl_context);
     SDL_Quit();
-
-    InputCommon::Shutdown();
 }
 
 void EmuWindow_SDL2::SwapBuffers() {
@@ -262,7 +260,6 @@ void EmuWindow_SDL2::PollEvents() {
             is_open = false;
             break;
         default:
-            InputCommon::SDL::HandleGameControllerEvent(event);
             break;
         }
     }