diff --git a/src/yuzu/configuration/configure_input_player.cpp b/src/yuzu/configuration/configure_input_player.cpp
index bff90a82e3..d3980eb493 100644
--- a/src/yuzu/configuration/configure_input_player.cpp
+++ b/src/yuzu/configuration/configure_input_player.cpp
@@ -238,12 +238,11 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
     setFocusPolicy(Qt::ClickFocus);
 
     button_map = {
-        ui->buttonA,         ui->buttonB,         ui->buttonX,         ui->buttonY,
-        ui->buttonLStick,    ui->buttonRStick,    ui->buttonL,         ui->buttonR,
-        ui->buttonZL,        ui->buttonZR,        ui->buttonPlus,      ui->buttonMinus,
-        ui->buttonDpadLeft,  ui->buttonDpadUp,    ui->buttonDpadRight, ui->buttonDpadDown,
-        ui->buttonSL,        ui->buttonSR,        ui->buttonHome,      ui->buttonScreenshot,
-        ui->buttonLStickMod, ui->buttonRStickMod,
+        ui->buttonA,        ui->buttonB,      ui->buttonX,         ui->buttonY,
+        ui->buttonLStick,   ui->buttonRStick, ui->buttonL,         ui->buttonR,
+        ui->buttonZL,       ui->buttonZR,     ui->buttonPlus,      ui->buttonMinus,
+        ui->buttonDpadLeft, ui->buttonDpadUp, ui->buttonDpadRight, ui->buttonDpadDown,
+        ui->buttonSL,       ui->buttonSR,     ui->buttonHome,      ui->buttonScreenshot,
     };
 
     analog_map_buttons = {{
@@ -264,7 +263,6 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
     analog_map_deadzone_label = {ui->labelLStickDeadzone, ui->labelRStickDeadzone};
     analog_map_deadzone_slider = {ui->sliderLStickDeadzone, ui->sliderRStickDeadzone};
     analog_map_modifier_groupbox = {ui->buttonLStickModGroup, ui->buttonRStickModGroup};
-    analog_map_modifier_button = {ui->buttonLStickMod, ui->buttonRStickMod};
     analog_map_modifier_label = {ui->labelLStickModifierRange, ui->labelRStickModifierRange};
     analog_map_modifier_slider = {ui->sliderLStickModifierRange, ui->sliderRStickModifierRange};
     analog_map_range_groupbox = {ui->buttonLStickRangeGroup, ui->buttonRStickRangeGroup};
@@ -327,15 +325,6 @@ ConfigureInputPlayer::ConfigureInputPlayer(QWidget* parent, std::size_t player_i
             });
         }
 
-        connect(analog_map_modifier_button[analog_id], &QPushButton::clicked, [=, this] {
-            HandleClick(
-                analog_map_modifier_button[analog_id],
-                [=, this](const Common::ParamPackage& params) {
-                    SetAnalogParam(params, analogs_param[analog_id], "modifier");
-                },
-                InputCommon::Polling::DeviceType::AnalogPreferred);
-        });
-
         connect(analog_map_range_spinbox[analog_id], qOverload<int>(&QSpinBox::valueChanged),
                 [=, this] {
                     const auto spinbox_value = analog_map_range_spinbox[analog_id]->value();
@@ -538,6 +527,13 @@ void ConfigureInputPlayer::RestoreDefaults() {
         buttons_param[button_id] = Common::ParamPackage{
             InputCommon::GenerateKeyboardParam(Config::default_buttons[button_id])};
     }
+
+    // Reset Modifier Buttons
+    lstick_mod =
+        Common::ParamPackage(InputCommon::GenerateKeyboardParam(Config::default_lstick_mod));
+    rstick_mod =
+        Common::ParamPackage(InputCommon::GenerateKeyboardParam(Config::default_rstick_mod));
+
     // Reset Analogs
     for (int analog_id = 0; analog_id < Settings::NativeAnalog::NumAnalogs; ++analog_id) {
         for (int sub_button_id = 0; sub_button_id < ANALOG_SUB_BUTTONS_NUM; ++sub_button_id) {
@@ -561,6 +557,9 @@ void ConfigureInputPlayer::ClearAll() {
         buttons_param[button_id].Clear();
     }
 
+    lstick_mod.Clear();
+    rstick_mod.Clear();
+
     for (int analog_id = 0; analog_id < Settings::NativeAnalog::NumAnalogs; ++analog_id) {
         for (int sub_button_id = 0; sub_button_id < ANALOG_SUB_BUTTONS_NUM; ++sub_button_id) {
             const auto* const analog_button = analog_map_buttons[analog_id][sub_button_id];
@@ -581,6 +580,9 @@ void ConfigureInputPlayer::UpdateUI() {
         button_map[button]->setText(ButtonToText(buttons_param[button]));
     }
 
+    ui->buttonLStickMod->setText(ButtonToText(lstick_mod));
+    ui->buttonRStickMod->setText(ButtonToText(rstick_mod));
+
     for (int analog_id = 0; analog_id < Settings::NativeAnalog::NumAnalogs; ++analog_id) {
         for (int sub_button_id = 0; sub_button_id < ANALOG_SUB_BUTTONS_NUM; ++sub_button_id) {
             auto* const analog_button = analog_map_buttons[analog_id][sub_button_id];
diff --git a/src/yuzu/configuration/configure_input_player.h b/src/yuzu/configuration/configure_input_player.h
index ca189019df..25d4cde5ed 100644
--- a/src/yuzu/configuration/configure_input_player.h
+++ b/src/yuzu/configuration/configure_input_player.h
@@ -124,11 +124,9 @@ private:
     std::array<Common::ParamPackage, Settings::NativeAnalog::NumAnalogs> analogs_param;
 
     static constexpr int ANALOG_SUB_BUTTONS_NUM = 4;
-    // Adds room for two extra push buttons LStick Modifier and RStick Modifier.
-    static constexpr int BUTTON_MAP_COUNT = Settings::NativeButton::NumButtons + 2;
 
     /// Each button input is represented by a QPushButton.
-    std::array<QPushButton*, BUTTON_MAP_COUNT> button_map;
+    std::array<QPushButton*, Settings::NativeButton::NumButtons> button_map;
     /// Extra buttons for the modifiers.
     Common::ParamPackage lstick_mod;
     Common::ParamPackage rstick_mod;
@@ -141,7 +139,6 @@ private:
     std::array<QLabel*, Settings::NativeAnalog::NumAnalogs> analog_map_deadzone_label;
     std::array<QSlider*, Settings::NativeAnalog::NumAnalogs> analog_map_deadzone_slider;
     std::array<QGroupBox*, Settings::NativeAnalog::NumAnalogs> analog_map_modifier_groupbox;
-    std::array<QPushButton*, Settings::NativeAnalog::NumAnalogs> analog_map_modifier_button;
     std::array<QLabel*, Settings::NativeAnalog::NumAnalogs> analog_map_modifier_label;
     std::array<QSlider*, Settings::NativeAnalog::NumAnalogs> analog_map_modifier_slider;
     std::array<QGroupBox*, Settings::NativeAnalog::NumAnalogs> analog_map_range_groupbox;