diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryView.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryView.kt
index 0ac2a8ac2a..e8b9febbb8 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryView.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryView.kt
@@ -27,6 +27,7 @@ import reactivecircus.flowbinding.recyclerview.scrollStateChanges
 import reactivecircus.flowbinding.swiperefreshlayout.refreshes
 import rx.subscriptions.CompositeSubscription
 import uy.kohesive.injekt.injectLazy
+import java.util.ArrayDeque
 
 /**
  * Fragment containing the library manga for a certain category.
@@ -66,7 +67,7 @@ class LibraryCategoryView @JvmOverloads constructor(context: Context, attrs: Att
      */
     private var subscriptions = CompositeSubscription()
 
-    private var lastClickPosition = -1
+    private var lastClickPositionStack = ArrayDeque(listOf(-1))
 
     fun onCreate(controller: LibraryController, binding: LibraryCategoryBinding) {
         this.controller = controller
@@ -205,7 +206,11 @@ class LibraryCategoryView @JvmOverloads constructor(context: Context, attrs: Att
             }
             is LibrarySelectionEvent.Unselected -> {
                 findAndToggleSelection(event.manga)
-                if (adapter.indexOf(event.manga) != -1) lastClickPosition = -1
+
+                with(adapter.indexOf(event.manga)) {
+                    if (this != -1) lastClickPositionStack.remove(this)
+                }
+
                 if (controller.selectedMangas.isEmpty()) {
                     adapter.mode = SelectableAdapter.Mode.SINGLE
                 }
@@ -213,7 +218,9 @@ class LibraryCategoryView @JvmOverloads constructor(context: Context, attrs: Att
             is LibrarySelectionEvent.Cleared -> {
                 adapter.mode = SelectableAdapter.Mode.SINGLE
                 adapter.clearSelection()
-                lastClickPosition = -1
+
+                lastClickPositionStack.clear()
+                lastClickPositionStack.push(-1)
             }
         }
     }
@@ -241,7 +248,11 @@ class LibraryCategoryView @JvmOverloads constructor(context: Context, attrs: Att
         // If the action mode is created and the position is valid, toggle the selection.
         val item = adapter.getItem(position) ?: return false
         return if (adapter.mode == SelectableAdapter.Mode.MULTI) {
-            lastClickPosition = position
+            if (adapter.isSelected(position)) {
+                lastClickPositionStack.remove(position)
+            } else {
+                lastClickPositionStack.push(position)
+            }
             toggleSelection(position)
             true
         } else {
@@ -257,6 +268,7 @@ class LibraryCategoryView @JvmOverloads constructor(context: Context, attrs: Att
      */
     override fun onItemLongClick(position: Int) {
         controller.createActionModeIfNeeded()
+        val lastClickPosition = lastClickPositionStack.peek()!!
         when {
             lastClickPosition == -1 -> setSelection(position)
             lastClickPosition > position ->
@@ -267,7 +279,10 @@ class LibraryCategoryView @JvmOverloads constructor(context: Context, attrs: Att
                     setSelection(i)
             else -> setSelection(position)
         }
-        lastClickPosition = position
+        if (lastClickPosition != position) {
+            lastClickPositionStack.remove(position)
+            lastClickPositionStack.push(position)
+        }
     }
 
     /**
diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt
index 5d09b136c6..aa8897e4ac 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/manga/MangaController.kt
@@ -82,6 +82,7 @@ import timber.log.Timber
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
 import uy.kohesive.injekt.injectLazy
+import java.util.ArrayDeque
 import kotlin.math.min
 
 class MangaController :
@@ -154,7 +155,7 @@ class MangaController :
 
     private val isLocalSource by lazy { presenter.source.id == LocalSource.ID }
 
-    private var lastClickPosition = -1
+    private var lastClickPositionStack = ArrayDeque(listOf(-1))
 
     private var isRefreshingInfo = false
     private var isRefreshingChapters = false
@@ -727,7 +728,12 @@ class MangaController :
         val adapter = chaptersAdapter ?: return false
         val item = adapter.getItem(position) ?: return false
         return if (actionMode != null && adapter.mode == SelectableAdapter.Mode.MULTI) {
-            lastClickPosition = position
+            if (adapter.isSelected(position)) {
+                lastClickPositionStack.remove(position) // possible that it's not there, but no harm
+            } else {
+                lastClickPositionStack.push(position)
+            }
+
             toggleSelection(position)
             true
         } else {
@@ -738,6 +744,7 @@ class MangaController :
 
     override fun onItemLongClick(position: Int) {
         createActionModeIfNeeded()
+        val lastClickPosition = lastClickPositionStack.peek()!!
         when {
             lastClickPosition == -1 -> setSelection(position)
             lastClickPosition > position ->
@@ -748,7 +755,10 @@ class MangaController :
                     setSelection(i)
             else -> setSelection(position)
         }
-        lastClickPosition = position
+        if (lastClickPosition != position) {
+            lastClickPositionStack.remove(position) // move to top if already exists
+            lastClickPositionStack.push(position)
+        }
         chaptersAdapter?.notifyDataSetChanged()
     }
 
@@ -797,7 +807,8 @@ class MangaController :
     }
 
     private fun destroyActionModeIfNeeded() {
-        lastClickPosition = -1
+        lastClickPositionStack.clear()
+        lastClickPositionStack.push(-1)
         actionMode?.finish()
     }