From 70ed49e4782579d6ce2e91ace75ef44f8460a64a Mon Sep 17 00:00:00 2001 From: stinky-lizard <38111803+stinky-lizard@users.noreply.github.com> Date: Thu, 1 Jul 2021 18:11:21 -0400 Subject: [PATCH] Imported implementation for updating library by next expected update from Neko (#5436) * Imported implementation for updating library by next expected update from Neko. This sort uses the last 4 updates for a manga to compute an average time between updates and then extrapolates when the next update should occur. Currently seems to work perfectly. However, I may have silently messed something up along the way. All code and algorithms are credited to kyjibo on GitHub. The original commit adding this functionality is here: https://github.com/CarlosEsco/Neko/commit/681003926ae1e07b925155d4e1f43972bbe2b843 * Imported implementation for updating library by next expected update from Neko. This sort uses the last 4 updates for a manga to compute an average time between updates and then extrapolates when the next update should occur. Currently seems to work perfectly. However, I may have silently messed something up along the way. All code and algorithms are credited to kyjibo on GitHub. The original commit adding this functionality is here: https://github.com/CarlosEsco/Neko/commit/681003926ae1e07b925155d4e1f43972bbe2b843 * Remove commented-out line from LibraryUpdateRanker I missed removing this when first committing. The removed line is a holdover from Neko, which requires 7+, but I removed the function that requires this. --- .../tachiyomi/data/database/DbOpenCallback.kt | 5 ++- .../data/database/mappers/MangaTypeMapping.kt | 3 ++ .../tachiyomi/data/database/models/Manga.kt | 2 + .../data/database/models/MangaImpl.kt | 2 + .../data/database/queries/MangaQueries.kt | 12 +++--- .../resolvers/MangaNextUpdatedPutResolver.kt | 31 +++++++++++++++ .../data/database/tables/MangaTable.kt | 6 +++ .../data/library/LibraryUpdateRanker.kt | 22 ++++++++++- .../ui/setting/SettingsLibraryController.kt | 3 +- .../util/chapter/ChapterSourceSync.kt | 38 ++++++++++++++++++- app/src/main/res/values/strings.xml | 1 + 11 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 app/src/main/java/eu/kanade/tachiyomi/data/database/resolvers/MangaNextUpdatedPutResolver.kt diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/DbOpenCallback.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/DbOpenCallback.kt index c0841b368a..de2300af46 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/DbOpenCallback.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/DbOpenCallback.kt @@ -20,7 +20,7 @@ class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) { /** * Version of the database. */ - const val DATABASE_VERSION = 11 + const val DATABASE_VERSION = 12 } override fun onCreate(db: SupportSQLiteDatabase) = with(db) { @@ -82,6 +82,9 @@ class DbOpenCallback : SupportSQLiteOpenHelper.Callback(DATABASE_VERSION) { db.execSQL(MangaTable.addDateAdded) db.execSQL(MangaTable.backfillDateAdded) } + if (oldVersion < 12) { + db.execSQL(MangaTable.addNextUpdateCol) + } } override fun onConfigure(db: SupportSQLiteDatabase) { diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/mappers/MangaTypeMapping.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/mappers/MangaTypeMapping.kt index 7392a170ac..4b91e26eb8 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/mappers/MangaTypeMapping.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/mappers/MangaTypeMapping.kt @@ -22,6 +22,7 @@ import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_GENRE import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_ID import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_INITIALIZED import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_LAST_UPDATE +import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_NEXT_UPDATE import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_SOURCE import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_STATUS import eu.kanade.tachiyomi.data.database.tables.MangaTable.COL_THUMBNAIL_URL @@ -62,6 +63,7 @@ class MangaPutResolver : DefaultPutResolver() { COL_THUMBNAIL_URL to obj.thumbnail_url, COL_FAVORITE to obj.favorite, COL_LAST_UPDATE to obj.last_update, + COL_NEXT_UPDATE to obj.next_update, COL_INITIALIZED to obj.initialized, COL_VIEWER to obj.viewer_flags, COL_CHAPTER_FLAGS to obj.chapter_flags, @@ -84,6 +86,7 @@ interface BaseMangaGetResolver { thumbnail_url = cursor.getString(cursor.getColumnIndex(COL_THUMBNAIL_URL)) favorite = cursor.getInt(cursor.getColumnIndex(COL_FAVORITE)) == 1 last_update = cursor.getLong(cursor.getColumnIndex(COL_LAST_UPDATE)) + next_update = cursor.getLong(cursor.getColumnIndex(COL_NEXT_UPDATE)) initialized = cursor.getInt(cursor.getColumnIndex(COL_INITIALIZED)) == 1 viewer_flags = cursor.getInt(cursor.getColumnIndex(COL_VIEWER)) chapter_flags = cursor.getInt(cursor.getColumnIndex(COL_CHAPTER_FLAGS)) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt index f0cfd444fa..95844a36c6 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/models/Manga.kt @@ -15,6 +15,8 @@ interface Manga : SManga { var last_update: Long + var next_update: Long + var date_added: Long var viewer_flags: Int diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/models/MangaImpl.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/models/MangaImpl.kt index 7e27238407..3dc131dcea 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/models/MangaImpl.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/models/MangaImpl.kt @@ -26,6 +26,8 @@ open class MangaImpl : Manga { override var last_update: Long = 0 + override var next_update: Long = 0 + override var date_added: Long = 0 override var initialized: Boolean = false diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/queries/MangaQueries.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/queries/MangaQueries.kt index ad3a96214f..8d2a8a6f2e 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/queries/MangaQueries.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/queries/MangaQueries.kt @@ -6,12 +6,7 @@ import com.pushtorefresh.storio.sqlite.queries.RawQuery import eu.kanade.tachiyomi.data.database.DbProvider import eu.kanade.tachiyomi.data.database.models.LibraryManga import eu.kanade.tachiyomi.data.database.models.Manga -import eu.kanade.tachiyomi.data.database.resolvers.LibraryMangaGetResolver -import eu.kanade.tachiyomi.data.database.resolvers.MangaCoverLastModifiedPutResolver -import eu.kanade.tachiyomi.data.database.resolvers.MangaFavoritePutResolver -import eu.kanade.tachiyomi.data.database.resolvers.MangaFlagsPutResolver -import eu.kanade.tachiyomi.data.database.resolvers.MangaLastUpdatedPutResolver -import eu.kanade.tachiyomi.data.database.resolvers.MangaTitlePutResolver +import eu.kanade.tachiyomi.data.database.resolvers.* import eu.kanade.tachiyomi.data.database.tables.CategoryTable import eu.kanade.tachiyomi.data.database.tables.ChapterTable import eu.kanade.tachiyomi.data.database.tables.MangaCategoryTable @@ -97,6 +92,11 @@ interface MangaQueries : DbProvider { .withPutResolver(MangaFlagsPutResolver(MangaTable.COL_VIEWER, Manga::viewer_flags, true)) .prepare() + fun updateNextUpdated(manga: Manga) = db.put() + .`object`(manga) + .withPutResolver(MangaNextUpdatedPutResolver()) + .prepare() + fun updateLastUpdated(manga: Manga) = db.put() .`object`(manga) .withPutResolver(MangaLastUpdatedPutResolver()) diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/resolvers/MangaNextUpdatedPutResolver.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/resolvers/MangaNextUpdatedPutResolver.kt new file mode 100644 index 0000000000..9ed7924fac --- /dev/null +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/resolvers/MangaNextUpdatedPutResolver.kt @@ -0,0 +1,31 @@ +package eu.kanade.tachiyomi.data.database.resolvers + +import android.content.ContentValues +import com.pushtorefresh.storio.sqlite.StorIOSQLite +import com.pushtorefresh.storio.sqlite.operations.put.PutResolver +import com.pushtorefresh.storio.sqlite.operations.put.PutResult +import com.pushtorefresh.storio.sqlite.queries.UpdateQuery +import eu.kanade.tachiyomi.data.database.inTransactionReturn +import eu.kanade.tachiyomi.data.database.models.Manga +import eu.kanade.tachiyomi.data.database.tables.MangaTable + +class MangaNextUpdatedPutResolver : PutResolver() { + + override fun performPut(db: StorIOSQLite, manga: Manga) = db.inTransactionReturn { + val updateQuery = mapToUpdateQuery(manga) + val contentValues = mapToContentValues(manga) + + val numberOfRowsUpdated = db.lowLevel().update(updateQuery, contentValues) + PutResult.newUpdateResult(numberOfRowsUpdated, updateQuery.table()) + } + + fun mapToUpdateQuery(manga: Manga) = UpdateQuery.builder() + .table(MangaTable.TABLE) + .where("${MangaTable.COL_ID} = ?") + .whereArgs(manga.id) + .build() + + fun mapToContentValues(manga: Manga) = ContentValues(1).apply { + put(MangaTable.COL_NEXT_UPDATE, manga.next_update) + } +} diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/database/tables/MangaTable.kt b/app/src/main/java/eu/kanade/tachiyomi/data/database/tables/MangaTable.kt index bd6f79f4fb..79ad0cd2ef 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/database/tables/MangaTable.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/database/tables/MangaTable.kt @@ -28,6 +28,8 @@ object MangaTable { const val COL_LAST_UPDATE = "last_update" + const val COL_NEXT_UPDATE = "next_update" + const val COL_DATE_ADDED = "date_added" const val COL_INITIALIZED = "initialized" @@ -57,6 +59,7 @@ object MangaTable { $COL_THUMBNAIL_URL TEXT, $COL_FAVORITE INTEGER NOT NULL, $COL_LAST_UPDATE LONG, + $COL_NEXT_UPDATE LONG, $COL_INITIALIZED BOOLEAN NOT NULL, $COL_VIEWER INTEGER NOT NULL, $COL_CHAPTER_FLAGS INTEGER NOT NULL, @@ -86,4 +89,7 @@ object MangaTable { "FROM $TABLE INNER JOIN ${ChapterTable.TABLE} " + "ON $TABLE.$COL_ID = ${ChapterTable.TABLE}.${ChapterTable.COL_MANGA_ID} " + "GROUP BY $TABLE.$COL_ID)" + + val addNextUpdateCol: String + get() = "ALTER TABLE $TABLE ADD COLUMN $COL_NEXT_UPDATE LONG DEFAULT 0" } diff --git a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateRanker.kt b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateRanker.kt index 172e1463e6..ea05374272 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateRanker.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/data/library/LibraryUpdateRanker.kt @@ -1,6 +1,9 @@ package eu.kanade.tachiyomi.data.library import eu.kanade.tachiyomi.data.database.models.Manga +import java.util.Collections +import kotlin.Comparator +import kotlin.math.abs /** * This class will provide various functions to rank manga to efficiently schedule manga to update. @@ -9,9 +12,26 @@ object LibraryUpdateRanker { val rankingScheme = listOf( (this::lexicographicRanking)(), - (this::latestFirstRanking)() + (this::latestFirstRanking)(), + (this::nextFirstRanking)() ) + /** + * Provides a total ordering over all the Mangas. + * + * Orders the manga based on the distance between the next expected update and now. + * The comparator is reversed, placing the smallest (and thus closest to updating now) first. + */ + fun nextFirstRanking(): Comparator { + val time = System.currentTimeMillis() + return Collections.reverseOrder( + Comparator { mangaFirst: Manga, + mangaSecond: Manga -> + compareValues(abs(mangaSecond.next_update - time), abs(mangaFirst.next_update - time)) + } + ) + } + /** * Provides a total ordering over all the [Manga]s. * diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsLibraryController.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsLibraryController.kt index d5f0a15bf6..7f3f358471 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsLibraryController.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/ui/setting/SettingsLibraryController.kt @@ -251,7 +251,8 @@ class SettingsLibraryController : SettingsController() { // ../../data/library/LibraryUpdateRanker.kt val priorities = arrayOf( Pair("0", R.string.action_sort_alpha), - Pair("1", R.string.action_sort_last_checked) + Pair("1", R.string.action_sort_last_checked), + Pair("2", R.string.action_sort_next_updated) ) val defaultPriority = priorities[0] diff --git a/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt b/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt index 5639433704..e229989e0c 100644 --- a/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt +++ b/app/src/main/java/eu/kanade/tachiyomi/util/chapter/ChapterSourceSync.kt @@ -96,6 +96,24 @@ fun syncChaptersWithSource( // Return if there's nothing to add, delete or change, avoiding unnecessary db transactions. if (toAdd.isEmpty() && toDelete.isEmpty() && toChange.isEmpty()) { + val topChapters = dbChapters.sortedByDescending { it.date_upload }.take(4) + val newestDate = topChapters.getOrNull(0)?.date_upload ?: 0L + + // Recalculate update rate if unset and enough chapters are present + if (manga.next_update == 0L && topChapters.size > 1) { + var delta = 0L + for (i in 0 until topChapters.size - 1) { + delta += (topChapters[i].date_upload - topChapters[i + 1].date_upload) + } + delta /= topChapters.size - 1 + manga.next_update = newestDate + delta + db.updateNextUpdated(manga).executeAsBlocking() + } + + if (newestDate != 0L && newestDate != manga.last_update) { + manga.last_update = newestDate + db.updateLastUpdated(manga).executeAsBlocking() + } return Pair(emptyList(), emptyList()) } @@ -140,11 +158,29 @@ fun syncChaptersWithSource( db.insertChapters(toChange).executeAsBlocking() } + val topChapters = db.getChapters(manga).executeAsBlocking().sortedByDescending { it.date_upload }.take(4) + // Recalculate next update since chapters were changed + if (topChapters.size > 1) { + var delta = 0L + for (i in 0 until topChapters.size - 1) { + delta += (topChapters[i].date_upload - topChapters[i + 1].date_upload) + } + delta /= topChapters.size - 1 + manga.next_update = topChapters[0].date_upload + delta + db.updateNextUpdated(manga).executeAsBlocking() + } + // Fix order in source. db.fixChaptersSourceOrder(sourceChapters).executeAsBlocking() // Set this manga as updated since chapters were changed - manga.last_update = Date().time + val newestChapter = topChapters.getOrNull(0) + val dateFetch = newestChapter?.date_upload ?: manga.last_update + if (dateFetch == 0L) { + if (toAdd.isNotEmpty()) { + manga.last_update = Date().time + } + } else manga.last_update = dateFetch db.updateLastUpdated(manga).executeAsBlocking() } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 054e61a4bd..be62e41011 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -40,6 +40,7 @@ Total chapters Last read Last checked + Next expected update Latest chapter Date fetched Date added