DataHearth/tachiyomi
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initialize download index disk cache (#9179)

Browse Source
This commit is contained in:
Ivan Iskandar 2023-03-17 09:18:11 +07:00 committed by GitHub
parent a335b4ee9e
commit 4d3e13b0d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 163 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadCache.kt
View File

@ -1,6 +1,8 @@
package eu.kanade.tachiyomi.data.download package eu.kanade.tachiyomi.data.download
import android.app.Application
import android.content.Context import android.content.Context
import android.net.Uri
import androidx.core.net.toUri import androidx.core.net.toUri
import com.hippo.unifile.UniFile import com.hippo.unifile.UniFile
import eu.kanade.core.util.mapNotNullKeys import eu.kanade.core.util.mapNotNullKeys
@ -14,6 +16,7 @@ import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.delay import kotlinx.coroutines.delay
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.SharingStarted import kotlinx.coroutines.flow.SharingStarted
import kotlinx.coroutines.flow.debounce import kotlinx.coroutines.flow.debounce
@ -23,7 +26,20 @@ import kotlinx.coroutines.flow.onStart
import kotlinx.coroutines.flow.receiveAsFlow import kotlinx.coroutines.flow.receiveAsFlow
import kotlinx.coroutines.flow.shareIn import kotlinx.coroutines.flow.shareIn
import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.flow.stateIn
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.coroutines.withTimeoutOrNull import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.serialization.KSerializer
import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromByteArray
import kotlinx.serialization.descriptors.PrimitiveKind
import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
import kotlinx.serialization.descriptors.SerialDescriptor
import kotlinx.serialization.encodeToByteArray
import kotlinx.serialization.encoding.Decoder
import kotlinx.serialization.encoding.Encoder
import kotlinx.serialization.protobuf.ProtoBuf
import logcat.LogPriority import logcat.LogPriority
import tachiyomi.core.util.lang.launchIO import tachiyomi.core.util.lang.launchIO
import tachiyomi.core.util.lang.launchNonCancellable import tachiyomi.core.util.lang.launchNonCancellable
@ -34,7 +50,7 @@ import tachiyomi.domain.manga.model.Manga
import tachiyomi.domain.source.service.SourceManager import tachiyomi.domain.source.service.SourceManager
import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get import uy.kohesive.injekt.api.get
import java.util.concurrent.ConcurrentHashMap import java.io.File
import kotlin.time.Duration.Companion.hours import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.seconds import kotlin.time.Duration.Companion.seconds
@ -76,7 +92,11 @@ class DownloadCache(
.debounce(1000L) // Don't notify if it finishes quickly enough .debounce(1000L) // Don't notify if it finishes quickly enough
.stateIn(scope, SharingStarted.WhileSubscribed(), false) .stateIn(scope, SharingStarted.WhileSubscribed(), false)
private var rootDownloadsDir = RootDirectory(getDirectoryFromPreference()) private val diskCacheFile: File
get() = File(context.cacheDir, "dl_index_cache")
private val rootDownloadsDirLock = Mutex()
private var rootDownloadsDir: RootDirectory
init { init {
downloadPreferences.downloadsDirectory().changes() downloadPreferences.downloadsDirectory().changes()
@ -85,6 +105,21 @@ class DownloadCache(
invalidateCache() invalidateCache()
} }
.launchIn(scope) .launchIn(scope)
rootDownloadsDir = runBlocking(Dispatchers.IO) {
try {
val diskCache = diskCacheFile.inputStream().use {
ProtoBuf.decodeFromByteArray<RootDirectory>(it.readBytes())
}
lastRenew = 1 // Just so that the banner won't show up
diskCache
} catch (e: Throwable) {
diskCacheFile.delete()
null
}
} ?: RootDirectory(getDirectoryFromPreference())
notifyChanges()
} }
/** /**
@ -158,27 +193,28 @@ class DownloadCache(
* @param mangaUniFile the directory of the manga. * @param mangaUniFile the directory of the manga.
* @param manga the manga of the chapter. * @param manga the manga of the chapter.
*/ */
@Synchronized suspend fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) {
fun addChapter(chapterDirName: String, mangaUniFile: UniFile, manga: Manga) { rootDownloadsDirLock.withLock {
// Retrieve the cached source directory or cache a new one // Retrieve the cached source directory or cache a new one
var sourceDir = rootDownloadsDir.sourceDirs[manga.source] var sourceDir = rootDownloadsDir.sourceDirs[manga.source]
if (sourceDir == null) { if (sourceDir == null) {
val source = sourceManager.get(manga.source) ?: return val source = sourceManager.get(manga.source) ?: return
val sourceUniFile = provider.findSourceDir(source) ?: return val sourceUniFile = provider.findSourceDir(source) ?: return
sourceDir = SourceDirectory(sourceUniFile) sourceDir = SourceDirectory(sourceUniFile)
rootDownloadsDir.sourceDirs += manga.source to sourceDir rootDownloadsDir.sourceDirs += manga.source to sourceDir
} }
// Retrieve the cached manga directory or cache a new one // Retrieve the cached manga directory or cache a new one
val mangaDirName = provider.getMangaDirName(manga.title) val mangaDirName = provider.getMangaDirName(manga.title)
var mangaDir = sourceDir.mangaDirs[mangaDirName] var mangaDir = sourceDir.mangaDirs[mangaDirName]
if (mangaDir == null) { if (mangaDir == null) {
mangaDir = MangaDirectory(mangaUniFile) mangaDir = MangaDirectory(mangaUniFile)
sourceDir.mangaDirs += mangaDirName to mangaDir sourceDir.mangaDirs += mangaDirName to mangaDir
} }
// Save the chapter directory // Save the chapter directory
mangaDir.chapterDirs += chapterDirName mangaDir.chapterDirs += chapterDirName
}
notifyChanges() notifyChanges()
} }
@ -189,13 +225,14 @@ class DownloadCache(
* @param chapter the chapter to remove. * @param chapter the chapter to remove.
* @param manga the manga of the chapter. * @param manga the manga of the chapter.
*/ */
@Synchronized suspend fun removeChapter(chapter: Chapter, manga: Manga) {
fun removeChapter(chapter: Chapter, manga: Manga) { rootDownloadsDirLock.withLock {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach { provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
if (it in mangaDir.chapterDirs) { if (it in mangaDir.chapterDirs) {
mangaDir.chapterDirs -= it mangaDir.chapterDirs -= it
}
} }
} }
@ -208,14 +245,15 @@ class DownloadCache(
* @param chapters the list of chapter to remove. * @param chapters the list of chapter to remove.
* @param manga the manga of the chapter. * @param manga the manga of the chapter.
*/ */
@Synchronized suspend fun removeChapters(chapters: List<Chapter>, manga: Manga) {
fun removeChapters(chapters: List<Chapter>, manga: Manga) { rootDownloadsDirLock.withLock {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return val mangaDir = sourceDir.mangaDirs[provider.getMangaDirName(manga.title)] ?: return
chapters.forEach { chapter -> chapters.forEach { chapter ->
provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach { provider.getValidChapterDirNames(chapter.name, chapter.scanlator).forEach {
if (it in mangaDir.chapterDirs) { if (it in mangaDir.chapterDirs) {
mangaDir.chapterDirs -= it mangaDir.chapterDirs -= it
}
} }
} }
} }
@ -228,20 +266,22 @@ class DownloadCache(
* *
* @param manga the manga to remove. * @param manga the manga to remove.
*/ */
@Synchronized suspend fun removeManga(manga: Manga) {
fun removeManga(manga: Manga) { rootDownloadsDirLock.withLock {
val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return val sourceDir = rootDownloadsDir.sourceDirs[manga.source] ?: return
val mangaDirName = provider.getMangaDirName(manga.title) val mangaDirName = provider.getMangaDirName(manga.title)
if (sourceDir.mangaDirs.containsKey(mangaDirName)) { if (sourceDir.mangaDirs.containsKey(mangaDirName)) {
sourceDir.mangaDirs -= mangaDirName sourceDir.mangaDirs -= mangaDirName
}
} }
notifyChanges() notifyChanges()
} }
@Synchronized suspend fun removeSource(source: Source) {
fun removeSource(source: Source) { rootDownloadsDirLock.withLock {
rootDownloadsDir.sourceDirs -= source.id rootDownloadsDir.sourceDirs -= source.id
}
notifyChanges() notifyChanges()
} }
@ -287,46 +327,48 @@ class DownloadCache(
} }
} }
val sourceDirs = rootDownloadsDir.dir.listFiles().orEmpty() rootDownloadsDirLock.withLock {
.associate { it.name to SourceDirectory(it) } val sourceDirs = rootDownloadsDir.dir.listFiles().orEmpty()
.mapNotNullKeys { entry -> .associate { it.name to SourceDirectory(it) }
sources.find { .mapNotNullKeys { entry ->
provider.getSourceDirName(it).equals(entry.key, ignoreCase = true) sources.find {
}?.id provider.getSourceDirName(it).equals(entry.key, ignoreCase = true)
} }?.id
}
rootDownloadsDir.sourceDirs = sourceDirs rootDownloadsDir.sourceDirs = sourceDirs
sourceDirs.values sourceDirs.values
.map { sourceDir -> .map { sourceDir ->
async { async {
val mangaDirs = sourceDir.dir.listFiles().orEmpty() sourceDir.mangaDirs = sourceDir.dir.listFiles().orEmpty()
.filterNot { it.name.isNullOrBlank() } .filterNot { it.name.isNullOrBlank() }
.associate { it.name!! to MangaDirectory(it) } .associate { it.name!! to MangaDirectory(it) }
sourceDir.mangaDirs = ConcurrentHashMap(mangaDirs) sourceDir.mangaDirs.values.forEach { mangaDir ->
val chapterDirs = mangaDir.dir.listFiles().orEmpty()
mangaDirs.values.forEach { mangaDir -> .mapNotNull {
val chapterDirs = mangaDir.dir.listFiles().orEmpty() when {
.mapNotNull { // Ignore incomplete downloads
when { it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null
// Ignore incomplete downloads // Folder of images
it.name?.endsWith(Downloader.TMP_DIR_SUFFIX) == true -> null it.isDirectory -> it.name
// Folder of images // CBZ files
it.isDirectory -> it.name it.isFile && it.name?.endsWith(".cbz") == true -> it.name!!.substringBeforeLast(
// CBZ files ".cbz",
it.isFile && it.name?.endsWith(".cbz") == true -> it.name!!.substringBeforeLast(".cbz") )
// Anything else is irrelevant // Anything else is irrelevant
else -> null else -> null
}
} }
} .toMutableSet()
.toMutableSet()
mangaDir.chapterDirs = chapterDirs mangaDir.chapterDirs = chapterDirs
}
} }
} }
} .awaitAll()
.awaitAll() }
_isInitializing.emit(false) _isInitializing.emit(false)
}.also { }.also {
@ -335,6 +377,7 @@ class DownloadCache(
logcat(LogPriority.ERROR, exception) { "Failed to create download cache" } logcat(LogPriority.ERROR, exception) { "Failed to create download cache" }
} }
lastRenew = System.currentTimeMillis() lastRenew = System.currentTimeMillis()
notifyChanges() notifyChanges()
} }
} }
@ -351,29 +394,67 @@ class DownloadCache(
scope.launchNonCancellable { scope.launchNonCancellable {
_changes.send(Unit) _changes.send(Unit)
} }
updateDiskCache()
}
private var updateDiskCacheJob: Job? = null
private fun updateDiskCache() {
updateDiskCacheJob?.cancel()
updateDiskCacheJob = scope.launchIO {
delay(1000)
ensureActive()
val bytes = ProtoBuf.encodeToByteArray(rootDownloadsDir)
ensureActive()
try {
diskCacheFile.writeBytes(bytes)
} catch (e: Throwable) {
logcat(
priority = LogPriority.ERROR,
throwable = e,
message = { "Failed to write disk cache file" },
)
}
}
} }
} }
/** /**
* Class to store the files under the root downloads directory. * Class to store the files under the root downloads directory.
*/ */
@Serializable
private class RootDirectory( private class RootDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile, val dir: UniFile,
var sourceDirs: ConcurrentHashMap<Long, SourceDirectory> = ConcurrentHashMap(), var sourceDirs: Map<Long, SourceDirectory> = mapOf(),
) )
/** /**
* Class to store the files under a source directory. * Class to store the files under a source directory.
*/ */
@Serializable
private class SourceDirectory( private class SourceDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile, val dir: UniFile,
var mangaDirs: ConcurrentHashMap<String, MangaDirectory> = ConcurrentHashMap(), var mangaDirs: Map<String, MangaDirectory> = mapOf(),
) )
/** /**
* Class to store the files under a manga directory. * Class to store the files under a manga directory.
*/ */
@Serializable
private class MangaDirectory( private class MangaDirectory(
@Serializable(with = UniFileAsStringSerializer::class)
val dir: UniFile, val dir: UniFile,
var chapterDirs: MutableSet<String> = mutableSetOf(), var chapterDirs: MutableSet<String> = mutableSetOf(),
) )
private object UniFileAsStringSerializer : KSerializer<UniFile> {
override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor("UniFile", PrimitiveKind.STRING)
override fun serialize(encoder: Encoder, value: UniFile) {
return encoder.encodeString(value.uri.toString())
}
override fun deserialize(decoder: Decoder): UniFile {
return UniFile.fromUri(Injekt.get<Application>(), Uri.parse(decoder.decodeString()))
}
}

2
app/src/main/java/eu/kanade/tachiyomi/data/download/DownloadManager.kt
View File

@ -325,7 +325,7 @@ class DownloadManager(
* @param oldChapter the existing chapter with the old name. * @param oldChapter the existing chapter with the old name.
* @param newChapter the target chapter with the new name. * @param newChapter the target chapter with the new name.
*/ */
fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) { suspend fun renameChapter(source: Source, manga: Manga, oldChapter: Chapter, newChapter: Chapter) {
val oldNames = provider.getValidChapterDirNames(oldChapter.name, oldChapter.scanlator) val oldNames = provider.getValidChapterDirNames(oldChapter.name, oldChapter.scanlator)
val mangaDir = provider.getMangaDir(manga.title, source) val mangaDir = provider.getMangaDir(manga.title, source)

2
app/src/main/java/eu/kanade/tachiyomi/data/download/Downloader.kt
View File

@ -527,7 +527,7 @@ class Downloader(
* @param tmpDir the directory where the download is currently stored. * @param tmpDir the directory where the download is currently stored.
* @param dirname the real (non temporary) directory name of the download. * @param dirname the real (non temporary) directory name of the download.
*/ */
private fun ensureSuccessfulDownload( private suspend fun ensureSuccessfulDownload(
download: Download, download: Download,
mangaDir: UniFile, mangaDir: UniFile,
tmpDir: UniFile, tmpDir: UniFile,