Convert downloadChapter to suspend function (#9127)

1:1 translation from the RxJava implementation, should match the
previous behavior.

Dropped the return value from functions of the form
```
fun foo(t: T, ...): Observable<T>
```
where the Observable produced the original argument `t`.
The caller already has the result if necessary.

While this conversion is not flow-based overall, some sections use
flows to use the flatMapMerge and retryWhen operators.

Removed RetryWithDelay as it was only used here.

Inlined fetchAllImageUrlsFromPageList instead of converting it to a
suspending equivalent. fetchAllImageUrlsFromPageList is no longer
used in the app, but was not removed as it is part of source-api.
(However, it does not seem to be used exposed in extensions-lib or
used in tachiyomi-extensions.)

runBlocking is used as a temporary stop-gap.
This commit is contained in:
Two-Ai 2023-02-21 23:02:10 -05:00 committed by GitHub
parent 92bd98e45f
commit fa61c8fe6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 123 additions and 123 deletions

View File

@ -18,13 +18,21 @@ import eu.kanade.tachiyomi.source.SourceManager
import eu.kanade.tachiyomi.source.UnmeteredSource import eu.kanade.tachiyomi.source.UnmeteredSource
import eu.kanade.tachiyomi.source.model.Page import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.online.HttpSource import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.fetchAllImageUrlsFromPageList
import eu.kanade.tachiyomi.util.lang.RetryWithDelay
import eu.kanade.tachiyomi.util.storage.DiskUtil import eu.kanade.tachiyomi.util.storage.DiskUtil
import eu.kanade.tachiyomi.util.storage.DiskUtil.NOMEDIA_FILE import eu.kanade.tachiyomi.util.storage.DiskUtil.NOMEDIA_FILE
import eu.kanade.tachiyomi.util.storage.saveTo import eu.kanade.tachiyomi.util.storage.saveTo
import eu.kanade.tachiyomi.util.system.ImageUtil import eu.kanade.tachiyomi.util.system.ImageUtil
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async import kotlinx.coroutines.async
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.asFlow
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.flatMapMerge
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.flowOn
import kotlinx.coroutines.flow.retryWhen
import kotlinx.coroutines.runBlocking
import logcat.LogPriority import logcat.LogPriority
import nl.adaptivity.xmlutil.serialization.XML import nl.adaptivity.xmlutil.serialization.XML
import okhttp3.Response import okhttp3.Response
@ -33,8 +41,10 @@ import rx.Subscription
import rx.android.schedulers.AndroidSchedulers import rx.android.schedulers.AndroidSchedulers
import rx.schedulers.Schedulers import rx.schedulers.Schedulers
import rx.subscriptions.CompositeSubscription import rx.subscriptions.CompositeSubscription
import tachiyomi.core.util.lang.awaitSingle
import tachiyomi.core.util.lang.launchIO import tachiyomi.core.util.lang.launchIO
import tachiyomi.core.util.lang.launchNow import tachiyomi.core.util.lang.launchNow
import tachiyomi.core.util.lang.withIOContext
import tachiyomi.core.util.lang.withUIContext import tachiyomi.core.util.lang.withUIContext
import tachiyomi.core.util.system.logcat import tachiyomi.core.util.system.logcat
import tachiyomi.domain.chapter.model.Chapter import tachiyomi.domain.chapter.model.Chapter
@ -205,7 +215,10 @@ class Downloader(
.flatMap( .flatMap(
{ bySource -> { bySource ->
bySource.concatMap { download -> bySource.concatMap { download ->
downloadChapter(download).subscribeOn(Schedulers.io()) Observable.fromCallable {
runBlocking { downloadChapter(download) }
download
}.subscribeOn(Schedulers.io())
} }
}, },
5, 5,
@ -298,27 +311,29 @@ class Downloader(
} }
/** /**
* Returns the observable which downloads a chapter. * Downloads a chapter.
* *
* @param download the chapter to be downloaded. * @param download the chapter to be downloaded.
*/ */
private fun downloadChapter(download: Download): Observable<Download> = Observable.defer { private suspend fun downloadChapter(download: Download) {
val mangaDir = provider.getMangaDir(download.manga.title, download.source) val mangaDir = provider.getMangaDir(download.manga.title, download.source)
val availSpace = DiskUtil.getAvailableStorageSpace(mangaDir) val availSpace = DiskUtil.getAvailableStorageSpace(mangaDir)
if (availSpace != -1L && availSpace < MIN_DISK_SPACE) { if (availSpace != -1L && availSpace < MIN_DISK_SPACE) {
download.status = Download.State.ERROR download.status = Download.State.ERROR
notifier.onError(context.getString(R.string.download_insufficient_space), download.chapter.name, download.manga.title) notifier.onError(context.getString(R.string.download_insufficient_space), download.chapter.name, download.manga.title)
return@defer Observable.just(download) return
} }
val chapterDirname = provider.getChapterDirName(download.chapter.name, download.chapter.scanlator) val chapterDirname = provider.getChapterDirName(download.chapter.name, download.chapter.scanlator)
val tmpDir = mangaDir.createDirectory(chapterDirname + TMP_DIR_SUFFIX) val tmpDir = mangaDir.createDirectory(chapterDirname + TMP_DIR_SUFFIX)
val pageListObservable = if (download.pages == null) { try {
// Pull page list from network and add them to download object // If the page list already exists, start from the file
download.source.fetchPageList(download.chapter.toSChapter()) val pageList = download.pages ?: run {
.map { pages -> // Otherwise, pull page list from network and add them to download object
val pages = download.source.getPageList(download.chapter.toSChapter())
if (pages.isEmpty()) { if (pages.isEmpty()) {
throw Exception(context.getString(R.string.page_list_empty_error)) throw Exception(context.getString(R.string.page_list_empty_error))
} }
@ -327,53 +342,60 @@ class Downloader(
download.pages = reIndexedPages download.pages = reIndexedPages
reIndexedPages reIndexedPages
} }
} else {
// Or if the page list already exists, start from the file
Observable.just(download.pages!!)
}
pageListObservable
.doOnNext { _ ->
// Delete all temporary (unfinished) files // Delete all temporary (unfinished) files
tmpDir.listFiles() tmpDir.listFiles()
?.filter { it.name!!.endsWith(".tmp") } ?.filter { it.name!!.endsWith(".tmp") }
?.forEach { it.delete() } ?.forEach { it.delete() }
download.status = Download.State.DOWNLOADING download.status = Download.State.DOWNLOADING
}
// Get all the URLs to the source images, fetch pages if necessary // Get all the URLs to the source images, fetch pages if necessary
.flatMap { download.source.fetchAllImageUrlsFromPageList(it) } pageList.filter { it.imageUrl.isNullOrEmpty() }.forEach { page ->
page.status = Page.State.LOAD_PAGE
try {
page.imageUrl = download.source.fetchImageUrl(page).awaitSingle()
} catch (e: Throwable) {
page.status = Page.State.ERROR
}
}
// Start downloading images, consider we can have downloaded images already // Start downloading images, consider we can have downloaded images already
// Concurrently do 2 pages at a time // Concurrently do 2 pages at a time
.flatMap({ page -> getOrDownloadImage(page, download, tmpDir).subscribeOn(Schedulers.io()) }, 2) pageList.asFlow()
.onBackpressureLatest() .flatMapMerge(concurrency = 2) { page ->
flow {
withIOContext { getOrDownloadImage(page, download, tmpDir) }
emit(page)
}.flowOn(Dispatchers.IO)
}
.collect {
// Do when page is downloaded. // Do when page is downloaded.
.doOnNext { notifier.onProgressChange(download) } notifier.onProgressChange(download)
.toList() }
.map { download }
// Do after download completes // Do after download completes
.doOnNext { ensureSuccessfulDownload(download, mangaDir, tmpDir, chapterDirname) } ensureSuccessfulDownload(download, mangaDir, tmpDir, chapterDirname)
} catch (error: Throwable) {
if (error is CancellationException) throw error
// If the page list threw, it will resume here // If the page list threw, it will resume here
.onErrorReturn { error ->
logcat(LogPriority.ERROR, error) logcat(LogPriority.ERROR, error)
download.status = Download.State.ERROR download.status = Download.State.ERROR
notifier.onError(error.message, download.chapter.name, download.manga.title) notifier.onError(error.message, download.chapter.name, download.manga.title)
download
} }
} }
/** /**
* Returns the observable which gets the image from the filesystem if it exists or downloads it * Gets the image from the filesystem if it exists or downloads it otherwise.
* otherwise.
* *
* @param page the page to download. * @param page the page to download.
* @param download the download of the page. * @param download the download of the page.
* @param tmpDir the temporary directory of the download. * @param tmpDir the temporary directory of the download.
*/ */
private fun getOrDownloadImage(page: Page, download: Download, tmpDir: UniFile): Observable<Page> { private suspend fun getOrDownloadImage(page: Page, download: Download, tmpDir: UniFile) {
// If the image URL is empty, do nothing // If the image URL is empty, do nothing
if (page.imageUrl == null) { if (page.imageUrl == null) {
return Observable.just(page) return
} }
val digitCount = (download.pages?.size ?: 0).toString().length.coerceAtLeast(3) val digitCount = (download.pages?.size ?: 0).toString().length.coerceAtLeast(3)
@ -386,16 +408,15 @@ class Downloader(
// Try to find the image file // Try to find the image file
val imageFile = tmpDir.listFiles()?.firstOrNull { it.name!!.startsWith("$filename.") || it.name!!.startsWith("${filename}__001") } val imageFile = tmpDir.listFiles()?.firstOrNull { it.name!!.startsWith("$filename.") || it.name!!.startsWith("${filename}__001") }
try {
// If the image is already downloaded, do nothing. Otherwise download from network // If the image is already downloaded, do nothing. Otherwise download from network
val pageObservable = when { val file = when {
imageFile != null -> Observable.just(imageFile) imageFile != null -> imageFile
chapterCache.isImageInCache(page.imageUrl!!) -> copyImageFromCache(chapterCache.getImageFile(page.imageUrl!!), tmpDir, filename) chapterCache.isImageInCache(page.imageUrl!!) -> copyImageFromCache(chapterCache.getImageFile(page.imageUrl!!), tmpDir, filename)
else -> downloadImage(page, download.source, tmpDir, filename) else -> downloadImage(page, download.source, tmpDir, filename)
} }
return pageObservable
// When the page is ready, set page path, progress (just in case) and status // When the page is ready, set page path, progress (just in case) and status
.doOnNext { file ->
val success = splitTallImageIfNeeded(page, tmpDir) val success = splitTallImageIfNeeded(page, tmpDir)
if (!success) { if (!success) {
notifier.onError(context.getString(R.string.download_notifier_split_failed), download.chapter.name, download.manga.title) notifier.onError(context.getString(R.string.download_notifier_split_failed), download.chapter.name, download.manga.title)
@ -403,30 +424,28 @@ class Downloader(
page.uri = file.uri page.uri = file.uri
page.progress = 100 page.progress = 100
page.status = Page.State.READY page.status = Page.State.READY
} } catch (e: Throwable) {
.map { page } if (e is CancellationException) throw e
// Mark this page as error and allow to download the remaining // Mark this page as error and allow to download the remaining
.onErrorReturn {
page.progress = 0 page.progress = 0
page.status = Page.State.ERROR page.status = Page.State.ERROR
notifier.onError(it.message, download.chapter.name, download.manga.title) notifier.onError(e.message, download.chapter.name, download.manga.title)
page
} }
} }
/** /**
* Returns the observable which downloads the image from network. * Downloads the image from network to a file in tmpDir.
* *
* @param page the page to download. * @param page the page to download.
* @param source the source of the page. * @param source the source of the page.
* @param tmpDir the temporary directory of the download. * @param tmpDir the temporary directory of the download.
* @param filename the filename of the image. * @param filename the filename of the image.
*/ */
private fun downloadImage(page: Page, source: HttpSource, tmpDir: UniFile, filename: String): Observable<UniFile> { private suspend fun downloadImage(page: Page, source: HttpSource, tmpDir: UniFile, filename: String): UniFile {
page.status = Page.State.DOWNLOAD_IMAGE page.status = Page.State.DOWNLOAD_IMAGE
page.progress = 0 page.progress = 0
return source.fetchImage(page) return flow {
.map { response -> val response = source.getImage(page)
val file = tmpDir.createFile("$filename.tmp") val file = tmpDir.createFile("$filename.tmp")
try { try {
response.body.source().saveTo(file.openOutputStream()) response.body.source().saveTo(file.openOutputStream())
@ -437,32 +456,38 @@ class Downloader(
file.delete() file.delete()
throw e throw e
} }
file emit(file)
} }
// Retry 3 times, waiting 2, 4 and 8 seconds between attempts. // Retry 3 times, waiting 2, 4 and 8 seconds between attempts.
.retryWhen(RetryWithDelay(3, { (2 shl it - 1) * 1000 }, Schedulers.trampoline())) .retryWhen { _, attempt ->
if (attempt < 3) {
delay((2L shl attempt.toInt()) * 1000)
true
} else {
false
}
}
.first()
} }
/** /**
* Return the observable which copies the image from cache. * Copies the image from cache to file in tmpDir.
* *
* @param cacheFile the file from cache. * @param cacheFile the file from cache.
* @param tmpDir the temporary directory of the download. * @param tmpDir the temporary directory of the download.
* @param filename the filename of the image. * @param filename the filename of the image.
*/ */
private fun copyImageFromCache(cacheFile: File, tmpDir: UniFile, filename: String): Observable<UniFile> { private fun copyImageFromCache(cacheFile: File, tmpDir: UniFile, filename: String): UniFile {
return Observable.just(cacheFile).map {
val tmpFile = tmpDir.createFile("$filename.tmp") val tmpFile = tmpDir.createFile("$filename.tmp")
cacheFile.inputStream().use { input -> cacheFile.inputStream().use { input ->
tmpFile.openOutputStream().use { output -> tmpFile.openOutputStream().use { output ->
input.copyTo(output) input.copyTo(output)
} }
} }
val extension = ImageUtil.findImageType(cacheFile.inputStream()) ?: return@map tmpFile val extension = ImageUtil.findImageType(cacheFile.inputStream()) ?: return tmpFile
tmpFile.renameTo("$filename.${extension.extension}") tmpFile.renameTo("$filename.${extension.extension}")
cacheFile.delete() cacheFile.delete()
tmpFile return tmpFile
}
} }
/** /**

View File

@ -1,25 +0,0 @@
package eu.kanade.tachiyomi.util.lang
import rx.Observable
import rx.Scheduler
import rx.functions.Func1
import rx.schedulers.Schedulers
import java.util.concurrent.TimeUnit.MILLISECONDS
class RetryWithDelay(
private val maxRetries: Int = 1,
private val retryStrategy: (Int) -> Int = { 1000 },
private val scheduler: Scheduler = Schedulers.computation(),
) : Func1<Observable<out Throwable>, Observable<*>> {
private var retryCount = 0
override fun call(attempts: Observable<out Throwable>) = attempts.flatMap { error ->
val count = ++retryCount
if (count <= maxRetries) {
Observable.timer(retryStrategy(count).toLong(), MILLISECONDS, scheduler)
} else {
Observable.error(error as Throwable)
}
}
}