From a78359e4a9ebdf8a90cf3d22f6e771b362fe3cf1 Mon Sep 17 00:00:00 2001 From: inorichi Date: Tue, 6 Oct 2015 00:24:29 +0200 Subject: [PATCH] Download chapter images --- app/build.gradle | 3 + .../eu/kanade/mangafeed/data/DataModule.java | 14 + .../mangafeed/data/caches/CacheManager.java | 223 ++++++ .../mangafeed/data/helpers/NetworkHelper.java | 62 ++ .../eu/kanade/mangafeed/sources/Batoto.java | 667 ++++++++++++++++++ .../eu/kanade/mangafeed/util/DiskUtils.java | 160 +++++ .../java/eu/kanade/mangafeed/BatotoTest.java | 40 ++ .../eu/kanade/mangafeed/DataManagerTest.java | 70 -- .../kanade/mangafeed/DatabaseHelperTest.java | 65 -- .../kanade/mangafeed/LibraryFragmentTest.java | 31 - 10 files changed, 1169 insertions(+), 166 deletions(-) create mode 100644 app/src/main/java/eu/kanade/mangafeed/data/caches/CacheManager.java create mode 100644 app/src/main/java/eu/kanade/mangafeed/data/helpers/NetworkHelper.java create mode 100644 app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java create mode 100644 app/src/main/java/eu/kanade/mangafeed/util/DiskUtils.java create mode 100644 app/src/test/java/eu/kanade/mangafeed/BatotoTest.java delete mode 100644 app/src/test/java/eu/kanade/mangafeed/DataManagerTest.java delete mode 100644 app/src/test/java/eu/kanade/mangafeed/DatabaseHelperTest.java delete mode 100644 app/src/test/java/eu/kanade/mangafeed/LibraryFragmentTest.java diff --git a/app/build.gradle b/app/build.gradle index 492781a296..211d2bc0ad 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -59,6 +59,9 @@ dependencies { compile "com.android.support:support-annotations:$SUPPORT_LIBRARY_VERSION" compile 'com.squareup.okhttp:okhttp-urlconnection:2.4.0' compile 'com.squareup.okhttp:okhttp:2.4.0' + compile 'com.squareup.okio:okio:1.6.0' + compile 'com.jakewharton:disklrucache:2.0.2' + compile 'org.jsoup:jsoup:1.8.3' compile 'io.reactivex:rxandroid:1.0.1' compile "com.pushtorefresh.storio:sqlite:$STORIO_VERSION" compile "com.pushtorefresh.storio:sqlite-annotations:$STORIO_VERSION" diff --git a/app/src/main/java/eu/kanade/mangafeed/data/DataModule.java b/app/src/main/java/eu/kanade/mangafeed/data/DataModule.java index 1f1ab48977..acd5bb8e2c 100644 --- a/app/src/main/java/eu/kanade/mangafeed/data/DataModule.java +++ b/app/src/main/java/eu/kanade/mangafeed/data/DataModule.java @@ -6,7 +6,9 @@ import javax.inject.Singleton; import dagger.Module; import dagger.Provides; +import eu.kanade.mangafeed.data.caches.CacheManager; import eu.kanade.mangafeed.data.helpers.DatabaseHelper; +import eu.kanade.mangafeed.data.helpers.NetworkHelper; import eu.kanade.mangafeed.data.helpers.PreferencesHelper; import rx.Scheduler; import rx.schedulers.Schedulers; @@ -35,4 +37,16 @@ public class DataModule { return Schedulers.io(); } + @Provides + @Singleton + CacheManager provideCacheManager(Application app) { + return new CacheManager(app); + } + + @Provides + @Singleton + NetworkHelper provideNetworkHelper() { + return new NetworkHelper(); + } + } \ No newline at end of file diff --git a/app/src/main/java/eu/kanade/mangafeed/data/caches/CacheManager.java b/app/src/main/java/eu/kanade/mangafeed/data/caches/CacheManager.java new file mode 100644 index 0000000000..ef28865ec8 --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/data/caches/CacheManager.java @@ -0,0 +1,223 @@ +package eu.kanade.mangafeed.data.caches; + +import android.content.Context; + +import com.bumptech.glide.Glide; +import com.bumptech.glide.request.FutureTarget; +import com.bumptech.glide.request.target.Target; +import com.jakewharton.disklrucache.DiskLruCache; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import eu.kanade.mangafeed.util.DiskUtils; +import rx.Observable; +import rx.Subscriber; +import rx.functions.Action0; + +public class CacheManager { + + private static final String PARAMETER_CACHE_DIRECTORY = "chapter_disk_cache"; + private static final int PARAMETER_APP_VERSION = 1; + private static final int PARAMETER_VALUE_COUNT = 1; + private static final long PARAMETER_CACHE_SIZE = 10 * 1024 * 1024; + private static final int READ_TIMEOUT = 60; + + private Context mContext; + + private DiskLruCache mDiskCache; + + public CacheManager(Context context) { + mContext = context; + + try { + mDiskCache = DiskLruCache.open( + new File(context.getCacheDir(), PARAMETER_CACHE_DIRECTORY), + PARAMETER_APP_VERSION, + PARAMETER_VALUE_COUNT, + PARAMETER_CACHE_SIZE + ); + } catch (IOException e) { + // Do Nothing. + } + } + + public Observable cacheImagesFromUrls(final List imageUrls) { + return Observable.create(new Observable.OnSubscribe() { + @Override + public void call(Subscriber subscriber) { + try { + for (String imageUrl : imageUrls) { + if (!subscriber.isUnsubscribed()) { + subscriber.onNext(cacheImageFromUrl(imageUrl)); + } + } + subscriber.onCompleted(); + } catch (Throwable e) { + subscriber.onError(e); + } + } + }); + } + + private File cacheImageFromUrl(String imageUrl) throws InterruptedException, ExecutionException, TimeoutException { + FutureTarget cacheFutureTarget = Glide.with(mContext) + .load(imageUrl) + .downloadOnly(Target.SIZE_ORIGINAL, Target.SIZE_ORIGINAL); + + return cacheFutureTarget.get(READ_TIMEOUT, TimeUnit.SECONDS); + } + + public Observable clearImageCache() { + return Observable.create(new Observable.OnSubscribe() { + @Override + public void call(Subscriber subscriber) { + try { + subscriber.onNext(clearImageCacheImpl()); + subscriber.onCompleted(); + } catch (Throwable e) { + subscriber.onError(e); + } + } + }); + } + + private boolean clearImageCacheImpl() { + boolean isSuccessful = true; + + File imageCacheDirectory = Glide.getPhotoCacheDir(mContext); + if (imageCacheDirectory.isDirectory()) { + for (File cachedFile : imageCacheDirectory.listFiles()) { + if (!cachedFile.delete()) { + isSuccessful = false; + } + } + } else { + isSuccessful = false; + } + + File urlCacheDirectory = getCacheDir(); + if (urlCacheDirectory.isDirectory()) { + for (File cachedFile : urlCacheDirectory.listFiles()) { + if (!cachedFile.delete()) { + isSuccessful = false; + } + } + } else { + isSuccessful = false; + } + + return isSuccessful; + } + + public Observable getImageUrlsFromDiskCache(final String chapterUrl) { + return Observable.create(new Observable.OnSubscribe() { + @Override + public void call(Subscriber subscriber) { + try { + String[] imageUrls = getImageUrlsFromDiskCacheImpl(chapterUrl); + + for (String imageUrl : imageUrls) { + if (!subscriber.isUnsubscribed()) { + subscriber.onNext(imageUrl); + } + } + subscriber.onCompleted(); + } catch (Throwable e) { + subscriber.onError(e); + } + } + }); + } + + private String[] getImageUrlsFromDiskCacheImpl(String chapterUrl) throws IOException { + DiskLruCache.Snapshot snapshot = null; + + try { + String key = DiskUtils.hashKeyForDisk(chapterUrl); + + snapshot = mDiskCache.get(key); + + String joinedImageUrls = snapshot.getString(0); + return joinedImageUrls.split(","); + } finally { + if (snapshot != null) { + snapshot.close(); + } + } + } + + public Action0 putImageUrlsToDiskCache(final String chapterUrl, final List imageUrls) { + return new Action0() { + @Override + public void call() { + try { + putImageUrlsToDiskCacheImpl(chapterUrl, imageUrls); + } catch (IOException e) { + // Do Nothing. + } + } + }; + } + + private void putImageUrlsToDiskCacheImpl(String chapterUrl, List imageUrls) throws IOException { + String cachedValue = joinImageUrlsToCacheValue(imageUrls); + + DiskLruCache.Editor editor = null; + OutputStream outputStream = null; + try { + String key = DiskUtils.hashKeyForDisk(chapterUrl); + editor = mDiskCache.edit(key); + if (editor == null) { + return; + } + + outputStream = new BufferedOutputStream(editor.newOutputStream(0)); + outputStream.write(cachedValue.getBytes()); + outputStream.flush(); + + mDiskCache.flush(); + editor.commit(); + } finally { + if (editor != null) { + try { + editor.abort(); + } catch (IOException ignore) { + // Do Nothing. + } + } + if (outputStream != null) { + try { + outputStream.close(); + } catch (IOException ignore) { + // Do Nothing. + } + } + } + } + + private String joinImageUrlsToCacheValue(List imageUrls) { + StringBuilder stringBuilder = new StringBuilder(); + for (int index = 0; index < imageUrls.size(); index++) { + if (index == 0) { + stringBuilder.append(imageUrls.get(index)); + } else { + stringBuilder.append(","); + stringBuilder.append(imageUrls.get(index)); + } + } + + return stringBuilder.toString(); + } + + public File getCacheDir() { + return mDiskCache.getDirectory(); + } +} + diff --git a/app/src/main/java/eu/kanade/mangafeed/data/helpers/NetworkHelper.java b/app/src/main/java/eu/kanade/mangafeed/data/helpers/NetworkHelper.java new file mode 100644 index 0000000000..4e6390fb71 --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/data/helpers/NetworkHelper.java @@ -0,0 +1,62 @@ +package eu.kanade.mangafeed.data.helpers; + + +import com.squareup.okhttp.CacheControl; +import com.squareup.okhttp.Headers; +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; + +import java.io.IOException; + +import rx.Observable; +import rx.Subscriber; +import timber.log.Timber; + +public final class NetworkHelper { + + private OkHttpClient mClient; + + public final CacheControl NULL_CACHE_CONTROL = new CacheControl.Builder().noCache().build(); + public final Headers NULL_HEADERS = new Headers.Builder().build(); + + public NetworkHelper() { + mClient = new OkHttpClient(); + } + + public Observable getResponse(final String url, final CacheControl cacheControl, final Headers headers) { + return Observable.create(subscriber -> { + try { + if (!subscriber.isUnsubscribed()) { + Request request = new Request.Builder() + .url(url) + .cacheControl(cacheControl != null ? cacheControl : NULL_CACHE_CONTROL) + .headers(headers != null ? headers : NULL_HEADERS) + .build(); + subscriber.onNext(mClient.newCall(request).execute()); + } + subscriber.onCompleted(); + } catch (Throwable e) { + subscriber.onError(e); + } + }); + } + + public Observable mapResponseToString(final Response response) { + return Observable.create(subscriber -> { + try { + subscriber.onNext(response.body().string()); + subscriber.onCompleted(); + } catch (Throwable e) { + subscriber.onError(e); + } + }); + } + + public Observable getStringResponse(final String url, final CacheControl cacheControl, final Headers headers) { + + return getResponse(url, cacheControl, headers) + .flatMap(this::mapResponseToString); + } + +} diff --git a/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java b/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java new file mode 100644 index 0000000000..ead3da5523 --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java @@ -0,0 +1,667 @@ +package eu.kanade.mangafeed.sources; + +import com.squareup.okhttp.Headers; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import eu.kanade.mangafeed.data.caches.CacheManager; +import eu.kanade.mangafeed.data.helpers.NetworkHelper; +import rx.Observable; +import rx.schedulers.Schedulers; +import timber.log.Timber; + +public class Batoto { + + public static final String NAME = "Batoto (EN)"; + public static final String BASE_URL = "www.bato.to"; + public static final String INITIAL_UPDATE_URL = "http://bato.to/search_ajax?order_cond=update&order=desc&p=1"; + + private static final Headers REQUEST_HEADERS = constructRequestHeaders(); + private static Headers constructRequestHeaders() { + Headers.Builder headerBuilder = new Headers.Builder(); + headerBuilder.add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)"); + headerBuilder.add("Cookie", "lang_option=English"); + + return headerBuilder.build(); + } + + private NetworkHelper mNetworkService; + private CacheManager mCacheManager; + + public Batoto(NetworkHelper networkService, CacheManager cacheManager) { + mNetworkService = networkService; + mCacheManager = cacheManager; + } + + public Observable getName() { + return Observable.just(NAME); + } + + public Observable getBaseUrl() { + return Observable.just(BASE_URL); + } + + public Observable getInitialUpdateUrl() { + return Observable.just(INITIAL_UPDATE_URL); + } + + public Observable> getGenres() { + List genres = new ArrayList(38); + + genres.add("4-Koma"); + genres.add("Action"); + genres.add("Adventure"); + genres.add("Award Winning"); + genres.add("Comedy"); + genres.add("Cooking"); + genres.add("Doujinshi"); + genres.add("Drama"); + genres.add("Ecchi"); + genres.add("Fantasy"); + genres.add("Gender Bender"); + genres.add("Harem"); + genres.add("Historical"); + genres.add("Horror"); + genres.add("Josei"); + genres.add("Martial Arts"); + genres.add("Mecha"); + genres.add("Medical"); + genres.add("Music"); + genres.add("Mystery"); + genres.add("One Shot"); + genres.add("Psychological"); + genres.add("Romance"); + genres.add("School Life"); + genres.add("Sci-fi"); + genres.add("Seinen"); + genres.add("Shoujo"); + genres.add("Shoujo Ai"); + genres.add("Shounen"); + genres.add("Shounen Ai"); + genres.add("Slice of Life"); + genres.add("Smut"); + genres.add("Sports"); + genres.add("Supernatural"); + genres.add("Tragedy"); + genres.add("Webtoon"); + genres.add("Yaoi"); + genres.add("Yuri"); + + return Observable.just(genres); + } + + /* + public Observable pullLatestUpdatesFromNetwork(final UpdatePageMarker newUpdate) { + return mNetworkService + .getResponse(newUpdate.getNextPageUrl(), NetworkModule.NULL_CACHE_CONTROL, REQUEST_HEADERS) + .flatMap(new Func1>() { + @Override + public Observable call(Response response) { + return mNetworkService.mapResponseToString(response); + } + }) + .flatMap(new Func1>() { + @Override + public Observable call(String unparsedHtml) { + return Observable.just(parseHtmlToLatestUpdates(newUpdate.getNextPageUrl(), unparsedHtml)); + } + }); + } + + private UpdatePageMarker parseHtmlToLatestUpdates(String requestUrl, String unparsedHtml) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + List updatedMangaList = scrapeUpdateMangasFromParsedDocument(parsedDocument); + updateLibraryInDatabase(updatedMangaList); + + String nextPageUrl = findNextUrlFromParsedDocument(requestUrl, unparsedHtml); + int lastMangaPostion = updatedMangaList.size(); + + return new UpdatePageMarker(nextPageUrl, lastMangaPostion); + } + + private List scrapeUpdateMangasFromParsedDocument(Document parsedDocument) { + List updatedMangaList = new ArrayList(); + + Elements updatedHtmlBlocks = parsedDocument.select("tr:not([id]):not([class])"); + for (Element currentHtmlBlock : updatedHtmlBlocks) { + Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock); + + updatedMangaList.add(currentlyUpdatedManga); + } + + return updatedMangaList; + } + + private Manga constructMangaFromHtmlBlock(Element htmlBlock) { + Manga mangaFromHtmlBlock = DefaultFactory.Manga.constructDefault(); + mangaFromHtmlBlock.setSource(NAME); + + Element urlElement = htmlBlock.select("a[href^=http://bato.to]").first(); + Element nameElement = urlElement; + Element updateElement = htmlBlock.select("td").get(5); + + if (urlElement != null) { + String fieldUrl = urlElement.attr("href"); + mangaFromHtmlBlock.setUrl(fieldUrl); + } + if (nameElement != null) { + String fieldName = nameElement.text().trim(); + mangaFromHtmlBlock.setName(fieldName); + } + if (updateElement != null) { + long fieldUpdate = parseUpdateFromElement(updateElement); + mangaFromHtmlBlock.setUpdated(fieldUpdate); + } + + int updateCount = 1; + mangaFromHtmlBlock.setUpdateCount(updateCount); + + return mangaFromHtmlBlock; + } + + private long parseUpdateFromElement(Element updateElement) { + String updatedDateAsString = updateElement.text(); + + try { + Date specificDate = new SimpleDateFormat("dd MMMMM yyyy - hh:mm a", Locale.ENGLISH).parse(updatedDateAsString); + + return specificDate.getTime(); + } catch (ParseException e) { + // Do Nothing. + } + + return DefaultFactory.Manga.DEFAULT_UPDATED; + } + + private void updateLibraryInDatabase(List mangaList) { + mQueryManager.beginLibraryTransaction(); + try { + List mangaToRemove = new ArrayList<>(); + for (Manga currentManga : mangaList) { + Manga existingManga = mQueryManager.retrieveManga(NAME, currentManga.getName()) + .toBlocking() + .single(); + + if (existingManga != null) { + existingManga.setUpdated(currentManga.getUpdated()); + existingManga.setUpdateCount(currentManga.getUpdateCount()); + + + mQueryManager.createManga(existingManga) + .toBlocking() + .single(); + } else { + mangaToRemove.add(currentManga); + } + } + mangaList.removeAll(mangaToRemove); + + mQueryManager.setLibraryTransactionSuccessful(); + } finally { + mQueryManager.endLibraryTransaction(); + } + } + + private String findNextUrlFromParsedDocument(String requestUrl, String unparsedHtml) { + if (!unparsedHtml.contains("No (more) comics found!")) { + requestUrl = requestUrl.replace("http://bato.to/search_ajax?order_cond=update&order=desc&p=", ""); + + return "http://bato.to/search_ajax?order_cond=update&order=desc&p=" + (Integer.valueOf(requestUrl) + 1); + } + + return DefaultFactory.UpdatePageMarker.DEFAULT_NEXT_PAGE_URL; + } + + public Observable pullMangaFromNetwork(final String mangaUrl) { + String mangaId = mangaUrl.substring(mangaUrl.lastIndexOf("r") + 1); + + return mNetworkService + .getResponse("http://bato.to/comic_pop?id=" + mangaId, NetworkModule.NULL_CACHE_CONTROL, REQUEST_HEADERS) + .flatMap(new Func1>() { + @Override + public Observable call(Response response) { + return mNetworkService.mapResponseToString(response); + } + }) + .flatMap(new Func1>() { + @Override + public Observable call(String unparsedHtml) { + return Observable.just(parseHtmlToManga(mangaUrl, unparsedHtml)); + } + }); + } + + private Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + Element artistElement = parsedDocument.select("a[href^=http://bato.to/search?artist_name]").first(); + Element descriptionElement = parsedDocument.select("tr").get(5); + Elements genreElements = parsedDocument.select("img[src=http://bato.to/forums/public/style_images/master/bullet_black.png]"); + Element thumbnailUrlElement = parsedDocument.select("img[src^=http://img.batoto.net/forums/uploads/]").first(); + + StringBuilder selection = new StringBuilder(); + List selectionArgs = new ArrayList(); + + selection.append(LibraryContract.Manga.COLUMN_SOURCE + " = ?"); + selectionArgs.add(NAME); + selection.append(" AND ").append(LibraryContract.Manga.COLUMN_URL + " = ?"); + selectionArgs.add(mangaUrl); + + Manga newManga = mQueryManager.retrieveMangaAsCursor( + null, + selection.toString(), + selectionArgs.toArray(new String[selectionArgs.size()]), + null, + null, + null, + "1" + ) + .map(new Func1() { + @Override + public Manga call(Cursor cursor) { + return DatabaseUtils.toObject(cursor, Manga.class); + } + }) + .filter(new Func1() { + @Override + public Boolean call(Manga manga) { + return manga != null; + } + }) + .toBlocking() + .single(); + + if (artistElement != null) { + String fieldArtist = artistElement.text(); + newManga.setArtist(fieldArtist); + newManga.setAuthor(fieldArtist); + } + if (descriptionElement != null) { + String fieldDescription = descriptionElement.text().substring("Description:".length()).trim(); + newManga.setDescription(fieldDescription); + } + if (genreElements != null) { + String fieldGenres = ""; + for (int index = 0; index < genreElements.size(); index++) { + String currentGenre = genreElements.get(index).attr("alt"); + + if (index < genreElements.size() - 1) { + fieldGenres += currentGenre + ", "; + } else { + fieldGenres += currentGenre; + } + } + newManga.setGenre(fieldGenres); + } + if (thumbnailUrlElement != null) { + String fieldThumbnailUrl = thumbnailUrlElement.attr("src"); + newManga.setThumbnailUrl(fieldThumbnailUrl); + } + + boolean fieldCompleted = unparsedHtml.contains("Complete"); + newManga.setCompleted(fieldCompleted); + + + newManga.setInitialized(true); + + mQueryManager.createManga(newManga) + .toBlocking() + .single(); + + return newManga; + } + + public Observable> pullChaptersFromNetwork(final String mangaUrl, final String mangaName) { + return mNetworkService + .getResponse(mangaUrl, NetworkModule.NULL_CACHE_CONTROL, REQUEST_HEADERS) + .flatMap(new Func1>() { + @Override + public Observable call(Response response) { + return mNetworkService.mapResponseToString(response); + } + }) + .flatMap(new Func1>>() { + @Override + public Observable> call(String unparsedHtml) { + return Observable.just(parseHtmlToChapters(mangaUrl, mangaName, unparsedHtml)); + } + }); + } + + private List parseHtmlToChapters(String mangaUrl, String mangaName, String unparsedHtml) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + List chapterList = scrapeChaptersFromParsedDocument(parsedDocument); + chapterList = setSourceForChapterList(chapterList); + chapterList = setParentInfoForChapterList(chapterList, mangaUrl, mangaName); + chapterList = setNumberForChapterList(chapterList); + + saveChaptersToDatabase(chapterList, mangaUrl); + + return chapterList; + } + + private List scrapeChaptersFromParsedDocument(Document parsedDocument) { + List chapterList = new ArrayList(); + + Elements chapterElements = parsedDocument.select("tr.row.lang_English.chapter_row"); + for (Element chapterElement : chapterElements) { + Chapter currentChapter = constructChapterFromHtmlBlock(chapterElement); + + chapterList.add(currentChapter); + } + + return chapterList; + } + + private Chapter constructChapterFromHtmlBlock(Element chapterElement) { + Chapter newChapter = DefaultFactory.Chapter.constructDefault(); + + Element urlElement = chapterElement.select("a[href^=http://bato.to/read/").first(); + Element nameElement = urlElement; + Element dateElement = chapterElement.select("td").get(4); + + if (urlElement != null) { + String fieldUrl = urlElement.attr("href"); + newChapter.setUrl(fieldUrl); + } + if (nameElement != null) { + String fieldName = nameElement.text().trim(); + newChapter.setName(fieldName); + } + if (dateElement != null) { + long fieldDate = parseDateFromElement(dateElement); + newChapter.setDate(fieldDate); + } + + return newChapter; + } + + private long parseDateFromElement(Element dateElement) { + String dateAsString = dateElement.text(); + + try { + Date specificDate = new SimpleDateFormat("dd MMMMM yyyy - hh:mm a", Locale.ENGLISH).parse(dateAsString); + + return specificDate.getTime(); + } catch (ParseException e) { + // Do Nothing. + } + + return DefaultFactory.Chapter.DEFAULT_DATE; + } + + private List setSourceForChapterList(List chapterList) { + for (Chapter currentChapter : chapterList) { + currentChapter.setSource(NAME); + } + + return chapterList; + } + + private List setParentInfoForChapterList(List chapterList, String parentUrl, String parentName) { + for (Chapter currentChapter : chapterList) { + currentChapter.setParentUrl(parentUrl); + currentChapter.setParentName(parentName); + } + + return chapterList; + } + + private List setNumberForChapterList(List chapterList) { + Collections.reverse(chapterList); + for (int index = 0; index < chapterList.size(); index++) { + chapterList.get(index).setNumber(index + 1); + } + + return chapterList; + } + + private void saveChaptersToDatabase(List chapterList, String parentUrl) { + StringBuilder selection = new StringBuilder(); + List selectionArgs = new ArrayList(); + + selection.append(ApplicationContract.Chapter.COLUMN_SOURCE + " = ?"); + selectionArgs.add(NAME); + selection.append(" AND ").append(ApplicationContract.Chapter.COLUMN_PARENT_URL + " = ?"); + selectionArgs.add(parentUrl); + + mQueryManager.beginApplicationTransaction(); + try { + mQueryManager.deleteAllChapter(selection.toString(), selectionArgs.toArray(new String[selectionArgs.size()])) + .toBlocking() + .single(); + + for (Chapter currentChapter : chapterList) { + mQueryManager.createChapter(currentChapter) + .toBlocking() + .single(); + } + + mQueryManager.setApplicationTransactionSuccessful(); + } finally { + mQueryManager.endApplicationTransaction(); + } + } + */ + + public Observable pullImageUrlsFromNetwork(final String chapterUrl) { + final List temporaryCachedImageUrls = new ArrayList<>(); + + return mCacheManager.getImageUrlsFromDiskCache(chapterUrl) + .onErrorResumeNext(throwable -> { + return mNetworkService + .getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, null) + .flatMap(unparsedHtml -> Observable.from(parseHtmlToPageUrls(unparsedHtml))) + .buffer(3) + .concatMap(batchedPageUrls -> { + List> imageUrlObservables = new ArrayList<>(); + for (String pageUrl : batchedPageUrls) { + Observable temporaryObservable = mNetworkService + .getStringResponse(pageUrl, mNetworkService.NULL_CACHE_CONTROL, null) + .flatMap(unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml))) + .subscribeOn(Schedulers.io()); + + imageUrlObservables.add(temporaryObservable); + } + + return Observable.merge(imageUrlObservables); + }) + .doOnNext(imageUrl -> temporaryCachedImageUrls.add(imageUrl)) + .doOnCompleted(mCacheManager.putImageUrlsToDiskCache(chapterUrl, temporaryCachedImageUrls)); + }) + .onBackpressureBuffer(); + } + + private List parseHtmlToPageUrls(String unparsedHtml) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + List pageUrlList = new ArrayList(); + + Elements pageUrlElements = parsedDocument.getElementById("page_select").getElementsByTag("option"); + for (Element pageUrlElement : pageUrlElements) { + pageUrlList.add(pageUrlElement.attr("value")); + } + + return pageUrlList; + } + + private String parseHtmlToImageUrl(String unparsedHtml) { + int beginIndex = unparsedHtml.indexOf("", beginIndex); + String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); + + Document parsedDocument = Jsoup.parse(trimmedHtml); + + Element imageElement = parsedDocument.getElementById("comic_page"); + + return imageElement.attr("src"); + } + + private static String INITIAL_DATABASE_URL_1 = "http://bato.to/comic_pop?id=1"; + private static String INITIAL_DATABASE_URL_2 = "http://bato.to/search_ajax?order_cond=views&order=desc&p=1"; + + private static AtomicInteger mCounter = new AtomicInteger(1); + + /* + public Observable recursivelyConstructDatabase(final String url) { + return mNetworkService + .getResponse(url, NetworkUtil.NULL_CACHE_CONTROL, REQUEST_HEADERS) + .flatMap(new Func1>() { + @Override + public Observable call(Response response) { + return mNetworkService.mapResponseToString(response); + } + }) + .flatMap(new Func1>() { + @Override + public Observable call(String unparsedHtml) { + return Observable.just(parseEnglish_Batoto(unparsedHtml)); + } + }); + } + + + private String parseEnglish_Batoto(String unparsedHtml) { + if (!unparsedHtml.equals("wtf?")) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + Manga newManga = new Manga(); + + Element temporaryElementOne = parsedDocument.getElementsByTag("a").first(); + Element temporaryElementTwo = parsedDocument.select("a[href^=http://bato.to/forums/forum/]").first(); + Element temporaryElementThree = parsedDocument.select("img[src^=http://img.batoto.net/forums/uploads/]").first(); + Elements temporaryElementsFour = parsedDocument.select("img[src=http://bato.to/forums/public/style_images/master/bullet_black.png]"); + + String fieldSource = English_Batoto.NAME; + newManga.setSource(fieldSource); + + String fieldUrl = "http://bato.to" + temporaryElementOne.attr("href"); + newManga.setUrl(fieldUrl); + + String fieldName = temporaryElementTwo.text(); + int startIndex = "Go to ".length(); + int endIndex = fieldName.lastIndexOf(" Forums!"); + newManga.setName(fieldName.substring(startIndex, endIndex)); + + String fieldThumbnailUrl = temporaryElementThree.attr("src"); + newManga.setThumbnailUrl(fieldThumbnailUrl); + + String fieldGenres = ""; + for (int index = 0; index < temporaryElementsFour.size(); index++) { + String currentGenre = temporaryElementsFour.get(index).attr("alt"); + + if (index < temporaryElementsFour.size() - 1) { + fieldGenres += currentGenre + ", "; + } else { + fieldGenres += currentGenre; + } + } + newManga.setGenre(fieldGenres); + + boolean fieldIsCompleted = unparsedHtml.contains("Complete"); + newManga.setCompleted(fieldIsCompleted); + + mQueryManager.createManga(newManga) + .toBlocking() + .single(); + } + + return "http://bato.to/comic_pop?id=" + mCounter.incrementAndGet(); + } + + private String parseEnglish_Batoto_Views(String unparsedHtml) { + if (!unparsedHtml.contains("No (more) comics found!")) { + Document parsedDocument = Jsoup.parse(unparsedHtml); + + List> updateList = new ArrayList>(); + Elements mangaElements = parsedDocument.select("tr:not([id]):not([class])"); + for (Element mangaElement : mangaElements) { + Element temporaryElementOne = mangaElement.select("a[href^=http://bato.to]").first(); + Element temporaryElementTwo = mangaElement.select("td").get(3); + String temporaryString = temporaryElementTwo.text(); + + String fieldUrl = temporaryElementOne.attr("href"); + + String fieldView = null; + if (temporaryString.contains("m")) { + temporaryString = temporaryString.replace("m", ""); + + int viewsAsNumber = (int)(Double.valueOf(temporaryString) * 1000000); + fieldView = String.valueOf(viewsAsNumber); + } else if (temporaryString.contains("k")) { + temporaryString = temporaryString.replace("k", ""); + + int viewsAsNumber = (int)(Double.valueOf(temporaryString) * 1000); + fieldView = String.valueOf(viewsAsNumber); + } else { + int viewsAsNumber = (int)(Double.valueOf(temporaryString) * 1); + fieldView = String.valueOf(viewsAsNumber); + } + + ContentValues fieldRanking = new ContentValues(1); + fieldRanking.put(LibraryContract.Manga.COLUMN_RANK, fieldView); + + updateList.add(Pair.create(fieldUrl, fieldRanking)); + } + + mQueryManager.beginLibraryTransaction(); + try { + for (Pair currentUpdate : updateList) { + mQueryManager.updateManga(currentUpdate.second, LibraryContract.Manga.COLUMN_URL + " = ?", new String[]{currentUpdate.first}) + .toBlocking() + .single(); + } + + mQueryManager.setLibraryTransactionSuccessful(); + } finally { + mQueryManager.endLibraryTransaction(); + } + + return "http://bato.to/search_ajax?order_cond=views&order=desc&p=" + mCounter.incrementAndGet(); + } + + return null; + } + + public void reorderEnglish_Batoto_Rankings() { + List mangaList = mQueryManager.retrieveAllMangaAsStream( + null, + LibraryContract.Manga.COLUMN_SOURCE + " = ?", + new String[]{NAME}, + null, + null, + LibraryContract.Manga.COLUMN_RANK + " DESC", + null + ) + .toList() + .toBlocking() + .single(); + + for (int index = 0; index < mangaList.size(); index++) { + mangaList.get(index).setRank(index + 1); + } + + mQueryManager.beginLibraryTransaction(); + try { + for (Manga currentManga : mangaList) { + mQueryManager.createManga(currentManga) + .toBlocking() + .single(); + } + mQueryManager.setLibraryTransactionSuccessful(); + } finally { + mQueryManager.endLibraryTransaction(); + } + } + */ +} + diff --git a/app/src/main/java/eu/kanade/mangafeed/util/DiskUtils.java b/app/src/main/java/eu/kanade/mangafeed/util/DiskUtils.java new file mode 100644 index 0000000000..c91593e7f3 --- /dev/null +++ b/app/src/main/java/eu/kanade/mangafeed/util/DiskUtils.java @@ -0,0 +1,160 @@ +package eu.kanade.mangafeed.util; + +import android.content.Context; +import android.os.Build; +import android.os.Environment; +import android.text.TextUtils; + +import java.io.File; +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; + +import okio.BufferedSink; +import okio.BufferedSource; +import okio.Okio; + +public final class DiskUtils { + private static final Pattern DIR_SEPORATOR = Pattern.compile("/"); + + private DiskUtils() { + throw new AssertionError(); + } + + // http://stackoverflow.com/questions/13976982/removable-storage-external-sdcard-path-by-manufacturers + // http://stackoverflow.com/questions/11281010/how-can-i-get-external-sd-card-path-for-android-4-0 + public static String[] getStorageDirectories(Context context) { + final Set storageDirectories = new HashSet(); + + storageDirectories.add(context.getFilesDir().getAbsolutePath()); + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { + File[] directories = context.getExternalFilesDirs(null); + if (directories != null) { + for (File storage : directories) { + if (storage != null) { + storageDirectories.add(storage.getAbsolutePath()); + } + } + } + } else { + final String rawExternalStorage = System.getenv("EXTERNAL_STORAGE"); + final String rawSecondaryStoragesStr = System.getenv("SECONDARY_STORAGE"); + final String rawEmulatedStorageTarget = System.getenv("EMULATED_STORAGE_TARGET"); + + if (TextUtils.isEmpty(rawEmulatedStorageTarget)) { + if (TextUtils.isEmpty(rawExternalStorage)) { + storageDirectories.add("/storage/sdcard0" + File.separator + context.getPackageName()); + } else { + storageDirectories.add(rawExternalStorage + File.separator + context.getPackageName()); + } + } else { + final String rawUserId; + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN_MR1) { + rawUserId = ""; + } else { + final String path = Environment.getExternalStorageDirectory().getAbsolutePath(); + final String[] folders = DIR_SEPORATOR.split(path); + final String lastFolder = folders[folders.length - 1]; + boolean isDigit = false; + + try { + Integer.valueOf(lastFolder); + isDigit = true; + } catch (NumberFormatException e) { + // Do Nothing. + } + + rawUserId = isDigit ? lastFolder : ""; + } + + if (TextUtils.isEmpty(rawUserId)) { + storageDirectories.add(rawEmulatedStorageTarget + File.separator + context.getPackageName()); + } else { + storageDirectories.add(rawEmulatedStorageTarget + File.separator + rawUserId + File.separator + context.getPackageName()); + } + } + + if (!TextUtils.isEmpty(rawSecondaryStoragesStr)) { + String[] rawSecondaryStorages = rawSecondaryStoragesStr.split(File.pathSeparator); + for (int index = 0; index < rawSecondaryStorages.length; index++) { + storageDirectories.add(rawSecondaryStorages[index] + File.separator + context.getPackageName()); + } + } + } + + return storageDirectories.toArray(new String[storageDirectories.size()]); + } + + public static String hashKeyForDisk(String key) { + String cacheKey; + try { + final MessageDigest mDigest = MessageDigest.getInstance("MD5"); + mDigest.update(key.getBytes()); + cacheKey = bytesToHexString(mDigest.digest()); + } catch (NoSuchAlgorithmException e) { + cacheKey = String.valueOf(key.hashCode()); + } + return cacheKey; + } + + private static String bytesToHexString(byte[] bytes) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < bytes.length; i++) { + String hex = Integer.toHexString(0xFF & bytes[i]); + if (hex.length() == 1) { + sb.append('0'); + } + sb.append(hex); + } + return sb.toString(); + } + + public static File saveBufferedSourceToDirectory(BufferedSource bufferedSource, String directory, String name) throws IOException { + File fileDirectory = new File(directory); + if (!fileDirectory.exists()) { + if (!fileDirectory.mkdirs()) { + throw new IOException("Failed Creating Directory"); + } + } + + File writeFile = new File(fileDirectory, name); + if (writeFile.exists()) { + if (writeFile.delete()) { + writeFile = new File(fileDirectory, name); + } else { + throw new IOException("Failed Deleting Existing File for Overwrite"); + } + } + + BufferedSink bufferedSink = null; + try { + bufferedSink = Okio.buffer(Okio.sink(writeFile)); + bufferedSink.writeAll(bufferedSource); + } finally { + if (bufferedSource != null) { + bufferedSource.close(); + } + if (bufferedSink != null) { + bufferedSink.close(); + } + } + + return writeFile; + } + + public static void deleteFiles(File inputFile) { + if (inputFile.isDirectory()) { + for (File childFile : inputFile.listFiles()) { + deleteFiles(childFile); + } + } + + inputFile.delete(); + } +} + diff --git a/app/src/test/java/eu/kanade/mangafeed/BatotoTest.java b/app/src/test/java/eu/kanade/mangafeed/BatotoTest.java new file mode 100644 index 0000000000..1511717244 --- /dev/null +++ b/app/src/test/java/eu/kanade/mangafeed/BatotoTest.java @@ -0,0 +1,40 @@ +package eu.kanade.mangafeed; + +import android.os.Build; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricGradleTestRunner; +import org.robolectric.RuntimeEnvironment; +import org.robolectric.annotation.Config; + +import eu.kanade.mangafeed.data.caches.CacheManager; +import eu.kanade.mangafeed.data.helpers.NetworkHelper; +import eu.kanade.mangafeed.sources.Batoto; +import rx.observers.TestSubscriber; + +@Config(constants = BuildConfig.class, sdk = Build.VERSION_CODES.LOLLIPOP) +@RunWith(RobolectricGradleTestRunner.class) +public class BatotoTest { + + NetworkHelper net; + CacheManager cache; + Batoto b; + final String chapterUrl ="http://bato.to/read/_/345144/minamoto-kun-monogatari_ch178_by_vortex-scans"; + + @Before + public void setUp() { + net = new NetworkHelper(); + cache = new CacheManager(RuntimeEnvironment.application.getApplicationContext()); + b = new Batoto(net, cache); + } + + @Test + public void testImageList() { + TestSubscriber a = new TestSubscriber(); + + b.pullImageUrlsFromNetwork(chapterUrl).subscribe(a); + a.assertNoErrors(); + } +} diff --git a/app/src/test/java/eu/kanade/mangafeed/DataManagerTest.java b/app/src/test/java/eu/kanade/mangafeed/DataManagerTest.java deleted file mode 100644 index c4a096ac9b..0000000000 --- a/app/src/test/java/eu/kanade/mangafeed/DataManagerTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package eu.kanade.mangafeed; - - -import android.database.Cursor; - -import eu.kanade.mangafeed.data.local.DatabaseHelper; -import eu.kanade.mangafeed.data.local.Db; -import eu.kanade.mangafeed.data.local.PreferencesHelper; -import eu.kanade.mangafeed.data.model.Character; -import eu.kanade.mangafeed.data.remote.AndroidBoilerplateService; -import eu.kanade.mangafeed.util.DefaultConfig; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.robolectric.RobolectricTestRunner; -import org.robolectric.RuntimeEnvironment; -import org.robolectric.annotation.Config; - -import java.util.List; - -import rx.Observable; -import rx.observers.TestSubscriber; -import rx.schedulers.Schedulers; - -import static junit.framework.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -@RunWith(RobolectricTestRunner.class) -@Config(constants = BuildConfig.class, sdk = DefaultConfig.EMULATE_SDK, manifest = DefaultConfig.MANIFEST) -public class DataManagerTest { - - private DataManager mDataManager; - private AndroidBoilerplateService mMockAndroidBoilerplateService; - private DatabaseHelper mDatabaseHelper; - - @Before - public void setUp() { - mMockAndroidBoilerplateService = mock(AndroidBoilerplateService.class); - mDatabaseHelper = new DatabaseHelper(RuntimeEnvironment.application); - mDatabaseHelper.clearTables().subscribe(); - mDataManager = new DataManager(mMockAndroidBoilerplateService, - mDatabaseHelper, - mock(Bus.class), - new PreferencesHelper(RuntimeEnvironment.application), - Schedulers.immediate()); - } - - @Test - public void shouldSyncCharacters() throws Exception { - int[] ids = new int[]{ 10034, 14050, 10435, 35093 }; - List characters = MockModelsUtil.createListOfMockCharacters(4); - for (int i = 0; i < ids.length; i++) { - when(mMockAndroidBoilerplateService.getCharacter(ids[i])) - .thenReturn(Observable.just(characters.get(i))); - } - - TestSubscriber result = new TestSubscriber<>(); - mDataManager.syncCharacters(ids).subscribe(result); - result.assertNoErrors(); - result.assertReceivedOnNext(characters); - - Cursor cursor = mDatabaseHelper.getBriteDb() - .query("SELECT * FROM " + Db.CharacterTable.TABLE_NAME); - assertEquals(4, cursor.getCount()); - cursor.close(); - } - -} diff --git a/app/src/test/java/eu/kanade/mangafeed/DatabaseHelperTest.java b/app/src/test/java/eu/kanade/mangafeed/DatabaseHelperTest.java deleted file mode 100644 index e47b6cf996..0000000000 --- a/app/src/test/java/eu/kanade/mangafeed/DatabaseHelperTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package eu.kanade.mangafeed; - -import android.database.Cursor; - -import eu.kanade.mangafeed.data.local.DatabaseHelper; -import eu.kanade.mangafeed.data.local.Db; -import eu.kanade.mangafeed.data.model.Character; -import eu.kanade.mangafeed.util.DefaultConfig; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.robolectric.RobolectricTestRunner; -import org.robolectric.RuntimeEnvironment; -import org.robolectric.annotation.Config; - -import java.util.Collections; -import java.util.List; - -import rx.observers.TestSubscriber; - -import static junit.framework.Assert.assertEquals; - -@RunWith(RobolectricTestRunner.class) -@Config(constants = BuildConfig.class, sdk = DefaultConfig.EMULATE_SDK, manifest = DefaultConfig.MANIFEST) -public class DatabaseHelperTest { - - private DatabaseHelper mDatabaseHelper; - - @Before - public void setUp() { - mDatabaseHelper = new DatabaseHelper(RuntimeEnvironment.application); - mDatabaseHelper.clearTables().subscribe(); - } - - @Test - public void shouldSetCharacters() throws Exception { - List characters = MockModelsUtil.createListOfMockCharacters(5); - - TestSubscriber result = new TestSubscriber<>(); - mDatabaseHelper.setCharacters(characters).subscribe(result); - result.assertNoErrors(); - result.assertReceivedOnNext(characters); - - Cursor cursor = mDatabaseHelper.getBriteDb() - .query("SELECT * FROM " + Db.CharacterTable.TABLE_NAME); - assertEquals(5, cursor.getCount()); - for (Character character : characters) { - cursor.moveToNext(); - assertEquals(character, Db.CharacterTable.parseCursor(cursor)); - } - } - - @Test - public void shouldGetCharacters() throws Exception { - List characters = MockModelsUtil.createListOfMockCharacters(5); - - mDatabaseHelper.setCharacters(characters).subscribe(); - - TestSubscriber> result = new TestSubscriber<>(); - mDatabaseHelper.getCharacters().subscribe(result); - result.assertNoErrors(); - result.assertReceivedOnNext(Collections.singletonList(characters)); - } -} \ No newline at end of file diff --git a/app/src/test/java/eu/kanade/mangafeed/LibraryFragmentTest.java b/app/src/test/java/eu/kanade/mangafeed/LibraryFragmentTest.java deleted file mode 100644 index 4567343816..0000000000 --- a/app/src/test/java/eu/kanade/mangafeed/LibraryFragmentTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package eu.kanade.mangafeed; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.robolectric.RobolectricGradleTestRunner; -import org.robolectric.RobolectricTestRunner; -import org.robolectric.annotation.Config; - - -import static org.robolectric.util.FragmentTestUtil.startFragment; -import static org.junit.Assert.assertNotNull; - -import eu.kanade.mangafeed.BuildConfig; -import eu.kanade.mangafeed.ui.fragment.LibraryFragment; -import eu.kanade.mangafeed.util.DefaultConfig; - -/** - * Created by len on 1/10/15. - */ - -@RunWith(RobolectricGradleTestRunner.class) -@Config(constants = BuildConfig.class, sdk = DefaultConfig.EMULATE_SDK) -public class LibraryFragmentTest { - - @Test - public void mangaList_shouldNotBeEmpty() { - LibraryFragment fragment = LibraryFragment.newInstance(); - startFragment(fragment); - assertNotNull(fragment); - } -}