New source (unfinished)

This commit is contained in:
inorichi 2015-11-14 15:33:07 +01:00
parent 985c5c09a7
commit e74f81d41a
6 changed files with 247 additions and 56 deletions

View File

@ -7,13 +7,15 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import eu.kanade.mangafeed.sources.Batoto; import eu.kanade.mangafeed.sources.Batoto;
import eu.kanade.mangafeed.sources.MangaHere; import eu.kanade.mangafeed.sources.Mangahere;
import eu.kanade.mangafeed.sources.Mangafox;
import eu.kanade.mangafeed.sources.base.Source; import eu.kanade.mangafeed.sources.base.Source;
public class SourceManager { public class SourceManager {
public static final int BATOTO = 1; public static final int BATOTO = 1;
public static final int MANGAHERE = 2; public static final int MANGAHERE = 2;
public static final int MANGAFOX = 3;
private HashMap<Integer, Source> mSourcesMap; private HashMap<Integer, Source> mSourcesMap;
private Context context; private Context context;
@ -37,7 +39,9 @@ public class SourceManager {
case BATOTO: case BATOTO:
return new Batoto(context); return new Batoto(context);
case MANGAHERE: case MANGAHERE:
return new MangaHere(context); return new Mangahere(context);
case MANGAFOX:
return new Mangafox(context);
} }
return null; return null;
@ -46,6 +50,7 @@ public class SourceManager {
private void initializeSources() { private void initializeSources() {
mSourcesMap.put(BATOTO, createSource(BATOTO)); mSourcesMap.put(BATOTO, createSource(BATOTO));
mSourcesMap.put(MANGAHERE, createSource(MANGAHERE)); mSourcesMap.put(MANGAHERE, createSource(MANGAHERE));
mSourcesMap.put(MANGAFOX, createSource(MANGAFOX));
} }
public List<Source> getSources() { public List<Source> getSources() {

View File

@ -4,7 +4,6 @@ import android.os.Bundle;
import com.pushtorefresh.storio.sqlite.operations.put.PutResult; import com.pushtorefresh.storio.sqlite.operations.put.PutResult;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@ -40,8 +39,8 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
private Subscription mQueryDebouncerSubscription; private Subscription mQueryDebouncerSubscription;
private Subscription mMangaDetailFetchSubscription; private Subscription mMangaDetailFetchSubscription;
private PublishSubject<Observable<String>> mQueryDebouncerSubject; private PublishSubject<String> mQueryDebouncerSubject;
private PublishSubject<Observable<List<Manga>>> mMangaDetailPublishSubject; private PublishSubject<List<Manga>> mMangaDetailPublishSubject;
private static final int GET_MANGA_LIST = 1; private static final int GET_MANGA_LIST = 1;
@ -59,7 +58,7 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
view.hideProgressBar(); view.hideProgressBar();
view.onAddPage(page); view.onAddPage(page);
if (mMangaDetailPublishSubject != null) if (mMangaDetailPublishSubject != null)
mMangaDetailPublishSubject.onNext(Observable.just(page.data)); mMangaDetailPublishSubject.onNext(page.data);
}); });
initializeSearch(); initializeSearch();
@ -130,7 +129,7 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
mSearchMode = false; mSearchMode = false;
mQueryDebouncerSubject = PublishSubject.create(); mQueryDebouncerSubject = PublishSubject.create();
add(mQueryDebouncerSubscription = Observable.switchOnNext(mQueryDebouncerSubject) add(mQueryDebouncerSubscription = mQueryDebouncerSubject
.debounce(SEARCH_TIMEOUT, TimeUnit.MILLISECONDS) .debounce(SEARCH_TIMEOUT, TimeUnit.MILLISECONDS)
.subscribeOn(Schedulers.io()) .subscribeOn(Schedulers.io())
.observeOn(AndroidSchedulers.mainThread()) .observeOn(AndroidSchedulers.mainThread())
@ -143,25 +142,12 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
mMangaDetailPublishSubject = PublishSubject.create(); mMangaDetailPublishSubject = PublishSubject.create();
add(mMangaDetailFetchSubscription = Observable.switchOnNext(mMangaDetailPublishSubject) add(mMangaDetailFetchSubscription = mMangaDetailPublishSubject
.subscribeOn(Schedulers.io()) .subscribeOn(Schedulers.io())
.flatMap(Observable::from) .flatMap(Observable::from)
.filter(manga -> !manga.initialized) .filter(manga -> !manga.initialized)
.buffer(3) .window(3)
.concatMap(localMangas -> { .concatMap(pack -> pack.concatMap(this::getMangaDetails))
List<Observable<Manga>> mangaObservables = new ArrayList<>();
for (Manga manga : localMangas) {
Observable<Manga> tempObs = selectedSource.pullMangaFromNetwork(manga.url)
.subscribeOn(Schedulers.io())
.flatMap(networkManga -> {
Manga.copyFromNetwork(manga, networkManga);
db.insertManga(manga).executeAsBlocking();
return Observable.just(manga);
});
mangaObservables.add(tempObs);
}
return Observable.merge(mangaObservables);
})
.filter(manga -> manga.initialized) .filter(manga -> manga.initialized)
.onBackpressureBuffer() .onBackpressureBuffer()
.observeOn(AndroidSchedulers.mainThread()) .observeOn(AndroidSchedulers.mainThread())
@ -171,12 +157,22 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
})); }));
} }
private Observable<Manga> getMangaDetails(final Manga manga) {
return selectedSource.pullMangaFromNetwork(manga.url)
.subscribeOn(Schedulers.io())
.flatMap(networkManga -> {
Manga.copyFromNetwork(manga, networkManga);
db.insertManga(manga).executeAsBlocking();
return Observable.just(manga);
});
}
public void onSearchEvent(String query, boolean now) { public void onSearchEvent(String query, boolean now) {
// If the query is empty or not debounced, resolve it instantly // If the query is empty or not debounced, resolve it instantly
if (now || query.equals("")) if (now || query.equals(""))
queryFromSearch(query); queryFromSearch(query);
else if (mQueryDebouncerSubject != null) else if (mQueryDebouncerSubject != null)
mQueryDebouncerSubject.onNext(Observable.just(query)); mQueryDebouncerSubject.onNext(query);
} }
private void queryFromSearch(String query) { private void queryFromSearch(String query) {

View File

@ -0,0 +1,179 @@
package eu.kanade.mangafeed.sources;
import android.content.Context;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.ArrayList;
import java.util.List;
import eu.kanade.mangafeed.data.helpers.SourceManager;
import eu.kanade.mangafeed.data.models.Chapter;
import eu.kanade.mangafeed.data.models.Manga;
import eu.kanade.mangafeed.sources.base.Source;
public class Mangafox extends Source {
public static final String NAME = "Mangafox (EN)";
private static final String INITIAL_UPDATE_URL = "http://mangafox.me/directory/";
private static final String INITIAL_SEARCH_URL =
"http://mangafox.me/search.php?name_method=cw&advopts=1&order=az&sort=name";
public Mangafox(Context context) {
super(context);
}
@Override
public String getName() {
return NAME;
}
@Override
public int getSourceId() {
return SourceManager.MANGAFOX;
}
@Override
public boolean isLoginRequired() {
return false;
}
@Override
protected String getUrlFromPageNumber(int page) {
return INITIAL_UPDATE_URL + page + ".htm";
}
@Override
public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml);
List<Manga> mangaList = new ArrayList<>();
Elements mangaHtmlBlocks = parsedDocument.select("div#mangalist > ul.list > li");
for (Element currentHtmlBlock : mangaHtmlBlocks) {
Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentManga);
}
return mangaList;
}
private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) {
Manga mangaFromHtmlBlock = new Manga();
mangaFromHtmlBlock.source = getSourceId();
Element urlElement = htmlBlock.select("a.title").first();
if (urlElement != null) {
mangaFromHtmlBlock.url = urlElement.attr("href");
mangaFromHtmlBlock.title = urlElement.text();
}
return mangaFromHtmlBlock;
}
@Override
protected String getSearchUrl(String query, int page) {
return INITIAL_SEARCH_URL + "&name=" + query + "&page=" + page;
}
@Override
protected List<Manga> parseSearchFromHtml(String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml);
List<Manga> mangaList = new ArrayList<>();
Elements mangaHtmlBlocks = parsedDocument.select("table#listing > tbody > tr:gt(0)");
for (Element currentHtmlBlock : mangaHtmlBlocks) {
Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentManga);
}
return mangaList;
}
private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) {
Manga mangaFromHtmlBlock = new Manga();
mangaFromHtmlBlock.source = getSourceId();
Element urlElement = htmlBlock.select("a.series_preview").first();
if (urlElement != null) {
mangaFromHtmlBlock.url = urlElement.attr("href");
mangaFromHtmlBlock.title = urlElement.text();
}
return mangaFromHtmlBlock;
}
@Override
protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml);
Element infoElement = parsedDocument.select("div#title").first();
Element titleElement = infoElement.select("h2 > a").first();
Element rowElement = infoElement.select("table > tbody > tr:eq(1)").first();
Element authorElement = rowElement.select("td:eq(1)").first();
Element artistElement = rowElement.select("td:eq(2)").first();
Element genreElement = rowElement.select("td:eq(3)").first();
Element descriptionElement = infoElement.select("p.summary").first();
Element thumbnailUrlElement = parsedDocument.select("div.cover > img").first();
Manga newManga = new Manga();
newManga.url = mangaUrl;
if (titleElement != null) {
String title = titleElement.text();
// Strip the last word
title = title.substring(0, title.lastIndexOf(" "));
newManga.title = title;
}
if (artistElement != null) {
String fieldArtist = artistElement.text();
newManga.artist = fieldArtist;
}
if (authorElement != null) {
String fieldAuthor = authorElement.text();
newManga.author = fieldAuthor;
}
if (descriptionElement != null) {
String fieldDescription = descriptionElement.text();
newManga.description = fieldDescription;
}
if (genreElement != null) {
String fieldGenre = genreElement.text();
newManga.genre = fieldGenre;
}
if (thumbnailUrlElement != null) {
String fieldThumbnailUrl = thumbnailUrlElement.attr("src");
newManga.thumbnail_url = fieldThumbnailUrl;
}
// if (statusElement != null) {
// boolean fieldCompleted = statusElement.text().contains("Completed");
// newManga.status = fieldCompleted + "";
// }
newManga.initialized = true;
return newManga;
}
@Override
protected List<Chapter> parseHtmlToChapters(String unparsedHtml) {
return null;
}
@Override
protected List<String> parseHtmlToPageUrls(String unparsedHtml) {
return null;
}
@Override
protected String parseHtmlToImageUrl(String unparsedHtml) {
return null;
}
}

View File

@ -21,15 +21,15 @@ import eu.kanade.mangafeed.data.models.Manga;
import eu.kanade.mangafeed.sources.base.Source; import eu.kanade.mangafeed.sources.base.Source;
import rx.Observable; import rx.Observable;
public class MangaHere extends Source { public class Mangahere extends Source {
public static final String NAME = "MangaHere (EN)"; public static final String NAME = "Mangahere (EN)";
public static final String BASE_URL = "www.mangahere.co"; public static final String BASE_URL = "www.mangahere.co";
private static final String INITIAL_UPDATE_URL = "http://www.mangahere.co/latest/"; private static final String INITIAL_UPDATE_URL = "http://www.mangahere.co/directory/";
private static final String INITIAL_SEARCH_URL = "http://www.mangahere.co/search.php?"; private static final String INITIAL_SEARCH_URL = "http://www.mangahere.co/search.php?";
public MangaHere(Context context) { public Mangahere(Context context) {
super(context); super(context);
} }
@ -45,7 +45,7 @@ public class MangaHere extends Source {
@Override @Override
protected String getUrlFromPageNumber(int page) { protected String getUrlFromPageNumber(int page) {
return INITIAL_UPDATE_URL + page + "/"; return INITIAL_UPDATE_URL + page + ".htm?views.za";
} }
@Override @Override
@ -101,42 +101,55 @@ public class MangaHere extends Source {
public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) { public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) {
Document parsedDocument = Jsoup.parse(unparsedHtml); Document parsedDocument = Jsoup.parse(unparsedHtml);
List<Manga> updatedMangaList = new ArrayList<>(); List<Manga> mangaList = new ArrayList<>();
Elements updatedHtmlBlocks = parsedDocument.select("div.manga_updates dl"); Elements mangaHtmlBlocks = parsedDocument.select("div.directory_list > ul > li");
for (Element currentHtmlBlock : updatedHtmlBlocks) { for (Element currentHtmlBlock : mangaHtmlBlocks) {
Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock); Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentManga);
updatedMangaList.add(currentlyUpdatedManga);
} }
return updatedMangaList; return mangaList;
}
private Manga constructPopularMangaFromHtmlBlock(Element htmlBlock) {
Manga mangaFromHtmlBlock = new Manga();
mangaFromHtmlBlock.source = getSourceId();
Element urlElement = htmlBlock.select("div.title > a").first();
if (urlElement != null) {
mangaFromHtmlBlock.url = urlElement.attr("href");
mangaFromHtmlBlock.title = urlElement.attr("title");
}
return mangaFromHtmlBlock;
} }
@Override @Override
protected List<Manga> parseSearchFromHtml(String unparsedHtml) { protected List<Manga> parseSearchFromHtml(String unparsedHtml) {
return null; Document parsedDocument = Jsoup.parse(unparsedHtml);
List<Manga> mangaList = new ArrayList<>();
Elements mangaHtmlBlocks = parsedDocument.select("div.result_search > dl");
for (Element currentHtmlBlock : mangaHtmlBlocks) {
Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock);
mangaList.add(currentManga);
} }
private Manga constructMangaFromHtmlBlock(Element htmlBlock) { return mangaList;
}
private Manga constructSearchMangaFromHtmlBlock(Element htmlBlock) {
Manga mangaFromHtmlBlock = new Manga(); Manga mangaFromHtmlBlock = new Manga();
mangaFromHtmlBlock.source = getSourceId(); mangaFromHtmlBlock.source = getSourceId();
Element urlElement = htmlBlock.select("a.manga_info").first(); Element urlElement = htmlBlock.select("a.manga_info").first();
Element nameElement = htmlBlock.select("a.manga_info").first();
Element updateElement = htmlBlock.select("span.time").first();
if (urlElement != null) { if (urlElement != null) {
String fieldUrl = urlElement.attr("href"); mangaFromHtmlBlock.url = urlElement.attr("href");
mangaFromHtmlBlock.url = fieldUrl; mangaFromHtmlBlock.title = urlElement.text();
}
if (nameElement != null) {
String fieldName = nameElement.text();
mangaFromHtmlBlock.title = fieldName;
}
if (updateElement != null) {
long fieldUpdate = parseUpdateFromElement(updateElement);
mangaFromHtmlBlock.last_update = fieldUpdate;
} }
return mangaFromHtmlBlock; return mangaFromHtmlBlock;

View File

@ -85,9 +85,7 @@ public abstract class Source extends BaseSource {
return Observable.from(pages) return Observable.from(pages)
.filter(page -> page.getImageUrl() == null) .filter(page -> page.getImageUrl() == null)
.window(overrideNumberOfConcurrentPageDownloads()) .window(overrideNumberOfConcurrentPageDownloads())
.concatMap(batchedPages -> .concatMap(batchedPages -> batchedPages.concatMap(this::getImageUrlFromPage));
batchedPages.concatMap(this::getImageUrlFromPage)
);
} }
private Observable<Page> getImageUrlFromPage(final Page page) { private Observable<Page> getImageUrlFromPage(final Page page) {

View File

@ -46,9 +46,9 @@ public class MainActivity extends BaseActivity {
new PrimaryDrawerItem() new PrimaryDrawerItem()
.withName(R.string.library_title) .withName(R.string.library_title)
.withIdentifier(R.id.nav_drawer_library), .withIdentifier(R.id.nav_drawer_library),
new PrimaryDrawerItem() // new PrimaryDrawerItem()
.withName(R.string.recent_updates_title) // .withName(R.string.recent_updates_title)
.withIdentifier(R.id.nav_drawer_recent_updates), // .withIdentifier(R.id.nav_drawer_recent_updates),
new PrimaryDrawerItem() new PrimaryDrawerItem()
.withName(R.string.catalogues_title) .withName(R.string.catalogues_title)
.withIdentifier(R.id.nav_drawer_catalogues), .withIdentifier(R.id.nav_drawer_catalogues),