Merge branch 'central-dev' into central-master

app/build.gradle

@@ -19,8 +19,8 @@ android {
         // or Perfect Viewer implements Content URI
         //noinspection ExpiringTargetSdkVersion
         targetSdkVersion 23
-        versionCode 85
-        versionName '1.6.6'
+        versionCode 87
+        versionName '1.6.7'
 
         def fkToken =  '\"' + (System.getenv("FK_TOKEN")?: "")+'\"'
         def includeObjectBoxBrowser = System.getenv("INCLUDE_OBJECTBOX_BROWSER")?:"false"

app/src/main/java/me/devsaki/hentoid/activities/ImportActivity.java

@@ -511,6 +511,8 @@ private boolean hasBooks() {
     }
 
     private void importFolder(File folder) {
+        // TODO - if .Hentoid or Hentoid folder found, ask user if he wants to use that one (#267)
+
         if (!FileHelper.checkAndSetRootFolder(folder.getAbsolutePath(), true)) {
             prepImport(null);
             return;

app/src/main/java/me/devsaki/hentoid/activities/websites/EHentaiActivity.java

@@ -1,5 +1,7 @@
 package me.devsaki.hentoid.activities.websites;
 
+import android.webkit.CookieManager;
+
 import io.reactivex.android.schedulers.AndroidSchedulers;
 import me.devsaki.hentoid.database.domains.Content;
 import me.devsaki.hentoid.enums.Site;
@@ -24,6 +26,7 @@ Site getStartSite() {
     @Override
     protected CustomWebViewClient getWebClient() {
         CustomWebViewClient client = new EHentaiWebClient(GALLERY_FILTER, this);
+        CookieManager.getInstance().setCookie(Site.EHENTAI.getUrl(), "sl=dm_2");
         client.restrictTo(DOMAIN_FILTER);
         return client;
     }

app/src/main/java/me/devsaki/hentoid/database/ObjectBoxDB.java

@@ -170,11 +170,17 @@ private void deleteContentById(long[] contentId) {
         for (long id : contentId) {
             Content c = contentBox.get(id);
             store.runInTx(() -> {
-                for (ImageFile i : c.getImageFiles()) imageFileBox.remove(i);   // Delete imageFiles
-                c.getImageFiles().clear();                                      // Clear links to all imageFiles
+                if (c.getImageFiles() != null) {
+                    for (ImageFile i : c.getImageFiles())
+                        imageFileBox.remove(i);   // Delete imageFiles
+                    c.getImageFiles().clear();                                      // Clear links to all imageFiles
+                }
 
-                for (ErrorRecord e : c.getErrorLog()) errorBox.remove(e);   // Delete error records
-                c.getErrorLog().clear();                                    // Clear links to all errorRecords
+                if (c.getErrorLog() != null) {
+                    for (ErrorRecord e : c.getErrorLog())
+                        errorBox.remove(e);   // Delete error records
+                    c.getErrorLog().clear();                                    // Clear links to all errorRecords
+                }
 
                 // Delete attribute when current content is the only content left on the attribute
                 for (Attribute a : c.getAttributes())
@@ -230,6 +236,10 @@ public void deleteQueue(Content content) {
         deleteQueue(content.getId());
     }
 
+    public void deleteQueue(int queueIndex) {
+        store.boxFor(QueueRecord.class).remove(selectQueue().get(queueIndex).id);
+    }
+
     private void deleteQueue(long contentId) {
         Box<QueueRecord> queueRecordBox = store.boxFor(QueueRecord.class);
         QueueRecord record = queueRecordBox.query().equal(QueueRecord_.contentId, contentId).build().findFirst();

app/src/main/java/me/devsaki/hentoid/database/domains/Content.java

@@ -1,5 +1,7 @@
 package me.devsaki.hentoid.database.domains;
 
+import android.support.annotation.Nullable;
+
 import com.google.gson.annotations.Expose;
 import com.google.gson.annotations.SerializedName;
 
@@ -292,10 +294,11 @@ public String getReaderUrl() {
             case ASMHENTAI_COMICS:
                 return site.getUrl() + "/gallery" + url;
             case EHENTAI:               // Won't work anyway because of the temporary key
-            case HENTAICAFE:
             case NHENTAI:
             case PANDA:
                 return getGalleryUrl();
+            case HENTAICAFE:
+                return site.getUrl() + "/manga/read/$1/en/0/1/"; // $1 has to be replaced by the textual unique site ID without the author name
             case PURURIN:
                 return site.getUrl() + "/read/" + url.substring(1).replace("/", "/01/");
             case FAKKU2:
@@ -409,6 +412,7 @@ public Content setStatus(StatusContent status) {
         return this;
     }
 
+    @Nullable
     public ToMany<ImageFile> getImageFiles() {
         return imageFiles;
     }
@@ -421,6 +425,7 @@ public Content addImageFiles(List<ImageFile> imageFiles) {
         return this;
     }
 
+    @Nullable
     public ToMany<ErrorRecord> getErrorLog() {
         return errorLog;
     }

app/src/main/java/me/devsaki/hentoid/fragments/QueueFragment.java

@@ -140,7 +140,7 @@ public void onDownloadEvent(DownloadEvent event) {
             case DownloadEvent.EV_COMPLETE:
                 mAdapter.removeFromQueue(event.content);
                 if (0 == mAdapter.getCount()) btnStats.setVisibility(View.GONE);
-            default: // EV_PAUSE, EV_CANCEL events
+            default: // EV_PAUSE, EV_CANCEL events + EV_COMPLETE that doesn't have a break
                 update(event.eventType);
         }
     }

app/src/main/java/me/devsaki/hentoid/parsers/HentaiCafeParser.java

@@ -30,7 +30,7 @@ public class HentaiCafeParser extends BaseParser {
     protected List<String> parseImages(Content content) throws IOException {
         List<String> result = new ArrayList<>();
 
-        Document doc = getOnlineDocument(content.getReaderUrl());
+        Document doc = getOnlineDocument(content.getGalleryUrl());
         if (doc != null) {
             Elements links = doc.select("a.x-btn");
 
@@ -46,12 +46,25 @@ protected List<String> parseImages(Content content) throws IOException {
                 for (int i = 0; i < links.size(); i++) {
 
                     String url = links.get(i).attr("href");
+                    if (url.equals("#")) { // Some pages are like this (e.g. 2606) -> reconstitute the reader URL manually
+                        // Get the canonical link
+                        Elements canonicalLink = doc.select("head [rel=canonical]");
+                        if (canonicalLink != null) {
+                            // Remove artist name
+                            String artist = content.getAuthor().replace(" ","-").toLowerCase()+"-";
+                            String canonicalUrl = canonicalLink.get(0).attr("href").replace(artist,"");
+                            // Get the last part
+                            String[] parts = canonicalUrl.split("/");
+                            String canonicalName = parts[parts.length - 1].replace("-","_");
+                            url = content.getReaderUrl().replace("$1", canonicalName); // $1 has to be replaced by the textual unique site ID without the author name
+                        }
+                    }
 
                     if (URLUtil.isValidUrl(url)) {
-                        Timber.d("Chapter Links: %s", links.get(i).attr("href"));
+                        Timber.d("Chapter Links: %s", url);
                         try {
 //                            doc = Jsoup.connect(links.get(i).attr("href")).timeout(TIMEOUT).get();
-                            doc = getOnlineDocument(links.get(i).attr("href"));
+                            doc = getOnlineDocument(url);
                             if (doc != null) {
                                 contents = doc.select("article#content");
                                 js = contents.select("script").last();

app/src/main/java/me/devsaki/hentoid/parsers/content/FakkuContent.java

@@ -56,7 +56,7 @@ public Content toContent() {
         for (Element e : information) {
             elementName = e.child(0).text().trim().toLowerCase();
             if (elementName.equals("language")) {
-                ParseHelper.parseAttributes(attributes, AttributeType.ARTIST, e.child(1).children(), true, Site.FAKKU2); // Language elements are A links
+                ParseHelper.parseAttributes(attributes, AttributeType.LANGUAGE, e.child(1).children(), true, Site.FAKKU2); // Language elements are A links
             }
             if (elementName.equals("pages")) {
                 qtyPages = Integer.parseInt(e.child(1).text().toLowerCase().replace("pages", "").replace("page", "").trim());

app/src/main/java/me/devsaki/hentoid/services/ContentDownloadService.java

@@ -132,37 +132,51 @@ private Content downloadFirstInQueue() {
 
         Content content = queue.get(0).content.getTarget();
 
-        if (null == content || StatusContent.DOWNLOADED == content.getStatus()) {
-            Timber.w("Content is unavailable, or already downloaded. Aborting download.");
+        if (null == content) {
+            Timber.w("Content is unavailable. Aborting download.");
+            db.deleteQueue(0);
+            content = new Content().setId(queue.get(0).content.getTargetId()); // Must supply content ID to the event for the UI to update properly
+            EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.EV_COMPLETE, 0, 0, 0));
             return null;
         }
 
-        content.setStatus(StatusContent.DOWNLOADING);
-        content.getErrorLog().clear();
-        db.insertContent(content);
+        if (StatusContent.DOWNLOADED == content.getStatus()) {
+            Timber.w("Content is already downloaded. Aborting download.");
+            db.deleteQueue(0);
+            EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.EV_COMPLETE, 0, 0, 0));
+            return null;
+        }
 
+        db.deleteErrorRecords(content.getId());
+
+        boolean hasError = false;
         // Check if images are already known
         List<ImageFile> images = content.getImageFiles();
-        if (images.isEmpty()) {
+        if (null == images || images.isEmpty()) {
             try {
                 images = fetchImageURLs(content);
                 content.addImageFiles(images);
                 db.insertContent(content);
             } catch (UnsupportedOperationException u) {
                 Timber.w(u, "A captcha has been found while parsing %s. Aborting download.", content.getTitle());
                 logErrorRecord(content.getId(), ErrorType.CAPTCHA, content.getUrl(), "Image list", u.getMessage());
-                content.setStatus(StatusContent.ERROR);
-                db.insertContent(content);
-                return null;
+                hasError = true;
             } catch (Exception e) {
                 Timber.w(e, "An exception has occurred while parsing %s. Aborting download.", content.getTitle());
                 logErrorRecord(content.getId(), ErrorType.PARSING, content.getUrl(), "Image list", e.getMessage());
-                content.setStatus(StatusContent.ERROR);
-                db.insertContent(content);
-                return null;
+                hasError = true;
             }
         }
 
+        if (hasError) {
+            content.setStatus(StatusContent.ERROR);
+            db.insertContent(content);
+            db.deleteQueue(content);
+            EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.EV_COMPLETE, 0, 0, 0));
+            HentoidApp.trackDownloadEvent("Error");
+            return null;
+        }
+
         File dir = FileHelper.createContentDownloadDir(this, content);
         if (!dir.exists()) {
             String title = content.getTitle();
@@ -174,6 +188,7 @@ private Content downloadFirstInQueue() {
 
         String fileRoot = Preferences.getRootFolderName();
         content.setStorageFolder(dir.getAbsolutePath().substring(fileRoot.length()));
+        content.setStatus(StatusContent.DOWNLOADING);
         db.insertContent(content);
 
 
@@ -223,7 +238,7 @@ private void watchProgress(Content content) {
             pagesKO = statuses.get(StatusContent.ERROR.getCode());
 
             String title = content.getTitle();
-            int totalPages = images.size();
+            int totalPages = (null == images) ? 0 : images.size();
             int progress = pagesOK + pagesKO;
             isDone = progress == totalPages;
             Timber.d("Progress: OK:%s KO:%s Total:%s", pagesOK, pagesKO, totalPages);
@@ -258,10 +273,19 @@ private void completeDownload(Content content, int pagesOK, int pagesKO) {
         if (!downloadCanceled && !downloadSkipped) {
             File dir = FileHelper.createContentDownloadDir(this, content);
             List<ImageFile> images = content.getImageFiles();
+            int nbImages = (null == images) ? 0 : images.size();
+
+            boolean hasError = false;
+            // Less pages than initially detected - More than 10% difference in number of pages
+            if (nbImages < content.getQtyPages() && Math.abs(nbImages - content.getQtyPages()) > content.getQtyPages() * 0.1) {
+                String errorMsg = String.format("The number of images found (%s) does not match the book's number of pages (%s)", nbImages, content.getQtyPages());
+                logErrorRecord(content.getId(), ErrorType.PARSING, content.getGalleryUrl(), "pages", errorMsg);
+                hasError = true;
+            }
 
             // Mark content as downloaded
             content.setDownloadDate(new Date().getTime());
-            content.setStatus((0 == pagesKO) ? StatusContent.DOWNLOADED : StatusContent.ERROR);
+            content.setStatus((0 == pagesKO && !hasError) ? StatusContent.DOWNLOADED : StatusContent.ERROR);
             // Clear download params from content and images
             content.setDownloadParams("");
 
@@ -301,7 +325,7 @@ private void completeDownload(Content content, int pagesOK, int pagesKO) {
 
             // Signals current download as completed
             Timber.d("CompleteActivity : OK = %s; KO = %s", pagesOK, pagesKO);
-            EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.EV_COMPLETE, pagesOK, pagesKO, images.size()));
+            EventBus.getDefault().post(new DownloadEvent(content, DownloadEvent.EV_COMPLETE, pagesOK, pagesKO, nbImages));
 
             // Tracking Event (Download Completed)
             HentoidApp.trackDownloadEvent("Completed");
@@ -328,11 +352,8 @@ private List<ImageFile> fetchImageURLs(Content content) throws Exception {
         ContentParser parser = ContentParserFactory.getInstance().getParser(content);
         imgs = parser.parseImageList(content);
 
-        if (imgs.isEmpty()) throw new Exception("An empty image list has been found while parsing " + content.getGalleryUrl());
-
-        // More than 10% difference in number of pages
-        if (Math.abs(imgs.size() - content.getQtyPages()) > content.getQtyPages() * 0.1)
-            throw new Exception(String.format("The number of images found (%s) does not match the book's number of pages (%s)", imgs.size(), content.getQtyPages()));
+        if (imgs.isEmpty())
+            throw new Exception("An empty image list has been found while parsing " + content.getGalleryUrl());
 
         for (ImageFile img : imgs) img.setStatus(StatusContent.SAVED);
         return imgs;
@@ -500,7 +521,8 @@ private static void saveImage(String fileName, File dir, String contentType, byt
     private void updateImageStatus(ImageFile img, boolean success) {
         img.setStatus(success ? StatusContent.DOWNLOADED : StatusContent.ERROR);
         if (success) img.setDownloadParams("");
-        if (img.getId() > 0) db.updateImageFileStatusAndParams(img); // because thumb image isn't in the DB
+        if (img.getId() > 0)
+            db.updateImageFileStatusAndParams(img); // because thumb image isn't in the DB
     }
 
     /**

app/src/main/java/me/devsaki/hentoid/services/ContentQueueManager.java

@@ -37,7 +37,7 @@ public static synchronized ContentQueueManager getInstance() {
 
 
     // QUEUE ACTIVITY CONTROL
-    public void pauseQueue() {
+    void pauseQueue() {
         isQueuePaused = true;
     }
     public void unpauseQueue() {
@@ -66,7 +66,7 @@ public void setDownloadCount(int downloadCount) {
     /**
      * Signals a new completed download
      */
-    public void downloadComplete() {
+    void downloadComplete() {
         downloadCount++;
     }
 }