From d4ca148f8181eb466446ce6bbdf36afb4d7c3a9f Mon Sep 17 00:00:00 2001 From: MetaPrime Date: Mon, 6 Jan 2025 21:37:42 -0800 Subject: [PATCH] Force each class to use its own Logger instance for better logs (Fixes #2070) --- src/main/java/com/rarchives/ripme/App.java | 8 +- .../ripme/ripper/AbstractHTMLRipper.java | 59 +++--- .../ripme/ripper/AbstractJSONRipper.java | 49 +++-- .../ripme/ripper/AbstractRipper.java | 68 +++--- .../rarchives/ripme/ripper/AlbumRipper.java | 25 ++- .../rarchives/ripme/ripper/VideoRipper.java | 24 ++- .../ripper/rippers/ArtStationRipper.java | 27 ++- .../ripme/ripper/rippers/ArtstnRipper.java | 13 +- .../ripme/ripper/rippers/BatoRipper.java | 6 +- .../ripme/ripper/rippers/ChanRipper.java | 30 +-- .../ripme/ripper/rippers/CheveretoRipper.java | 7 +- .../ripper/rippers/CoomerPartyRipper.java | 16 +- .../ripme/ripper/rippers/DanbooruRipper.java | 37 ++-- .../ripper/rippers/DeviantartRipper.java | 196 +++++++++--------- .../ripme/ripper/rippers/E621Ripper.java | 7 +- .../ripme/ripper/rippers/EHentaiRipper.java | 52 +++-- .../ripper/rippers/EightmusesRipper.java | 31 +-- .../ripme/ripper/rippers/ErofusRipper.java | 31 +-- .../ripme/ripper/rippers/EromeRipper.java | 10 +- .../ripper/rippers/FivehundredpxRipper.java | 28 ++- .../ripme/ripper/rippers/FlickrRipper.java | 44 ++-- .../ripper/rippers/FuraffinityRipper.java | 34 +-- .../ripme/ripper/rippers/FuskatorRipper.java | 6 +- .../ripper/rippers/GirlsOfDesireRipper.java | 9 +- .../ripper/rippers/Hentai2readRipper.java | 9 +- .../ripper/rippers/HentaidudeRipper.java | 10 +- .../ripper/rippers/HentaifoundryRipper.java | 18 +- .../ripme/ripper/rippers/HentaifoxRipper.java | 8 +- .../ripme/ripper/rippers/HitomiRipper.java | 6 +- .../ripme/ripper/rippers/HqpornerRipper.java | 28 +-- .../ripme/ripper/rippers/HypnohubRipper.java | 8 +- .../ripme/ripper/rippers/ImagebamRipper.java | 23 +- .../ripme/ripper/rippers/ImagefapRipper.java | 55 ++--- .../ripper/rippers/ImagevenueRipper.java | 10 +- .../ripme/ripper/rippers/ImgurRipper.java | 54 ++--- .../ripme/ripper/rippers/InstagramRipper.java | 52 +++-- .../ripper/rippers/JagodibujaRipper.java | 10 +- .../ripme/ripper/rippers/Jpg3Ripper.java | 17 +- .../ripme/ripper/rippers/ListalRipper.java | 27 ++- .../ripme/ripper/rippers/LusciousRipper.java | 22 +- .../ripme/ripper/rippers/ManganeloRipper.java | 6 +- .../ripper/rippers/MotherlessRipper.java | 17 +- .../ripme/ripper/rippers/NatalieMuRipper.java | 7 +- .../ripper/rippers/NewgroundsRipper.java | 15 +- .../ripme/ripper/rippers/NfsfwRipper.java | 21 +- .../ripme/ripper/rippers/NhentaiRipper.java | 28 +-- .../ripper/rippers/PhotobucketRipper.java | 22 +- .../ripme/ripper/rippers/PicstatioRipper.java | 10 +- .../ripme/ripper/rippers/PornhubRipper.java | 15 +- .../ripme/ripper/rippers/RedditRipper.java | 48 +++-- .../ripme/ripper/rippers/RedgifsRipper.java | 23 +- .../ripper/rippers/SankakuComplexRipper.java | 11 +- .../ripme/ripper/rippers/ScrolllerRipper.java | 23 +- .../ripme/ripper/rippers/SinfestRipper.java | 6 +- .../ripme/ripper/rippers/SpankbangRipper.java | 10 +- .../ripme/ripper/rippers/StaRipper.java | 12 +- .../ripme/ripper/rippers/TapasticRipper.java | 20 +- .../ripme/ripper/rippers/ThechiveRipper.java | 29 +-- .../ripme/ripper/rippers/TsuminoRipper.java | 22 +- .../ripme/ripper/rippers/TumblrRipper.java | 63 +++--- .../ripme/ripper/rippers/TwitterRipper.java | 30 +-- .../ripper/rippers/TwodgalleriesRipper.java | 8 +- .../ripme/ripper/rippers/ViewcomicRipper.java | 96 ++++----- .../ripme/ripper/rippers/VkRipper.java | 35 ++-- .../ripme/ripper/rippers/VscoRipper.java | 65 +++--- .../ripme/ripper/rippers/XhamsterRipper.java | 27 +-- .../ripme/ripper/rippers/XvideosRipper.java | 6 +- .../ripme/ripper/rippers/ZizkiRipper.java | 13 +- .../rippers/video/CliphunterRipper.java | 8 +- .../rippers/video/MotherlessVideoRipper.java | 9 +- .../ripper/rippers/video/PornhubRipper.java | 7 +- .../rippers/video/TwitchVideoRipper.java | 15 +- .../ripper/rippers/video/ViddmeRipper.java | 8 +- .../ripper/rippers/video/VidearnRipper.java | 6 +- .../ripme/ripper/rippers/video/VkRipper.java | 8 +- .../ripper/rippers/video/YuvutuRipper.java | 9 +- .../rarchives/ripme/ui/ClipboardUtils.java | 6 +- .../java/com/rarchives/ripme/utils/Utils.java | 41 ++-- .../com/rarchives/ripme/tst/UtilsTest.java | 4 +- 79 files changed, 1167 insertions(+), 816 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 2c37fdc95..bd373a4e0 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -43,7 +43,7 @@ */ public class App { - public static final Logger logger = LogManager.getLogger(App.class); + private static final Logger logger = LogManager.getLogger(App.class); public static String stringToAppendToFoldername = null; private static final History HISTORY = new History(); @@ -140,7 +140,7 @@ private static void handleArguments(String[] args) throws IOException { Utils.setConfigString("history.location", historyLocation); logger.info("Set history file to " + historyLocation); } - + //Allow file overwriting if (cl.hasOption('w')) { Utils.setConfigBoolean("file.overwrite", true); @@ -173,7 +173,7 @@ private static void handleArguments(String[] args) throws IOException { // change the default rips directory Utils.setConfigString("rips.directory", cl.getOptionValue('l')); } - + //Re-rip all previous albums if (cl.hasOption('r')) { // Re-rip all via command-line @@ -386,7 +386,7 @@ private static void loadHistory() throws IOException { } } - /* + /* * @see MainWindow.saveHistory */ private static void saveHistory() { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 3733fb153..0740f62c4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -4,7 +4,10 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.*; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -16,19 +19,23 @@ import java.util.List; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.ui.MainWindow; import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; /** * Simplified ripper, designed for ripping from sites by parsing HTML. */ public abstract class AbstractHTMLRipper extends AbstractRipper { + private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class); + private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); @@ -116,7 +123,7 @@ protected boolean pageContainsAlbums(URL url) { public void rip() throws IOException, URISyntaxException { int index = 0; int textindex = 0; - LOGGER.info("Retrieving " + this.url); + logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); var doc = getCachedFirstPage(); @@ -128,20 +135,20 @@ public void rip() throws IOException, URISyntaxException { // We set doc to null here so the while loop below this doesn't fire doc = null; - LOGGER.debug("Adding items from " + this.url + " to queue"); + logger.debug("Adding items from " + this.url + " to queue"); } List doclocation = new ArrayList<>(); - LOGGER.info("Got doc location " + doc.location()); + logger.info("Got doc location " + doc.location()); while (doc != null) { - LOGGER.info("Processing a doc..."); + logger.info("Processing a doc..."); // catch if we saw a doc location already, save the ones seen in a list if (doclocation.contains(doc.location())) { - LOGGER.info("Already processed location " + doc.location() + " breaking"); + logger.info("Already processed location " + doc.location() + " breaking"); break; } doclocation.add(doc.location()); @@ -151,7 +158,7 @@ public void rip() throws IOException, URISyntaxException { break; } - LOGGER.info("retrieving urls from doc"); + logger.info("retrieving urls from doc"); List imageURLs = getURLsFromPage(doc); // If hasASAPRipping() returns true then the ripper will handle downloading the files @@ -170,7 +177,7 @@ public void rip() throws IOException, URISyntaxException { for (String imageURL : imageURLs) { index += 1; - LOGGER.debug("Found image url #" + index + ": '" + imageURL + "'"); + logger.debug("Found image url #" + index + ": '" + imageURL + "'"); downloadURL(new URI(imageURL).toURL(), index); if (isStopped() || isThisATest()) { break; @@ -178,17 +185,17 @@ public void rip() throws IOException, URISyntaxException { } } if (hasDescriptionSupport() && Utils.getConfigBoolean("descriptions.save", false)) { - LOGGER.debug("Fetching description(s) from " + doc.location()); + logger.debug("Fetching description(s) from " + doc.location()); List textURLs = getDescriptionsFromPage(doc); if (!textURLs.isEmpty()) { - LOGGER.debug("Found description link(s) from " + doc.location()); + logger.debug("Found description link(s) from " + doc.location()); for (String textURL : textURLs) { if (isStopped() || isThisATest()) { break; } textindex += 1; - LOGGER.debug("Getting description from " + textURL); + logger.debug("Getting description from " + textURL); String[] tempDesc = getDescription(textURL,doc); if (tempDesc != null) { @@ -204,11 +211,11 @@ public void rip() throws IOException, URISyntaxException { + ".txt").exists(); if (Utils.getConfigBoolean("file.overwrite", false) || !fileExists) { - LOGGER.debug("Got description from " + textURL); + logger.debug("Got description from " + textURL); saveText(url, "", tempDesc[0], textindex, (tempDesc.length > 1 ? tempDesc[1] : filename)); sleep(descSleepTime()); } else { - LOGGER.debug("Description from " + textURL + " already exists."); + logger.debug("Description from " + textURL + " already exists."); } } @@ -224,14 +231,14 @@ public void rip() throws IOException, URISyntaxException { sendUpdate(STATUS.LOADING_RESOURCE, "next page"); doc = getNextPage(doc); } catch (IOException e) { - LOGGER.info("Can't get next page: " + e.getMessage()); + logger.info("Can't get next page: " + e.getMessage()); break; } } // If they're using a thread pool, wait for it. if (getThreadPool() != null) { - LOGGER.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); + logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); getThreadPool().waitForThreads(); } waitForThreads(); @@ -296,12 +303,12 @@ private boolean saveText(URL url, String subdirectory, String text, int index, S out.write(text.getBytes()); out.close(); } catch (IOException e) { - LOGGER.error("[!] Error creating save file path for description '" + url + "':", e); + logger.error("[!] Error creating save file path for description '" + url + "':", e); return false; } - LOGGER.debug("Downloading " + url + "'s description to " + saveFileAs); + logger.debug("Downloading " + url + "'s description to " + saveFileAs); if (!saveFileAs.getParentFile().exists()) { - LOGGER.info("[+] Creating directory: " + saveFileAs.getParent()); + logger.info("[+] Creating directory: " + saveFileAs.getParent()); saveFileAs.getParentFile().mkdirs(); } return true; @@ -355,7 +362,7 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map itemsPending = Collections.synchronizedMap(new HashMap()); private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); private Map itemsErrored = Collections.synchronizedMap(new HashMap()); @@ -65,18 +70,18 @@ public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException @Override public void rip() throws IOException, URISyntaxException { int index = 0; - LOGGER.info("Retrieving " + this.url); + logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); JSONObject json = getFirstPage(); while (json != null) { List imageURLs = getURLsFromJSON(json); - + if (alreadyDownloadedUrls >= Utils.getConfigInteger("history.end_rip_after_already_seen", 1000000000) && !isThisATest()) { sendUpdate(STATUS.DOWNLOAD_COMPLETE, "Already seen the last " + alreadyDownloadedUrls + " images ending rip"); break; } - + // Remove all but 1 image if (isThisATest()) { while (imageURLs.size() > 1) { @@ -92,9 +97,9 @@ public void rip() throws IOException, URISyntaxException { if (isStopped()) { break; } - + index += 1; - LOGGER.debug("Found image url #" + index+ ": " + imageURL); + logger.debug("Found image url #" + index+ ": " + imageURL); downloadURL(new URI(imageURL).toURL(), index); } @@ -106,14 +111,14 @@ public void rip() throws IOException, URISyntaxException { sendUpdate(STATUS.LOADING_RESOURCE, "next page"); json = getNextPage(json); } catch (IOException | URISyntaxException e) { - LOGGER.info("Can't get next page: " + e.getMessage()); + logger.info("Can't get next page: " + e.getMessage()); break; } } // If they're using a thread pool, wait for it. if (getThreadPool() != null) { - LOGGER.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); + logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); getThreadPool().waitForThreads(); } waitForThreads(); @@ -126,11 +131,11 @@ protected String getPrefix(int index) { } return prefix; } - + /* * ------ Methods copied from AlbumRipper ------ */ - + protected boolean allowDuplicates() { return false; } @@ -159,7 +164,7 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map itemsPending = Collections.synchronizedMap(new HashMap()); private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); private Map itemsErrored = Collections.synchronizedMap(new HashMap()); @@ -69,7 +74,7 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map bakedin_explicit_domains = Arrays.asList( new ChanSite("boards.4chan.org", Arrays.asList("4cdn.org", "is.4chan.org", "is2.4chan.org", "is3.4chan.org")), new ChanSite("boards.4channel.org", Arrays.asList("4cdn.org", "is.4chan.org", "is2.4chan.org", "is3.4chan.org")), @@ -51,12 +57,12 @@ public static List getChansFromConfig(String rawChanString) { if (chanInfo.contains("[")) { String siteUrl = chanInfo.split("\\[")[0]; String[] cdns = chanInfo.replaceAll(siteUrl + "\\[", "").replaceAll("]", "").split("\\|"); - LOGGER.debug("site url: " + siteUrl); - LOGGER.debug("cdn: " + Arrays.toString(cdns)); + logger.debug("site url: " + siteUrl); + logger.debug("cdn: " + Arrays.toString(cdns)); userChans.add(new ChanSite(siteUrl, Arrays.asList(cdns))); } else { // We're parsing a site without cdns - LOGGER.debug("site: " + chanInfo); + logger.debug("site: " + chanInfo); userChans.add(new ChanSite(chanInfo)); } } @@ -78,7 +84,7 @@ public static List getChansFromConfig(String rawChanString) { public ChanRipper(URL url) throws IOException { super(url); for (ChanSite _chanSite : explicit_domains) { - LOGGER.info(_chanSite.domains); + logger.info(_chanSite.domains); if (_chanSite.domains.contains(url.getHost())) { chanSite = _chanSite; generalChanSite = false; @@ -110,11 +116,11 @@ public String getAlbumTitle(URL url) throws MalformedURLException { String subject = doc.select(".post.op > .postinfo > .subject").first().text(); return getHost() + "_" + getGID(url) + "_" + subject; } catch (NullPointerException e) { - LOGGER.warn("Failed to get thread title from " + url); + logger.warn("Failed to get thread title from " + url); } } catch (Exception e) { // Fall back to default album naming convention - LOGGER.warn("Failed to get album title from " + url, e); + logger.warn("Failed to get album title from " + url, e); } // Fall back on the GID return getHost() + "_" + getGID(url); @@ -202,7 +208,7 @@ public Document getFirstPage() throws IOException, URISyntaxException { private boolean isURLBlacklisted(String url) { for (String blacklist_item : url_piece_blacklist) { if (url.contains(blacklist_item)) { - LOGGER.debug("Skipping link that contains '"+blacklist_item+"': " + url); + logger.debug("Skipping link that contains '"+blacklist_item+"': " + url); return true; } } @@ -243,7 +249,7 @@ public List getURLsFromPage(Document page) throws URISyntaxException { } // Don't download the same URL twice if (imageURLs.contains(href)) { - LOGGER.debug("Already attempted: " + href); + logger.debug("Already attempted: " + href); continue; } imageURLs.add(href); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java index c66465eb4..28018ed5e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CheveretoRipper.java @@ -12,6 +12,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -19,6 +21,9 @@ import com.rarchives.ripme.utils.Http; public class CheveretoRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(CheveretoRipper.class); + private static final Map CONSENT_COOKIE; static { CONSENT_COOKIE = new TreeMap(); @@ -60,7 +65,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + title.trim(); } catch (IOException e) { // Fall back to default album naming convention - LOGGER.info("Unable to find title at " + url); + logger.info("Unable to find title at " + url); } return super.getAlbumTitle(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java index 6124cd4f9..1bb2cd83c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/CoomerPartyRipper.java @@ -24,7 +24,9 @@ * See this link for the API schema. */ public class CoomerPartyRipper extends AbstractJSONRipper { - private static final Logger LOGGER = LogManager.getLogger(CoomerPartyRipper.class); + + private static final Logger logger = LogManager.getLogger(CoomerPartyRipper.class); + private static final String IMG_URL_BASE = "https://c3.coomer.su/data"; private static final String VID_URL_BASE = "https://c1.coomer.su/data"; private static final Pattern IMG_PATTERN = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff)$", Pattern.CASE_INSENSITIVE); @@ -62,10 +64,10 @@ public CoomerPartyRipper(URL url) throws IOException { user = pathElements.get(2); if (service == null || user == null || service.isBlank() || user.isBlank()) { - LOGGER.warn("service=" + service + ", user=" + user); + logger.warn("service=" + service + ", user=" + user); throw new MalformedURLException("Invalid coomer.party URL: " + url); } - LOGGER.debug("Parsed service=" + service + " and user=" + user + " from " + url); + logger.debug("Parsed service=" + service + " and user=" + user + " from " + url); } @Override @@ -127,7 +129,7 @@ protected List getURLsFromJSON(JSONObject json) { pullFileUrl(post, urls); pullAttachmentUrls(post, urls); } - LOGGER.debug("Pulled " + urls.size() + " URLs from " + posts.length() + " posts"); + logger.debug("Pulled " + urls.size() + " URLs from " + posts.length() + " posts"); return urls; } @@ -148,11 +150,11 @@ private void pullFileUrl(JSONObject post, ArrayList results) { String url = VID_URL_BASE + path; results.add(url); } else { - LOGGER.error("Unknown extension for coomer.su path: " + path); + logger.error("Unknown extension for coomer.su path: " + path); } } catch (JSONException e) { /* No-op */ - LOGGER.error("Unable to Parse FileURL " + e.getMessage()); + logger.error("Unable to Parse FileURL " + e.getMessage()); } } @@ -165,7 +167,7 @@ private void pullAttachmentUrls(JSONObject post, ArrayList results) { } } catch (JSONException e) { /* No-op */ - LOGGER.error("Unable to Parse AttachmentURL " + e.getMessage()); + logger.error("Unable to Parse AttachmentURL " + e.getMessage()); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java index 9496bb577..765edea43 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java @@ -1,34 +1,33 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractJSONRipper; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import org.jetbrains.annotations.Nullable; -import org.json.JSONArray; -import org.json.JSONObject; -import org.jsoup.Connection; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; +import org.json.JSONObject; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import com.rarchives.ripme.utils.Utils; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; public class DanbooruRipper extends AbstractJSONRipper { + + private static final Logger logger = LogManager.getLogger(DanbooruRipper.class); + private static final String DOMAIN = "danbooru.donmai.us", HOST = "danbooru"; private final OkHttpClient client; @@ -59,7 +58,6 @@ private String getPage(int num) throws MalformedURLException { return "https://" + getDomain() + "/posts.json?page=" + num + "&tags=" + getTag(url); } - private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0"; @Override protected JSONObject getFirstPage() throws MalformedURLException { return getCurrentPage(); @@ -123,7 +121,7 @@ public String getGID(URL url) throws MalformedURLException { try { return Utils.filesystemSafe(new URI(getTag(url).replaceAll("([?&])tags=", "")).getPath()); } catch (URISyntaxException ex) { - LOGGER.error(ex); + logger.error(ex); } throw new MalformedURLException("Expected booru URL format: " + getDomain() + "/posts?tags=searchterm - got " + url + " instead"); @@ -144,5 +142,4 @@ private String getTag(URL url) throws MalformedURLException { throw new MalformedURLException("Expected danbooru URL format: " + getDomain() + "/posts?tags=searchterm - got " + url + " instead"); } - } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 9f26a2681..32ab17b59 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -1,11 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; - import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -25,6 +19,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.Connection; import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; @@ -33,57 +29,65 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; + /** - * + * * @author MrPlaygon - * + * * NOT using Deviantart API like the old JSON ripper because it is SLOW * and somehow annoying to use. Things to consider: Using the API might * be less work/maintenance later because APIs do not change as * frequently as HTML source code does...? - * - * - * + * + * + * * Tested for: - * + * * SFW: - * + * * https://www.deviantart.com/apofiss/gallery/41388863/sceneries * https://www.deviantart.com/kageuri/gallery/ * https://www.deviantart.com/kageuri/gallery/?catpath=/ * https://www.deviantart.com/apofiss/favourites/39881418/gifts-and * https://www.deviantart.com/kageuri/favourites/ * https://www.deviantart.com/kageuri/favourites/?catpath=/ - * + * * NSFW: - * + * * https://www.deviantart.com/revpeng/gallery/67734353/Siren-Lee-Agent-of-S-I-R-E-N-S - * - * + * + * * Deactivated account: - * + * * https://www.deviantart.com/gingerbreadpony/gallery - * + * * Banned Account: - * + * * https://www.deviantart.com/ghostofflossenburg/gallery - * - * - * - * + * + * + * + * * Login Data (PLEASE DONT ACTUALLY USE!!!): - * + * * email: 5g5_8l4dii5lbbpc@byom.de - * + * * username: 5g58l4dii5lbbpc - * + * * password: 5g5_8l4dii5lbbpc - * - * + * + * * */ public class DeviantartRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(DeviantartRipper.class); + private final String username = "5g58l4dii5lbbpc"; private final String password = "5g5_8l4dii5lbbpc"; private int offset = 0; @@ -137,7 +141,7 @@ protected Document getFirstPage() throws IOException { /** * Checks if the URL refers to a deactivated account using the HTTP status Codes - * + * * @return true when the account is good * @throws IOException when the account is deactivated */ @@ -150,9 +154,9 @@ private boolean isDeactivated() throws IOException { /** * Stores logged in Cookies. Needed for art pieces only visible to logged in - * users. - * - * + * users. + * + * * @throws IOException when failed to load webpage or failed to read/write * cookies in file (used when running multiple instances of * RipMe) @@ -168,11 +172,11 @@ private void login() throws IOException { e.printStackTrace(); } if (getDACookie() == null || !checkLogin()) { - LOGGER.info("Do Login now"); + logger.info("Do Login now"); // Do login now Map tmpCookies = new HashMap(); - + // Load login page Response res = Http.url("https://www.deviantart.com/users/login").connection().method(Method.GET) .referrer(referer).userAgent(userAgent).execute(); @@ -181,13 +185,13 @@ private void login() throws IOException { // Find tokens Document doc = res.parse(); - + tmpCookies.putAll(res.cookies()); - + Element form = doc.getElementById("login"); String token = form.select("input[name=\"validate_token\"]").first().attr("value"); String key = form.select("input[name=\"validate_key\"]").first().attr("value"); - LOGGER.info("Token: " + token + " & Key: " + key); + logger.info("Token: " + token + " & Key: " + key); // Build Login Data HashMap loginData = new HashMap(); @@ -201,7 +205,7 @@ private void login() throws IOException { // Log in using data. Handle redirect res = Http.url("https://www.deviantart.com/users/login").connection().referrer(referer).userAgent(userAgent) .method(Method.POST).data(loginData).cookies(tmpCookies).followRedirects(false).execute(); - + tmpCookies.putAll(res.cookies()); res = Http.url(res.header("location")).connection().referrer(referer).userAgent(userAgent) @@ -209,17 +213,17 @@ private void login() throws IOException { // Store cookies tmpCookies.putAll(res.cookies()); - + updateCookie(tmpCookies); - + } else { - LOGGER.info("No new Login needed"); + logger.info("No new Login needed"); } - LOGGER.info("DA Cookies: " + getDACookie()); + logger.info("DA Cookies: " + getDACookie()); } - + /** * Returns next page Document using offset. @@ -232,12 +236,12 @@ public Document getNextPage(Document doc) throws IOException { //updateCookie(re.cookies()); Document docu = re.parse(); Elements messages = docu.getElementsByClass("message"); - LOGGER.info("Current Offset: " + this.offset); + logger.info("Current Offset: " + this.offset); if (messages.size() > 0) { // if message exists -> last page - LOGGER.info("Messages amount: " + messages.size() + " - Next Page does not exists"); + logger.info("Messages amount: " + messages.size() + " - Next Page does not exists"); throw new IOException("No more pages"); } @@ -248,9 +252,9 @@ public Document getNextPage(Document doc) throws IOException { /** * Returns list of Links to the Image pages. NOT links to fullsize image!!! e.g. * https://www.deviantart.com/kageuri/art/RUBY-568396655 - * + * * @param page Page of album with multiple images - * + * */ @Override protected List getURLsFromPage(Document page) { @@ -272,22 +276,22 @@ protected List getURLsFromPage(Document page) { } - LOGGER.info("Amount of Images on Page: " + result.size()); - LOGGER.info(page.location()); + logger.info("Amount of Images on Page: " + result.size()); + logger.info(page.location()); return result; } /** * Starts new Thread to find download link + filename + filetype - * + * * @param url The URL to an image site. */ @Override protected void downloadURL(URL url, int index) { this.downloadCount += 1; - LOGGER.info("Downloading URL Number " + this.downloadCount); - LOGGER.info("Deviant Art URL: " + url.toExternalForm()); + logger.info("Downloading URL Number " + this.downloadCount); + logger.info("Deviant Art URL: " + url.toExternalForm()); try { Response re = Http.url(urlWithParams(this.offset)).cookies(getDACookie()).referrer(referer) .userAgent(userAgent).response(); @@ -310,7 +314,7 @@ public String normalizeUrl(String url) { /** * Returns name of album. Album name consists of 3 words: - Artist (owner of * gallery) - Type (gallery or favorites folder) - Name of the folder - * + * * Returns artist_type_name */ @Override @@ -359,14 +363,14 @@ public String getGID(URL url) throws MalformedURLException { } else if (artistM.matches()) { albumname = artistM.group(1); } - LOGGER.info("Album Name: " + artist + "_" + what + "_" + albumname); + logger.info("Album Name: " + artist + "_" + what + "_" + albumname); return artist + "_" + what + "_" + albumname; } /** - * + * * @return Clean URL as String */ private String cleanURL() { @@ -376,7 +380,7 @@ private String cleanURL() { /** * Return correct url with params (catpath) and current offset * Offset misleasing because it might say 24 but it is not the 24th image. (DA site is bugged I guess) - * + * * @return URL to page with offset */ private URL urlWithParams(int offset) { @@ -396,7 +400,7 @@ private URL urlWithParams(int offset) { /** * Returns Hashmap usable as Cookie for NSFW Artworks. Method Not really needed but * maybe useful later. - * + * * @return Cookie Hashmap */ private Map getDACookie() { @@ -405,7 +409,7 @@ private Map getDACookie() { /** * Updates cookies and saves to config file. - * + * * @param m new Cookies */ private void updateCookie(Map m) { @@ -422,12 +426,12 @@ private void updateCookie(Map m) { } }*/ - LOGGER.info("Updating Cookies"); - LOGGER.info("Old Cookies: " + getDACookie() + " "); - LOGGER.info("New Cookies: " + m + " "); + logger.info("Updating Cookies"); + logger.info("Old Cookies: " + getDACookie() + " "); + logger.info("New Cookies: " + m + " "); this.cookies.putAll(m); this.cookies.put("agegate_state", "1"); - LOGGER.info("Merged Cookies: " + getDACookie() + " "); + logger.info("Merged Cookies: " + getDACookie() + " "); try { Utils.setConfigString(utilsKey, serialize(new HashMap(getDACookie()))); @@ -442,7 +446,7 @@ private void updateCookie(Map m) { * Serializes an Object and returns a String ready to store Used to store * cookies in the config file because the deviantart cookies contain all sort of * special characters like ; , = : and so on. - * + * * @param o Object to serialize * @return The serialized base64 encoded object * @throws IOException @@ -457,7 +461,7 @@ private String serialize(Serializable o) throws IOException { /** * Recreates the object from the base64 encoded String. Used for Cookies - * + * * @param s the Base64 encoded string * @return the Cookie Map * @throws IOException @@ -473,30 +477,30 @@ private Map deserialize(String s) throws IOException, ClassNotFo } /** - * Checks if the current cookies are still valid/usable. + * Checks if the current cookies are still valid/usable. * Also checks if agegate is given. - * - * + * + * * @return True when all is good. */ private boolean checkLogin() { if (!getDACookie().containsKey("agegate_state")) { - LOGGER.info("No agegate key"); + logger.info("No agegate key"); return false; } else if (!getDACookie().get("agegate_state").equals("1")) { // agegate == 1 -> all is fine. NSFW is visible - LOGGER.info("Wrong agegate value"); + logger.info("Wrong agegate value"); return false; } try { - LOGGER.info("Login with Cookies: " + getDACookie()); + logger.info("Login with Cookies: " + getDACookie()); Response res = Http.url("https://www.deviantart.com/users/login").connection().followRedirects(true) .cookies(getDACookie()).referrer(this.referer).userAgent(this.userAgent).execute(); if (!res.url().toExternalForm().equals("https://www.deviantart.com/users/login") && !res.url().toExternalForm().startsWith("https://www.deviantart.com/users/wrong-password")) { - LOGGER.info("Cookies are valid: " + res.url()); + logger.info("Cookies are valid: " + res.url()); return true; } else { - LOGGER.info("Cookies invalid. Wrong URL: " + res.url() + " " + res.statusCode()); + logger.info("Cookies invalid. Wrong URL: " + res.url() + " " + res.statusCode()); return false; } } catch (IOException e) { @@ -508,13 +512,13 @@ private boolean checkLogin() { /** * Analyzes an image page like * https://www.deviantart.com/kageuri/art/RUBY-568396655 . - * + * * Looks for download button, follows the authentications and redirects and adds * the Image URL to the download queue. If no download button is present it will * use the largest version of the image. - * + * * Should work with all filetypes on Deviantart. Tested with .JPG .PNG and .PDF - * + * * @author MrPlaygon * */ @@ -532,15 +536,15 @@ public void run() { /** * Get URL to Artwork and return fullsize URL with file ending. - * + * * @return URL like * https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/07f7a6bb-2d35-4630-93fc-be249af22b3e/d7jak0y-d20e5932-df72-4d13-b002-5e122037b373.jpg - * - * + * + * */ private void getFullSizeURL() { - LOGGER.info("Searching max. Resolution for " + url); + logger.info("Searching max. Resolution for " + url); sendUpdate(STATUS.LOADING_RESOURCE, "Searching max. resolution for " + url); try { Response re = Http.url(url).connection().referrer(referer).userAgent(userAgent).cookies(getDACookie()) @@ -567,7 +571,7 @@ private void getFullSizeURL() { // Download Button if (downloadButton != null) { - LOGGER.info("Download Button found for "+ url +" : " + downloadButton.attr("href")); + logger.info("Download Button found for "+ url +" : " + downloadButton.attr("href")); Response download = Http.url(downloadButton.attr("href")).connection().cookies(getDACookie()) .method(Method.GET).referrer(referer).userAgent(userAgent).ignoreContentType(true) @@ -576,9 +580,9 @@ private void getFullSizeURL() { String[] filetypePart = download.header("Content-Disposition").split("\\."); - LOGGER.info("Found Image URL"); - LOGGER.info(url); - LOGGER.info(location); + logger.info("Found Image URL"); + logger.info(url); + logger.info(location); addURLToDownload(location, "", "", "", getDACookie(), title + "." + filetypePart[filetypePart.length - 1]); @@ -586,18 +590,18 @@ private void getFullSizeURL() { } // No Download Button - LOGGER.info("No Download Button for: "+ url); - + logger.info("No Download Button for: "+ url); + Element div = doc.select("div.dev-view-deviation").first(); Element image = div.getElementsByTag("img").first(); String scaledImage = ""; if (image == null) { - LOGGER.error("ERROR on " + url); + logger.error("ERROR on " + url); - LOGGER.error("Cookies: " + getDACookie() + " "); - LOGGER.error(div); + logger.error("Cookies: " + getDACookie() + " "); + logger.error(div); sendUpdate(STATUS.DOWNLOAD_ERRORED, "ERROR at\n" + url); return; } @@ -605,20 +609,20 @@ private void getFullSizeURL() { // When it is text art (e.g. story) the only image is the profile // picture if (image.hasClass("avatar")) { - LOGGER.error("No Image found, probably text art: " + url); + logger.error("No Image found, probably text art: " + url); return; } scaledImage = image.attr("src").split("\\?")[0]; String[] parts = scaledImage.split("/v1/"); // Image page uses scaled down version. Split at /v1/ to receive max size. - + if (parts.length > 2) { - LOGGER.error("Unexpected URL Format"); + logger.error("Unexpected URL Format"); sendUpdate(STATUS.DOWNLOAD_ERRORED, "Unexpected URL Format"); return; } - + String originalImage = parts[0]; // URL to original image without scaling (works not alwys. weird 404 errors.) String downloadString = originalImage; // this works always try { @@ -627,7 +631,7 @@ private void getFullSizeURL() { downloadString = scaledImage; //revert back to save url because of error } String[] tmpParts = downloadString.split("\\."); //split to get file ending - + addURLToDownload(new URI(downloadString).toURL(), "", "", "", new HashMap(), title + "." + tmpParts[tmpParts.length - 1]); return; @@ -636,11 +640,11 @@ private void getFullSizeURL() { e.printStackTrace(); } - LOGGER.error("No Full Size URL for: " + url); + logger.error("No Full Size URL for: " + url); sendUpdate(STATUS.DOWNLOAD_ERRORED, "No image found for " + url); return; } } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 209afe598..9b40f0542 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -27,6 +27,7 @@ import com.rarchives.ripme.utils.Utils; public class E621Ripper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(E621Ripper.class); private static Pattern gidPattern = null; @@ -163,7 +164,7 @@ private String getTerm(URL url) throws MalformedURLException { Matcher m = gidPattern.matcher(url.toExternalForm()); if (m.matches()) { - LOGGER.info(m.group(2)); + logger.info(m.group(2)); return m.group(2); } @@ -174,13 +175,13 @@ private String getTerm(URL url) throws MalformedURLException { m = gidPatternNew.matcher(url.toExternalForm()); if (m.matches()) { - LOGGER.info(m.group(3)); + logger.info(m.group(3)); return m.group(3); } m = gidPatternPoolNew.matcher(url.toExternalForm()); if (m.matches()) { - LOGGER.info(m.group(2)); + logger.info(m.group(2)); return m.group(2); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index cccfeb09f..5349f55c7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -1,16 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.RipUtils; -import com.rarchives.ripme.utils.Utils; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; @@ -25,7 +14,24 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.ui.RipStatusMessage.STATUS; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.RipUtils; +import com.rarchives.ripme.utils.Utils; + public class EHentaiRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(EHentaiRipper.class); + // All sleep times are in milliseconds private static final int PAGE_SLEEP_TIME = 3000; private static final int IMAGE_SLEEP_TIME = 1500; @@ -72,7 +78,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + elems.first().text(); } catch (Exception e) { // Fall back to default album naming convention - LOGGER.warn("Failed to get album title from " + url, e); + logger.warn("Failed to get album title from " + url, e); } return super.getAlbumTitle(url); } @@ -99,7 +105,7 @@ private Document getPageWithRetries(URL url) throws IOException { int retries = 3; while (true) { sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); - LOGGER.info("Retrieving " + url); + logger.info("Retrieving " + url); doc = Http.url(url) .referrer(this.url) .cookies(cookies) @@ -108,7 +114,7 @@ private Document getPageWithRetries(URL url) throws IOException { if (retries == 0) { throw new IOException("Hit rate limit and maximum number of retries, giving up"); } - LOGGER.warn("Hit rate limit while loading " + url + ", sleeping for " + IP_BLOCK_SLEEP_TIME + "ms, " + retries + " retries remaining"); + logger.warn("Hit rate limit while loading " + url + ", sleeping for " + IP_BLOCK_SLEEP_TIME + "ms, " + retries + " retries remaining"); retries--; try { Thread.sleep(IP_BLOCK_SLEEP_TIME); @@ -123,9 +129,9 @@ private Document getPageWithRetries(URL url) throws IOException { public List getTags(Document doc) { List tags = new ArrayList<>(); - LOGGER.info("Getting tags"); + logger.info("Getting tags"); for (Element tag : doc.select("td > div > a")) { - LOGGER.info("Found tag " + tag.text()); + logger.info("Found tag " + tag.text()); tags.add(tag.text()); } return tags; @@ -138,7 +144,7 @@ public Document getFirstPage() throws IOException { albumDoc = getPageWithRetries(this.url); } this.lastURL = this.url.toExternalForm(); - LOGGER.info("Checking blacklist"); + logger.info("Checking blacklist"); String blacklistedTag = RipUtils.checkTags(Utils.getConfigStringArray("ehentai.blacklist.tags"), getTags(albumDoc)); if (blacklistedTag != null) { sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Skipping " + url.toExternalForm() + " as it " + @@ -157,13 +163,13 @@ public Document getNextPage(Document doc) throws IOException, URISyntaxException // Find next page Elements hrefs = doc.select(".ptt a"); if (hrefs.isEmpty()) { - LOGGER.info("doc: " + doc.html()); + logger.info("doc: " + doc.html()); throw new IOException("No navigation links found"); } // Ensure next page is different from the current page String nextURL = hrefs.last().attr("href"); if (nextURL.equals(this.lastURL)) { - LOGGER.info("lastURL = nextURL : " + nextURL); + logger.info("lastURL = nextURL : " + nextURL); throw new IOException("Reached last page of results"); } // Sleep before loading next page @@ -192,7 +198,7 @@ public void downloadURL(URL url, int index) { try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { - LOGGER.warn("Interrupted while waiting to load next image", e); + logger.warn("Interrupted while waiting to load next image", e); } } @@ -228,13 +234,13 @@ private void fetchImage() { // Attempt to find image elsewise (Issue #41) images = doc.select("img#img"); if (images.isEmpty()) { - LOGGER.warn("Image not found at " + this.url); + logger.warn("Image not found at " + this.url); return; } } Element image = images.first(); String imgsrc = image.attr("src"); - LOGGER.info("Found URL " + imgsrc + " via " + images.get(0)); + logger.info("Found URL " + imgsrc + " via " + images.get(0)); Pattern p = Pattern.compile("^http://.*/ehg/image.php.*&n=([^&]+).*$"); Matcher m = p.matcher(imgsrc); if (m.matches()) { @@ -254,7 +260,7 @@ private void fetchImage() { addURLToDownload(new URI(imgsrc).toURL(), prefix); } } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); + logger.error("[!] Exception while loading/parsing " + this.url, e); } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java index 7cfd568f3..8765c4daf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EightmusesRipper.java @@ -1,7 +1,10 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; -import java.net.*; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -9,6 +12,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -20,6 +25,8 @@ public class EightmusesRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(EightmusesRipper.class); + private Map cookies = new HashMap<>(); public EightmusesRipper(URL url) throws IOException { @@ -62,7 +69,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + title.trim(); } catch (IOException e) { // Fall back to default album naming convention - LOGGER.info("Unable to find title at " + url); + logger.info("Unable to find title at " + url); } return super.getAlbumTitle(url); } @@ -85,19 +92,19 @@ public List getURLsFromPage(Document page) { if (thumb.attr("href").contains("/comics/album/")) { String subUrl = "https://www.8muses.com" + thumb.attr("href"); try { - LOGGER.info("Retrieving " + subUrl); + logger.info("Retrieving " + subUrl); sendUpdate(STATUS.LOADING_RESOURCE, subUrl); Document subPage = Http.url(subUrl).get(); // If the page below this one has images this line will download them List subalbumImages = getURLsFromPage(subPage); - LOGGER.info("Found " + subalbumImages.size() + " images in subalbum"); + logger.info("Found " + subalbumImages.size() + " images in subalbum"); } catch (IOException e) { - LOGGER.warn("Error while loading subalbum " + subUrl, e); + logger.warn("Error while loading subalbum " + subUrl, e); } } else if (thumb.attr("href").contains("/comics/picture/")) { - LOGGER.info("This page is a album"); - LOGGER.info("Ripping image"); + logger.info("This page is a album"); + logger.info("Ripping image"); if (super.isStopped()) break; // Find thumbnail image source String image = null; @@ -110,8 +117,8 @@ public List getURLsFromPage(Document page) { URL imageUrl = new URI(image).toURL(); addURLToDownload(imageUrl, getSubdir(page.select("title").text()), this.url.toExternalForm(), cookies, getPrefixShort(i), "", null, true); } catch (MalformedURLException | URISyntaxException e) { - LOGGER.error("\"" + image + "\" is malformed"); - LOGGER.error(e.getMessage()); + logger.error("\"" + image + "\" is malformed"); + logger.error(e.getMessage()); } } if (!image.contains("8muses.com")) { @@ -127,7 +134,7 @@ public List getURLsFromPage(Document page) { } public String getSubdir(String rawHref) { - LOGGER.info("Raw title: " + rawHref); + logger.info("Raw title: " + rawHref); String title = rawHref; title = title.replaceAll("8muses - Sex and Porn Comics", ""); title = title.replaceAll("\\s+", " "); @@ -135,7 +142,7 @@ public String getSubdir(String rawHref) { title = title.replaceAll("\\| ", ""); title = title.replace(" - ", "-"); title = title.replace(" ", "-"); - LOGGER.info(title); + logger.info(title); return title; } @@ -151,4 +158,4 @@ public String getPrefixLong(int index) { public String getPrefixShort(int index) { return String.format("%03d", index); } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java index 95528470c..fe063ad85 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java @@ -1,12 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; @@ -19,8 +12,20 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.utils.Http; + public class ErofusRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(ErofusRipper.class); + public ErofusRipper(URL url) throws IOException { super(url); } @@ -52,11 +57,11 @@ public String getGID(URL url) throws MalformedURLException { @Override public List getURLsFromPage(Document page) { - LOGGER.info(page); + logger.info(page); List imageURLs = new ArrayList<>(); int x = 1; if (pageContainsImages(page)) { - LOGGER.info("Page contains images"); + logger.info("Page contains images"); ripAlbum(page); } else { // This contains the thumbnails of all images on the page @@ -66,12 +71,12 @@ public List getURLsFromPage(Document page) { if (pageLink.attr("href").contains("comics")) { String subUrl = "https://erofus.com" + pageLink.attr("href"); try { - LOGGER.info("Retrieving " + subUrl); + logger.info("Retrieving " + subUrl); sendUpdate(RipStatusMessage.STATUS.LOADING_RESOURCE, subUrl); Document subPage = Http.url(subUrl).get(); List subalbumImages = getURLsFromPage(subPage); } catch (IOException e) { - LOGGER.warn("Error while loading subalbum " + subUrl, e); + logger.warn("Error while loading subalbum " + subUrl, e); } } if (isThisATest()) break; @@ -93,7 +98,7 @@ public void ripAlbum(Document page) { opts.put("prefix", getPrefix(x)); addURLToDownload(new URI(image).toURL(), opts); } catch (MalformedURLException | URISyntaxException e) { - LOGGER.info(e.getMessage()); + logger.info(e.getMessage()); } x++; } @@ -113,4 +118,4 @@ private boolean pageContainsImages(Document page) { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java index 3035d7465..fd5ddb85f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java @@ -11,13 +11,15 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; /** * @@ -25,6 +27,8 @@ */ public class EromeRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(EromeRipper.class); + boolean rippingProfile; private HashMap cookies = new HashMap<>(); @@ -78,7 +82,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + getGID(url) + "_" + title.trim(); } catch (IOException e) { // Fall back to default album naming convention - LOGGER.info("Unable to find title at " + url); + logger.info("Unable to find title at " + url); } catch (NullPointerException e) { return getHost() + "_" + getGID(url); } @@ -158,7 +162,7 @@ private List getMediaFromPage(Document doc) { if (results.size() == 0) { if (cookies.isEmpty()) { - LOGGER.warn("You might try setting erome.laravel_session manually " + + logger.warn("You might try setting erome.laravel_session manually " + "if you think this page definitely contains media."); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java index bba284f14..daf66ca69 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FivehundredpxRipper.java @@ -1,7 +1,11 @@ package com.rarchives.ripme.ripper.rippers; import java.io.IOException; -import java.net.*; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -9,6 +13,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.nodes.Document; @@ -37,6 +43,8 @@ */ public class FivehundredpxRipper extends AbstractJSONRipper { + private static final Logger logger = LogManager.getLogger(FivehundredpxRipper.class); + private int page = 1; private String baseURL = "https://api.500px.com/v1"; private static final String CONSUMER_KEY = "XPm2br2zGBq6TOfd2xbDIHYoLnt3cLxr1HYryGCv"; @@ -152,7 +160,7 @@ public String getGID(URL url) throws MalformedURLException { /** Convert username to UserID. */ private String getUserID(String username) throws IOException { - LOGGER.info("Fetching user ID for " + username); + logger.info("Fetching user ID for " + username); JSONObject json = new Http("https://api.500px.com/v1/" + "users/show" + "?username=" + username + @@ -164,7 +172,7 @@ private String getUserID(String username) throws IOException { @Override public JSONObject getFirstPage() throws IOException, URISyntaxException { URL apiURL = new URI(baseURL + "&consumer_key=" + CONSUMER_KEY).toURL(); - LOGGER.debug("apiURL: " + apiURL); + logger.debug("apiURL: " + apiURL); JSONObject json = Http.url(apiURL).getJSON(); if (baseURL.contains("/galleries?")) { @@ -184,7 +192,7 @@ public JSONObject getFirstPage() throws IOException, URISyntaxException { + "?rpp=100" + "&image_size=5" + "&consumer_key=" + CONSUMER_KEY; - LOGGER.info("Loading " + blogURL); + logger.info("Loading " + blogURL); sendUpdate(STATUS.LOADING_RESOURCE, "Gallery ID " + galleryID + " for userID " + userID); JSONObject thisJSON = Http.url(blogURL).getJSON(); JSONArray thisPhotos = thisJSON.getJSONArray("photos"); @@ -215,7 +223,7 @@ else if (baseURL.contains("/blogs?")) { + "&rpp=100" + "&image_size=5" + "&consumer_key=" + CONSUMER_KEY; - LOGGER.info("Loading " + blogURL); + logger.info("Loading " + blogURL); sendUpdate(STATUS.LOADING_RESOURCE, "Story ID " + blogid + " for user " + username); JSONObject thisJSON = Http.url(blogURL).getJSON(); JSONArray thisPhotos = thisJSON.getJSONArray("photos"); @@ -267,20 +275,20 @@ public List getURLsFromJSON(JSONObject json) { Document doc; Elements images = new Elements(); try { - LOGGER.debug("Loading " + rawUrl); + logger.debug("Loading " + rawUrl); super.retrievingSource(rawUrl); doc = Http.url(rawUrl).get(); images = doc.select("div#preload img"); } catch (IOException e) { - LOGGER.error("Error fetching full-size image from " + rawUrl, e); + logger.error("Error fetching full-size image from " + rawUrl, e); } if (!images.isEmpty()) { imageURL = images.first().attr("src"); - LOGGER.debug("Found full-size non-watermarked image: " + imageURL); + logger.debug("Found full-size non-watermarked image: " + imageURL); } else { - LOGGER.debug("Falling back to image_url from API response"); + logger.debug("Falling back to image_url from API response"); imageURL = photo.getString("image_url"); imageURL = imageURL.replaceAll("/4\\.", "/5."); // See if there's larger images @@ -288,7 +296,7 @@ public List getURLsFromJSON(JSONObject json) { String fsURL = imageURL.replaceAll("/5\\.", "/" + imageSize + "."); sleep(10); if (urlExists(fsURL)) { - LOGGER.info("Found larger image at " + fsURL); + logger.info("Found larger image at " + fsURL); imageURL = fsURL; break; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index c58a7e717..1f1954207 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -5,23 +5,29 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ui.RipStatusMessage; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; -import org.json.JSONObject; import org.json.JSONException; +import org.json.JSONObject; import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Element; public class FlickrRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(FlickrRipper.class); + private final DownloadThreadPool flickrThreadPool; private enum UrlType { @@ -87,11 +93,11 @@ private String getAPIKey(Document doc) { // You have to use .html here as .text will strip most of the javascript m = p.matcher(e.html()); if (m.find()) { - LOGGER.info("Found api key:" + m.group(1)); + logger.info("Found api key:" + m.group(1)); return m.group(1); } } - LOGGER.error("Unable to get api key"); + logger.error("Unable to get api key"); // A nice error message to tell our users what went wrong sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Unable to extract api key from flickr"); sendUpdate(RipStatusMessage.STATUS.DOWNLOAD_WARN, "Using hardcoded api key"); @@ -132,14 +138,14 @@ private JSONObject getJSON(String page, String apiKey) { apiURL = apiURLBuilder(getAlbum(url.toExternalForm()), page, apiKey); pageURL = new URI(apiURL).toURL(); } catch (MalformedURLException | URISyntaxException e) { - LOGGER.error("Unable to get api link " + apiURL + " is malformed"); + logger.error("Unable to get api link " + apiURL + " is malformed"); } try { - LOGGER.info("Fetching: " + apiURL); - LOGGER.info("Response: " + Http.url(pageURL).ignoreContentType().get().text()); + logger.info("Fetching: " + apiURL); + logger.info("Response: " + Http.url(pageURL).ignoreContentType().get().text()); return new JSONObject(Http.url(pageURL).ignoreContentType().get().text()); } catch (IOException e) { - LOGGER.error("Unable to get api link " + apiURL + " is malformed"); + logger.error("Unable to get api link " + apiURL + " is malformed"); return null; } } @@ -168,7 +174,7 @@ private Album getAlbum(String url) throws MalformedURLException { String errorMessage = "Failed to extract photoset ID from url: " + url; - LOGGER.error(errorMessage); + logger.error(errorMessage); throw new MalformedURLException(errorMessage); } @@ -177,7 +183,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce if (!url.toExternalForm().contains("/sets/")) { return super.getAlbumTitle(url); } - try { + try { // Attempt to use album title as GID Document doc = getCachedFirstPage(); String user = url.toExternalForm(); @@ -249,21 +255,21 @@ public List getURLsFromPage(Document doc) { try { rootData = jsonData.getJSONObject("photos"); } catch (JSONException innerE) { - LOGGER.error("Unable to find photos in response"); + logger.error("Unable to find photos in response"); break; } } int totalPages = rootData.getInt("pages"); - LOGGER.info(jsonData); + logger.info(jsonData); JSONArray pictures = rootData.getJSONArray("photo"); for (int i = 0; i < pictures.length(); i++) { - LOGGER.info(i); + logger.info(i); JSONObject data = (JSONObject) pictures.get(i); try { addURLToDownload(getLargestImageURL(data.getString("id"), apiKey)); } catch (MalformedURLException | URISyntaxException e) { - LOGGER.error("Flickr MalformedURLException: " + e.getMessage()); + logger.error("Flickr MalformedURLException: " + e.getMessage()); } } @@ -297,11 +303,11 @@ private URL getLargestImageURL(String imageID, String apiKey) throws MalformedUR } } catch (org.json.JSONException e) { - LOGGER.error("Error in parsing of Flickr API: " + e.getMessage()); + logger.error("Error in parsing of Flickr API: " + e.getMessage()); } catch (MalformedURLException e) { - LOGGER.error("Malformed URL returned by API"); + logger.error("Malformed URL returned by API"); } catch (IOException e) { - LOGGER.error("IOException while looking at image sizes: " + e.getMessage()); + logger.error("IOException while looking at image sizes: " + e.getMessage()); } return new URI(imageURLMap.lastEntry().getValue()).toURL(); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index dbb46fe1c..ad9075d0d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -1,5 +1,7 @@ package com.rarchives.ripme.ripper.rippers; +import static com.rarchives.ripme.utils.RipUtils.getCookiesFromString; + import java.io.IOException; import java.io.OutputStream; import java.net.MalformedURLException; @@ -14,8 +16,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.Connection.Response; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -25,18 +27,20 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Http; - -import static com.rarchives.ripme.utils.RipUtils.getCookiesFromString; +import com.rarchives.ripme.utils.Utils; public class FuraffinityRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(FuraffinityRipper.class); + private static final String urlBase = "https://www.furaffinity.net"; private Map cookies = new HashMap<>(); private void setCookies() { if (Utils.getConfigBoolean("furaffinity.login", true)) { - LOGGER.info("Logging in using cookies"); + logger.info("Logging in using cookies"); String faCookies = Utils.getConfigString("furaffinity.cookies", "a=897bc45b-1f87-49f1-8a85-9412bc103e7a;b=c8807f36-7a85-4caf-80ca-01c2a2368267"); warnAboutSharedAccount(faCookies); cookies = getCookiesFromString(faCookies); @@ -104,7 +108,7 @@ private String getImageFromPost(String url) { Elements links = d.getElementsByTag("a"); for (Element link : links) { if (link.text().equals("Download")) { - LOGGER.info("Found image " + link.attr("href")); + logger.info("Found image " + link.attr("href")); return "https:" + link.attr("href"); } } @@ -137,7 +141,7 @@ public List getDescriptionsFromPage(Document page) { Elements urlElements = page.select("figure.t-image > b > u > a"); for (Element e : urlElements) { urls.add(urlBase + e.select("a").first().attr("href")); - LOGGER.debug("Desc2 " + urlBase + e.select("a").first().attr("href")); + logger.debug("Desc2 " + urlBase + e.select("a").first().attr("href")); } return urls; } @@ -156,21 +160,21 @@ public String getDescription(String page) { // Try to find the description Elements els = resp.parse().select("td[class=alt1][width=\"70%\"]"); if (els.isEmpty()) { - LOGGER.debug("No description at " + page); + logger.debug("No description at " + page); throw new IOException("No description found"); } - LOGGER.debug("Description found!"); + logger.debug("Description found!"); Document documentz = resp.parse(); Element ele = documentz.select("td[class=alt1][width=\"70%\"]").get(0); // This is where the description is. // Would break completely if FurAffinity changed site layout. documentz.outputSettings(new Document.OutputSettings().prettyPrint(false)); ele.select("br").append("\\n"); ele.select("p").prepend("\\n\\n"); - LOGGER.debug("Returning description at " + page); + logger.debug("Returning description at " + page); String tempPage = Jsoup.clean(ele.html().replaceAll("\\\\n", System.getProperty("line.separator")), "", Safelist.none(), new Document.OutputSettings().prettyPrint(false)); return documentz.select("meta[property=og:title]").attr("content") + "\n" + tempPage; // Overridden saveText takes first line and makes it the file name. } catch (IOException ioe) { - LOGGER.info("Failed to get description " + page + " : '" + ioe.getMessage() + "'"); + logger.info("Failed to get description " + page + " : '" + ioe.getMessage() + "'"); return null; } } @@ -203,12 +207,12 @@ public boolean saveText(URL url, String subdirectory, String text, int index) { out.write(text.getBytes()); out.close(); } catch (IOException e) { - LOGGER.error("[!] Error creating save file path for description '" + url + "':", e); + logger.error("[!] Error creating save file path for description '" + url + "':", e); return false; } - LOGGER.debug("Downloading " + url + "'s description to " + saveFileAs); + logger.debug("Downloading " + url + "'s description to " + saveFileAs); if (!Files.exists(saveFileAs.getParent())) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); + logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); try { Files.createDirectory(saveFileAs.getParent()); } catch (IOException e) { @@ -242,4 +246,4 @@ public String getGID(URL url) throws MalformedURLException { } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java index 62a60fccd..12bedc256 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuskatorRipper.java @@ -11,6 +11,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Connection.Method; @@ -22,6 +24,8 @@ public class FuskatorRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(FuskatorRipper.class); + private String jsonurl = "https://fuskator.com/ajax/gal.aspx"; private String xAuthUrl = "https://fuskator.com/ajax/auth.aspx"; private String xAuthToken; @@ -87,7 +91,7 @@ public List getURLsFromPage(Document doc) { json = Http.url(jsonurl).cookies(cookies).data("X-Auth", xAuthToken).data("hash", getGID(url)) .data("_", Long.toString(System.currentTimeMillis())).getJSON(); } catch (IOException e) { - LOGGER.error("Couldnt fetch images.", e.getCause()); + logger.error("Couldnt fetch images.", e.getCause()); return imageURLs; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java index 49cbfc604..e6983febd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/GirlsOfDesireRipper.java @@ -13,10 +13,15 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.rarchives.ripme.ripper.AbstractHTMLRipper; public class GirlsOfDesireRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(EromeRipper.class); + public GirlsOfDesireRipper(URL url) throws IOException { super(url); } @@ -38,7 +43,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + elems.first().text(); } catch (Exception e) { // Fall back to default album naming convention - LOGGER.warn("Failed to get album title from " + url, e); + logger.warn("Failed to get album title from " + url, e); } return super.getAlbumTitle(url); } @@ -79,4 +84,4 @@ public void downloadURL(URL url, int index) { // Send referrer when downloading images addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), null); } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java index 2b8ac9674..91cf47d55 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Hentai2readRipper.java @@ -12,10 +12,15 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; public class Hentai2readRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(Hentai2readRipper.class); + String lastPage; public Hentai2readRipper(URL url) throws IOException { @@ -39,7 +44,7 @@ public boolean hasQueueSupport() { @Override public boolean pageContainsAlbums(URL url) { - LOGGER.info("Page contains albums"); + logger.info("Page contains albums"); Pattern pat = Pattern.compile("https?://hentai2read\\.com/([a-zA-Z0-9_-]*)/?"); Matcher mat = pat.matcher(url.toExternalForm()); if (mat.matches()) { @@ -96,7 +101,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + getGID(url); } catch (Exception e) { // Fall back to default album naming convention - LOGGER.warn("Failed to get album title from " + url, e); + logger.warn("Failed to get album title from " + url, e); } return super.getAlbumTitle(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java index 246258597..0d3d8ba5d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaidudeRipper.java @@ -18,9 +18,13 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class HentaidudeRipper extends AbstractSingleFileRipper { + private static final Logger logger = LogManager.getLogger(HentaidudeRipper.class); + private Pattern p1 = Pattern.compile("https?://hentaidude\\.com/([a-zA-Z0-9_-]*)/?$"); // to match URLs. private Pattern p2 = Pattern.compile("data:\\s?(\\{.*?\\})", Pattern.DOTALL); @@ -95,7 +99,7 @@ public void run() { URL videoSourceUrl = new URI(getVideoUrl(doc)).toURL(); addURLToDownload(videoSourceUrl, "", "", "", null, getVideoName(), "mp4"); } catch (Exception e) { - LOGGER.error("Could not get video url for " + getVideoName(), e); + logger.error("Could not get video url for " + getVideoName(), e); } } @@ -103,7 +107,7 @@ private String getVideoName() { try { return getGID(url); } catch (MalformedURLException e) { - LOGGER.error("Unable to get video title from " + url.toExternalForm()); + logger.error("Unable to get video title from " + url.toExternalForm()); e.printStackTrace(); } return "unknown"; @@ -161,4 +165,4 @@ public String getVideoUrl(Document doc) throws IOException { throw new IOException("Could not get video download url."); } } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java index d4482b4cb..6184c7fc6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoundryRipper.java @@ -10,7 +10,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.Connection.Method; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; @@ -19,9 +20,12 @@ import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; public class HentaifoundryRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(HentaifoundryRipper.class); + private Map cookies = new HashMap<>(); public HentaifoundryRipper(URL url) throws IOException { super(url); @@ -99,7 +103,7 @@ public Document getFirstPage() throws IOException { cookies.putAll(resp.cookies()); } else { - LOGGER.info("unable to find csrf_token and set filter"); + logger.info("unable to find csrf_token and set filter"); } resp = Http.url(url) @@ -136,7 +140,7 @@ public List getURLsFromPage(Document doc) { // this if is for ripping pdf stories if (url.toExternalForm().contains("/stories/")) { for (Element pdflink : doc.select("a.pdfLink")) { - LOGGER.info("grabbing " + "https://www.hentai-foundry.com" + pdflink.attr("href")); + logger.info("grabbing " + "https://www.hentai-foundry.com" + pdflink.attr("href")); imageURLs.add("https://www.hentai-foundry.com" + pdflink.attr("href")); } return imageURLs; @@ -148,19 +152,19 @@ public List getURLsFromPage(Document doc) { } Matcher imgMatcher = imgRegex.matcher(thumb.attr("href")); if (!imgMatcher.matches()) { - LOGGER.info("Couldn't find user & image ID in " + thumb.attr("href")); + logger.info("Couldn't find user & image ID in " + thumb.attr("href")); continue; } Document imagePage; try { - LOGGER.info("grabbing " + "https://www.hentai-foundry.com" + thumb.attr("href")); + logger.info("grabbing " + "https://www.hentai-foundry.com" + thumb.attr("href")); imagePage = Http.url("https://www.hentai-foundry.com" + thumb.attr("href")).cookies(cookies).get(); } catch (IOException e) { - LOGGER.debug(e.getMessage()); - LOGGER.debug("Warning: imagePage is null!"); + logger.debug(e.getMessage()); + logger.debug("Warning: imagePage is null!"); imagePage = null; } // This is here for when the image is resized to a thumbnail because ripme doesn't report a screensize diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java index d6dba4190..2858725b6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HentaifoxRipper.java @@ -9,6 +9,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -16,6 +18,8 @@ public class HentaifoxRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(HentaifoxRipper.class); + public HentaifoxRipper(URL url) throws IOException { super(url); } @@ -43,7 +47,7 @@ public String getGID(URL url) throws MalformedURLException { @Override public List getURLsFromPage(Document doc) { - LOGGER.info(doc); + logger.info(doc); List result = new ArrayList<>(); for (Element el : doc.select("div.preview_thumb > a > img")) { String imageSource = "https:" + el.attr("data-src").replaceAll("t\\.jpg", ".jpg"); @@ -60,7 +64,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + title + "_" + getGID(url); } catch (Exception e) { // Fall back to default album naming convention - LOGGER.warn("Failed to get album title from " + url, e); + logger.warn("Failed to get album title from " + url, e); } return super.getAlbumTitle(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java index d312b75b0..186fa7336 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HitomiRipper.java @@ -10,6 +10,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.jsoup.nodes.Document; @@ -18,6 +20,8 @@ public class HitomiRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(HitomiRipper.class); + private String galleryId = ""; public HitomiRipper(URL url) throws IOException { @@ -73,7 +77,7 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce return getHost() + "_" + getGID(url) + "_" + doc.select("title").text().replaceAll(" - Read Online - hentai artistcg \\| Hitomi.la", ""); } catch (IOException e) { - LOGGER.info("Falling back"); + logger.info("Falling back"); } return super.getAlbumTitle(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index 0f69c75be..7183f1d79 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -1,14 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.utils.Http; - -import org.jsoup.Connection.Response; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; @@ -19,10 +10,21 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static com.rarchives.ripme.App.logger; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.Connection.Response; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.utils.Http; public class HqpornerRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(HqpornerRipper.class); + private static final String VIDEO_URL_PREFIX = "https://hqporner.com"; private Pattern p1 = Pattern.compile("https?://hqporner.com/hdporn/([a-zA-Z0-9_-]*).html/?$"); // video pattern. @@ -169,7 +171,7 @@ public void fetchVideo() { } } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while downloading video.", e); + logger.error("[!] Exception while downloading video.", e); } } @@ -282,8 +284,8 @@ private String getVideoName() { }// class HqpornerDownloadThread public String getBestQualityLink(List list) { - // return link with the highest quality subsubstring. Keeping it simple for now. - // 1080 > 720 > 480 > 360 > 240 + // return link with the highest quality subsubstring. Keeping it simple for now. + // 1080 > 720 > 480 > 360 > 240 if (list.isEmpty()) { return null; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java index 154206552..693c37749 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HypnohubRipper.java @@ -8,6 +8,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -16,6 +18,8 @@ public class HypnohubRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(HypnohubRipper.class); + public HypnohubRipper(URL url) throws IOException { super(url); } @@ -47,14 +51,14 @@ public String getGID(URL url) throws MalformedURLException { } private String ripPost(String url) throws IOException { - LOGGER.info(url); + logger.info(url); Document doc = Http.url(url).get(); return "https:" + doc.select("img.image").attr("src"); } private String ripPost(Document doc) { - LOGGER.info(url); + logger.info(url); return "https:" + doc.select("img.image").attr("src"); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 0699273f1..a4155b629 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -22,8 +22,13 @@ import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + public class ImagebamRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(ImagebamRipper.class); + // Thread pool for finding direct image links from "image" pages (html) private DownloadThreadPool imagebamThreadPool = new DownloadThreadPool("imagebam"); @Override @@ -95,13 +100,13 @@ public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxExce // Attempt to use album title as GID Elements elems = getCachedFirstPage().select("[id=gallery-name]"); String title = elems.first().text(); - LOGGER.info("Title text: '" + title + "'"); + logger.info("Title text: '" + title + "'"); if (StringUtils.isNotBlank(title)) { return getHost() + "_" + getGID(url) + " (" + title + ")"; } } catch (Exception e) { // Fall back to default album naming convention - LOGGER.warn("Failed to get album title from " + url, e); + logger.warn("Failed to get album title from " + url, e); } return super.getAlbumTitle(url); } @@ -125,7 +130,7 @@ private class ImagebamImageThread implements Runnable { public void run() { fetchImage(); } - + /** * Rips useful image from "image page" */ @@ -139,28 +144,28 @@ private void fetchImage() { // Find image Elements metaTags = doc.getElementsByTag("meta"); - + String imgsrc = "";//initialize, so no NullPointerExceptions should ever happen. Elements elem = doc.select("img[class*=main-image]"); if ((elem != null) && (elem.size() > 0)) { imgsrc = elem.first().attr("src"); } - + //for debug, or something goes wrong. if (imgsrc.isEmpty()) { - LOGGER.warn("Image not found at " + this.url); + logger.warn("Image not found at " + this.url); return; } - + // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; if (Utils.getConfigBoolean("download.save_order", true)) { prefix = String.format("%03d_", index); } - + addURLToDownload(new URI(imgsrc).toURL(), prefix); } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); + logger.error("[!] Exception while loading/parsing " + this.url, e); } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java index 4fcf22012..0f94e7c42 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagefapRipper.java @@ -1,18 +1,17 @@ package com.rarchives.ripme.ripper.rippers; -import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -23,6 +22,8 @@ public class ImagefapRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(ImagefapRipper.class); + private int callsMade = 0; private long startTime = System.nanoTime(); @@ -56,7 +57,7 @@ public String getDomain() { public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String gid = getGID(url); String newURL = "https://www.imagefap.com/pictures/" + gid + "/random-string"; - LOGGER.debug("Changed URL from " + url + " to " + newURL); + logger.debug("Changed URL from " + url + " to " + newURL); return new URI(newURL).toURL(); } @@ -121,9 +122,9 @@ public Document getNextPage(Document doc) throws IOException, URISyntaxException } // Sleep before fetching next page. sleep(PAGE_SLEEP_TIME); - + sendUpdate(STATUS.LOADING_RESOURCE, "Loading next page URL: " + nextURL); - LOGGER.info("Attempting to load next page URL: " + nextURL); + logger.info("Attempting to load next page URL: " + nextURL); // Load next page Document nextPage = getPageWithRetries(new URI(nextURL).toURL()); @@ -136,7 +137,7 @@ public List getURLsFromPage(Document doc) { List imageURLs = new ArrayList<>(); - LOGGER.debug("Trying to get URLs from document... "); + logger.debug("Trying to get URLs from document... "); for (Element thumb : doc.select("#gallery img")) { if (!thumb.hasAttr("src") || !thumb.hasAttr("width")) { @@ -156,14 +157,14 @@ public List getURLsFromPage(Document doc) { throw new RuntimeException("Unable to extract image URL from single image page! Unable to continue"); } - LOGGER.debug("Adding imageURL: '" + image + "'"); + logger.debug("Adding imageURL: '" + image + "'"); imageURLs.add(image); if (isThisATest()) { break; } } - LOGGER.debug("Adding " + imageURLs.size() + " URLs to download"); + logger.debug("Adding " + imageURLs.size() + " URLs to download"); return imageURLs; } @@ -200,23 +201,23 @@ private String getFullSizedImage(String pageURL) { // we use a no query param version of the URL to reduce failure rate because of some query params that change between the li elements and the mainPhotoURL String noQueryPhotoUrl = framedPhotoUrl.split("\\?")[0]; - LOGGER.debug("noQueryPhotoUrl: " + noQueryPhotoUrl); - + logger.debug("noQueryPhotoUrl: " + noQueryPhotoUrl); + // we look for a li > a element who's framed attribute starts with the noQueryPhotoUrl (only reference in the page to the full URL) Elements selectedItem = doc.select("ul.thumbs > li > a[framed^='"+noQueryPhotoUrl+"']"); - + // the fullsize URL is in the href attribute String fullSizedUrl = selectedItem.attr("href"); - + if("".equals(fullSizedUrl)) throw new IOException("JSoup full URL extraction failed from '" + selectedItem.html() + "'"); - - LOGGER.debug("fullSizedUrl: " + fullSizedUrl); + + logger.debug("fullSizedUrl: " + fullSizedUrl); return fullSizedUrl; } catch (IOException | URISyntaxException e) { - LOGGER.debug("Unable to get full size image URL from page: " + pageURL + " because: " + e.getMessage()); + logger.debug("Unable to get full size image URL from page: " + pageURL + " because: " + e.getMessage()); return null; } } @@ -238,8 +239,8 @@ private Document getPageWithRetries(URL url) throws IOException { callsMade++; checkRateLimit(); - LOGGER.info("Retrieving " + url); - + logger.info("Retrieving " + url); + boolean httpCallThrottled = false; int httpAttempts = 0; @@ -250,32 +251,32 @@ private Document getPageWithRetries(URL url) throws IOException { doc = Http.url(url).get(); } catch(IOException e) { - LOGGER.info("Retrieving " + url + " error: " + e.getMessage()); + logger.info("Retrieving " + url + " error: " + e.getMessage()); if(e.getMessage().contains("404")) throw new IOException("Gallery/Page not found!"); - + if(httpAttempts < HTTP_RETRY_LIMIT) { sendUpdate(STATUS.DOWNLOAD_WARN, "HTTP call failed: " + e.getMessage() + " retrying " + httpAttempts + " / " + HTTP_RETRY_LIMIT); - + // we sleep for a few seconds sleep(PAGE_SLEEP_TIME); continue; } else { sendUpdate(STATUS.DOWNLOAD_WARN, "HTTP call failed too many times: " + e.getMessage() + " treating this as a throttle"); httpCallThrottled = true; - } + } } // no errors, we exit break; } - + if (httpCallThrottled || (doc != null && doc.toString().contains("Your IP made too many requests to our servers and we need to check that you are a real human being"))) { if (retries == 0) { throw new IOException("Hit rate limit and maximum number of retries, giving up"); } String message = "Probably hit rate limit while loading " + url + ", sleeping for " + IP_BLOCK_SLEEP_TIME + "ms, " + retries + " retries remaining"; - LOGGER.warn(message); + logger.warn(message); sendUpdate(STATUS.DOWNLOAD_WARN, message); retries--; try { @@ -303,11 +304,11 @@ private long checkRateLimit() { int rateLimitHour = RATE_LIMIT_HOUR; // Request allowed every 3.6 seconds. if(duration / 1000 < 60){ - LOGGER.debug("Rate limit: " + (rateLimitMinute - callsMade) + " calls remaining for first minute mark."); + logger.debug("Rate limit: " + (rateLimitMinute - callsMade) + " calls remaining for first minute mark."); } else if(duration / 1000 < 300){ - LOGGER.debug("Rate limit: " + (rateLimitFiveMinutes - callsMade) + " calls remaining for first 5 minute mark."); + logger.debug("Rate limit: " + (rateLimitFiveMinutes - callsMade) + " calls remaining for first 5 minute mark."); } else if(duration / 1000 < 3600){ - LOGGER.debug("Rate limit: " + (RATE_LIMIT_HOUR - callsMade) + " calls remaining for first hour mark."); + logger.debug("Rate limit: " + (RATE_LIMIT_HOUR - callsMade) + " calls remaining for first hour mark."); } return duration; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index 4691c7c63..5421e14dd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -10,6 +10,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -21,6 +23,8 @@ public class ImagevenueRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(ImagevenueRipper.class); + // Thread pool for finding direct image links from "image" pages (html) private DownloadThreadPool imagevenueThreadPool = new DownloadThreadPool("imagevenue"); @Override @@ -99,7 +103,7 @@ private void fetchImage() { // Find image Elements images = doc.select("a > img"); if (images.isEmpty()) { - LOGGER.warn("Image not found at " + this.url); + logger.warn("Image not found at " + this.url); return; } Element image = images.first(); @@ -112,8 +116,8 @@ private void fetchImage() { } addURLToDownload(new URI(imgsrc).toURL(), prefix); } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); + logger.error("[!] Exception while loading/parsing " + this.url, e); } } } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 4904ac60a..5bb7b0020 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -13,6 +13,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -29,6 +31,8 @@ public class ImgurRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(ImgurRipper.class); + private static final String DOMAIN = "imgur.com", HOST = "imgur"; @@ -136,21 +140,21 @@ public String getAlbumTitle(URL url) throws MalformedURLException { String title; final String defaultTitle1 = "Imgur: The most awesome images on the Internet"; final String defaultTitle2 = "Imgur: The magic of the Internet"; - LOGGER.info("Trying to get album title"); + logger.info("Trying to get album title"); elems = albumDoc.select("meta[property=og:title]"); title = elems.attr("content"); - LOGGER.debug("Title is " + title); + logger.debug("Title is " + title); // This is here encase the album is unnamed, to prevent // Imgur: The most awesome images on the Internet from being added onto the album name if (title.contains(defaultTitle1) || title.contains(defaultTitle2)) { - LOGGER.debug("Album is untitled or imgur is returning the default title"); + logger.debug("Album is untitled or imgur is returning the default title"); // We set the title to "" here because if it's found in the next few attempts it will be changed // but if it's nto found there will be no reason to set it later title = ""; - LOGGER.debug("Trying to use title tag to get title"); + logger.debug("Trying to use title tag to get title"); elems = albumDoc.select("title"); if (elems.text().contains(defaultTitle1) || elems.text().contains(defaultTitle2)) { - LOGGER.debug("Was unable to get album title or album was untitled"); + logger.debug("Was unable to get album title or album was untitled"); } else { title = elems.text(); @@ -176,25 +180,25 @@ public void rip() throws IOException { case ALBUM: // Fall-through case USER_ALBUM: - LOGGER.info("Album type is USER_ALBUM"); + logger.info("Album type is USER_ALBUM"); // Don't call getAlbumTitle(this.url) with this // as it seems to cause the album to be downloaded to a subdir. ripAlbum(this.url); break; case SINGLE_IMAGE: - LOGGER.info("Album type is SINGLE_IMAGE"); + logger.info("Album type is SINGLE_IMAGE"); ripSingleImage(this.url); break; case USER: - LOGGER.info("Album type is USER"); + logger.info("Album type is USER"); ripUserAccount(url); break; case SUBREDDIT: - LOGGER.info("Album type is SUBREDDIT"); + logger.info("Album type is SUBREDDIT"); ripSubreddit(url); break; case USER_IMAGES: - LOGGER.info("Album type is USER_IMAGES"); + logger.info("Album type is USER_IMAGES"); ripUserImages(url); break; } @@ -211,9 +215,9 @@ private void ripSingleImage(URL url) throws IOException, URISyntaxException { var media = json.getJSONArray("media"); if (media.length()==0) { throw new IOException(String.format("Failed to fetch image for url %s", strUrl)); - } + } if (media.length()>1) { - LOGGER.warn(String.format("Got multiple images for url %s", strUrl)); + logger.warn(String.format("Got multiple images for url %s", strUrl)); } addURLToDownload(extractImageUrlFromJson((JSONObject)media.get(0)), ""); } @@ -252,7 +256,7 @@ public static ImgurAlbum getImgurAlbum(URL url) throws IOException, URISyntaxExc if (!strUrl.contains(",")) { strUrl += "/all"; } - LOGGER.info(" Retrieving " + strUrl); + logger.info(" Retrieving " + strUrl); Document doc = getAlbumData("https://api.imgur.com/3/album/" + strUrl.split("/a/")[1]); // Try to use embedded JSON to retrieve images try { @@ -260,17 +264,17 @@ public static ImgurAlbum getImgurAlbum(URL url) throws IOException, URISyntaxExc JSONArray jsonImages = json.getJSONObject("data").getJSONArray("images"); return createImgurAlbumFromJsonArray(url, jsonImages); } catch (JSONException | URISyntaxException e) { - LOGGER.debug("Error while parsing JSON at " + url + ", continuing", e); + logger.debug("Error while parsing JSON at " + url + ", continuing", e); } // TODO If album is empty, use this to check for cached images: // http://i.rarchives.com/search.cgi?cache=http://imgur.com/a/albumID // At the least, get the thumbnails. - LOGGER.info("[!] Falling back to /noscript method"); + logger.info("[!] Falling back to /noscript method"); String newUrl = url.toExternalForm() + "/noscript"; - LOGGER.info(" Retrieving " + newUrl); + logger.info(" Retrieving " + newUrl); doc = Jsoup.connect(newUrl) .userAgent(USER_AGENT) .get(); @@ -287,7 +291,7 @@ public static ImgurAlbum getImgurAlbum(URL url) throws IOException, URISyntaxExc image = "http:" + thumb.select("img").attr("src"); } else { // Unable to find image in this div - LOGGER.error("[!] Unable to find image in div: " + thumb); + logger.error("[!] Unable to find image in div: " + thumb); continue; } if (image.endsWith(".gif") && Utils.getConfigBoolean("prefer.mp4", false)) { @@ -356,14 +360,14 @@ private static JSONObject getUserData(String userUrl) throws IOException { * URL to imgur user account (http://username.imgur.com | https://imgur.com/user/username) */ private void ripUserAccount(URL url) throws IOException, URISyntaxException { - int cPage = -1, cImage = 0; + int cPage = -1, cImage = 0; String apiUrl = "https://api.imgur.com/3/account/%s/submissions/%d/newest?album_previews=1"; // Strip 'user_' from username var username = getGID(url).replace("user_", ""); - LOGGER.info("Retrieving " + url); + logger.info("Retrieving " + url); sendUpdate(STATUS.LOADING_RESOURCE, url.toExternalForm()); - while (true) { + while (true) { cPage += 1; var pageUrl = String.format(apiUrl, username, cPage); var json = getUserData(pageUrl); @@ -392,10 +396,10 @@ private void ripUserAccount(URL url) throws IOException, URISyntaxException { try { Thread.sleep(SLEEP_BETWEEN_ALBUMS * 1000L); } catch (InterruptedException e) { - LOGGER.error(String.format("Error! Interrupted ripping album %s for user account %s", l, username), e); + logger.error(String.format("Error! Interrupted ripping album %s for user account %s", l, username), e); } } else { - // For direct links + // For direct links if (d.has("mp4") && Utils.getConfigBoolean("prefer.mp4", false)) { l = d.getString("mp4"); } @@ -437,7 +441,7 @@ private void ripUserImages(URL url) { } Thread.sleep(1000); } catch (Exception e) { - LOGGER.error("Error while ripping user images: " + e.getMessage(), e); + logger.error("Error while ripping user images: " + e.getMessage(), e); break; } } @@ -452,7 +456,7 @@ private void ripSubreddit(URL url) throws IOException, URISyntaxException { pageURL += "/"; } pageURL += "page/" + page + "/miss?scrolled"; - LOGGER.info(" Retrieving " + pageURL); + logger.info(" Retrieving " + pageURL); Document doc = Http.url(pageURL).get(); Elements imgs = doc.select(".post img"); for (Element img : imgs) { @@ -473,7 +477,7 @@ private void ripSubreddit(URL url) throws IOException, URISyntaxException { try { Thread.sleep(1000); } catch (InterruptedException e) { - LOGGER.error("Interrupted while waiting to load next album: ", e); + logger.error("Interrupted while waiting to load next album: ", e); break; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java index 4a4122adb..b0de4929e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java @@ -1,18 +1,6 @@ package com.rarchives.ripme.ripper.rippers; -import com.oracle.js.parser.ErrorManager; -import com.oracle.js.parser.Parser; -import com.oracle.js.parser.ScriptEnvironment; -import com.oracle.js.parser.Source; -import com.oracle.js.parser.ir.*; -import com.rarchives.ripme.ripper.AbstractJSONRipper; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.Utils; -import org.json.JSONArray; -import org.json.JSONObject; -import org.jsoup.Connection; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; +import static java.lang.String.format; import java.io.IOException; import java.net.MalformedURLException; @@ -21,7 +9,12 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Spliterators; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -32,13 +25,34 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; -import static java.lang.String.format; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.Connection; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.oracle.js.parser.ErrorManager; +import com.oracle.js.parser.Parser; +import com.oracle.js.parser.ScriptEnvironment; +import com.oracle.js.parser.Source; +import com.oracle.js.parser.ir.Block; +import com.oracle.js.parser.ir.CallNode; +import com.oracle.js.parser.ir.ExpressionStatement; +import com.oracle.js.parser.ir.FunctionNode; +import com.oracle.js.parser.ir.Statement; +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; // Available configuration options: // instagram.download_images_only - use to skip video links // instagram.session_id - should be set for stories and private accounts (look for sessionid cookie) public class InstagramRipper extends AbstractJSONRipper { + private static final Logger logger = LogManager.getLogger(ImagebamRipper.class); + private String qHash; private Map cookies = new HashMap<>(); private String idString; @@ -234,7 +248,7 @@ public JSONObject getNextPage(JSONObject source) throws IOException { if (pageInfo.getBoolean("has_next_page")) { return graphqlRequest(nextPageQuery.put("after", pageInfo.getString("end_cursor"))); } else { - failedItems.forEach(LOGGER::error); + failedItems.forEach(logger::error); return null; } } @@ -335,7 +349,7 @@ private JSONObject downloadItemDetailsJson(String shortcode) { return new JSONObject(response.body()); } catch (Exception e) { failedItems.add(shortcode); - LOGGER.trace(format("No item %s found", shortcode), e); + logger.trace(format("No item %s found", shortcode), e); } return null; } @@ -392,7 +406,7 @@ private String getVideoUrlFromPage(String videoID) { Document doc = Http.url("https://www.instagram.com/p/" + videoID).cookies(cookies).get(); return doc.select("meta[property=og:video]").attr("content"); } catch (Exception e) { - LOGGER.warn("Unable to get page " + "https://www.instagram.com/p/" + videoID); + logger.warn("Unable to get page " + "https://www.instagram.com/p/" + videoID); } return ""; } @@ -400,7 +414,7 @@ private String getVideoUrlFromPage(String videoID) { @Override protected void downloadURL(URL url, int index) { if (Utils.getConfigBoolean("instagram.download_images_only", false) && url.toString().contains(".mp4?")) { - LOGGER.info("Skipped video url: " + url); + logger.info("Skipped video url: " + url); return; } addURLToDownload(url, itemPrefixes.get(index - 1), "", null, cookies); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java index 2f2d5c336..d06622af3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/JagodibujaRipper.java @@ -10,6 +10,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -18,6 +20,8 @@ public class JagodibujaRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(JagodibujaRipper.class); + public JagodibujaRipper(URL url) throws IOException { super(url); } @@ -56,16 +60,16 @@ public List getURLsFromPage(Document doc) { sleep(500); Document comicPage = Http.url(comicPageUrl.attr("href")).get(); Element elem = comicPage.select("span.full-size-link > a").first(); - LOGGER.info("Got link " + elem.attr("href")); + logger.info("Got link " + elem.attr("href")); try { addURLToDownload(new URI(elem.attr("href")).toURL(), ""); } catch (MalformedURLException | URISyntaxException e) { - LOGGER.warn("Malformed URL"); + logger.warn("Malformed URL"); e.printStackTrace(); } result.add(elem.attr("href")); } catch (IOException e) { - LOGGER.info("Error loading " + comicPageUrl); + logger.info("Error loading " + comicPageUrl); } } return result; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java index c79e02bc4..ed4aea37f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/Jpg3Ripper.java @@ -1,10 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; @@ -13,8 +8,18 @@ import java.util.ArrayList; import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + public class Jpg3Ripper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(Jpg3Ripper.class); + public Jpg3Ripper(URL url) throws IOException { super(url); } @@ -44,7 +49,7 @@ public List getURLsFromPage(Document page) { public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { String u = url.toExternalForm(); u = u.replaceAll("https?://jpg3.su/a/([^/]+)/?.*", "https://jpg3.su/a/$1"); - LOGGER.debug("Changed URL from " + url + " to " + u); + logger.debug("Changed URL from " + url + " to " + u); return new URI(u).toURL(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index 408310a7a..7157e49dd 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -11,26 +11,31 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; + import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; - - /** * @author Tushar * */ public class ListalRipper extends AbstractHTMLRipper { - private Pattern p1 = Pattern.compile("https:\\/\\/www.listal.com\\/list\\/([a-zA-Z0-9-]+)"); - private Pattern p2 = + private static final Logger logger = LogManager.getLogger(ListalRipper.class); + + private final Pattern p1 = Pattern.compile("https:\\/\\/www.listal.com\\/list\\/([a-zA-Z0-9-]+)"); + private final Pattern p2 = Pattern.compile("https:\\/\\/www.listal.com\\/((?:(?:[a-zA-Z0-9-_%]+)\\/?)+)"); + private final String postUrl = "https://www.listal.com/item-list/"; //to load more images. + private String listId = null; // listId to get more images via POST. - private String postUrl = "https://www.listal.com/item-list/"; //to load more images. private UrlType urlType = UrlType.UNKNOWN; private DownloadThreadPool listalThreadPool = new DownloadThreadPool("listalThreadPool"); @@ -61,7 +66,7 @@ public Document getFirstPage() throws IOException { @Override public List getURLsFromPage(Document page) { if (urlType == UrlType.LIST) { - // for url of type LIST, https://www.listal.com/list/my-list + // for url of type LIST, https://www.listal.com/list/my-list return getURLsForListType(page); } else if (urlType == UrlType.FOLDER) { // for url of type FOLDER, https://www.listal.com/jim-carrey/pictures @@ -111,7 +116,7 @@ public Document getNextPage(Document page) throws IOException, URISyntaxExceptio try { nextPage = Http.url(postUrl).data(postParams).retries(3).post(); } catch (IOException e1) { - LOGGER.error("Failed to load more images after " + offSet, e1); + logger.error("Failed to load more images after " + offSet, e1); throw e1; } } @@ -179,7 +184,7 @@ public String getFolderTypeGid(String group) throws MalformedURLException { } } catch (Exception e) { - LOGGER.error(e); + logger.error(e); } throw new MalformedURLException("Unable to fetch the gid for given url."); } @@ -209,10 +214,10 @@ public void getImage() { addURLToDownload(new URI(imageUrl).toURL(), getPrefix(index), "", null, null, getImageName()); } else { - LOGGER.error("Couldnt find image from url: " + url); + logger.error("Couldnt find image from url: " + url); } } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while downloading image: " + url, e); + logger.error("[!] Exception while downloading image: " + url, e); } } @@ -222,7 +227,7 @@ public String getImageName() { try { name = name.substring(name.lastIndexOf("/") + 1); } catch (Exception e) { - LOGGER.info("Failed to get name for the image."); + logger.info("Failed to get name for the image."); name = null; } // Listal stores images as .jpg diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java index 9a57b06f2..02446a39e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/LusciousRipper.java @@ -1,12 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; -import org.json.JSONArray; -import org.json.JSONObject; -import org.jsoup.Connection; -import org.jsoup.nodes.Document; - import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; @@ -19,7 +12,20 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.jsoup.Connection; +import org.jsoup.nodes.Document; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + public class LusciousRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(LusciousRipper.class); + private static String albumid; private static final Pattern p = Pattern.compile("^https?://(?:www\\.)?(?:members\\.||legacy\\.||old\\.)?luscious\\.net/albums/([-_.0-9a-zA-Z]+).*$"); @@ -33,7 +39,7 @@ public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException String URLToReturn = url.toExternalForm(); URLToReturn = URLToReturn.replaceAll("https?://(?:www\\.)?luscious\\.", "https://old.luscious."); URL san_url = new URI(URLToReturn).toURL(); - LOGGER.info("sanitized URL is " + san_url.toExternalForm()); + logger.info("sanitized URL is " + san_url.toExternalForm()); return san_url; } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java index c5f6b1429..2d7f7cf62 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ManganeloRipper.java @@ -9,6 +9,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -17,6 +19,8 @@ public class ManganeloRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(ManganeloRipper.class); + public ManganeloRipper(URL url) throws IOException { super(url); } @@ -71,7 +75,7 @@ private List getURLsFromChap(String url) { } private List getURLsFromChap(Document doc) { - LOGGER.debug("Getting urls from " + doc.location()); + logger.debug("Getting urls from " + doc.location()); List result = new ArrayList<>(); for (Element el : doc.select(".vung-doc > img")) { result.add(el.attr("src")); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index d2af02a15..955e85a34 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -10,16 +10,21 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; -import org.jsoup.select.Elements; public class MotherlessRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(MotherlessRipper.class); + // All sleep times are in milliseconds private static final int IMAGE_SLEEP_TIME = 1000; @@ -61,7 +66,7 @@ protected Document getFirstPage() throws IOException { StringBuilder newPath = new StringBuilder(path); newPath.insert(2, "M"); firstURL = URI.create("https://" + DOMAIN + newPath).toURL(); - LOGGER.info("Changed URL to " + firstURL); + logger.info("Changed URL to " + firstURL); } return Http.url(firstURL).referrer("https://motherless.com").get(); } @@ -116,7 +121,7 @@ protected void downloadURL(URL url, int index) { try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { - LOGGER.warn("Interrupted while waiting to load next image", e); + logger.warn("Interrupted while waiting to load next image", e); } } @@ -154,7 +159,7 @@ public String getGID(URL url) throws MalformedURLException { protected DownloadThreadPool getThreadPool() { return motherlessThreadPool; } - + /** * Helper class to find and download images found on "image" pages */ @@ -188,10 +193,10 @@ public void run() { } addURLToDownload(new URI(file).toURL(), prefix); } else { - LOGGER.warn("[!] could not find '__fileurl' at " + url); + logger.warn("[!] could not find '__fileurl' at " + url); } } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); + logger.error("[!] Exception while loading/parsing " + this.url, e); } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java index 8cf24fd8e..d96aa0c61 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NatalieMuRipper.java @@ -8,14 +8,17 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; public class NatalieMuRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(NatalieMuRipper.class); + public int news_id = 0; public NatalieMuRipper(URL url) throws IOException { @@ -104,7 +107,7 @@ public List getURLsFromPage(Document page) { imgUrl = imgUrl.replace("list_thumb_inbox","xlarge"); // Don't download the same URL twice if (imageURLs.contains(imgUrl)) { - LOGGER.debug("Already attempted: " + imgUrl); + logger.debug("Already attempted: " + imgUrl); continue; } imageURLs.add(imgUrl); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NewgroundsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NewgroundsRipper.java index a7be157a3..3c5001f69 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NewgroundsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NewgroundsRipper.java @@ -1,9 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; -import org.jsoup.nodes.Document; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -13,8 +9,17 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.nodes.Document; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + public class NewgroundsRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(NewgroundsRipper.class); + private String username = ""; // Name of artist // Extensions supported by Newgrounds @@ -120,7 +125,7 @@ protected List getURLsFromPage(Document page) { } } catch (IOException e) { - LOGGER.error("IO Error on trying to check extension: " + inLink + m.group(1)); + logger.error("IO Error on trying to check extension: " + inLink + m.group(1)); } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index 35a1f8add..d6b17b02f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -10,20 +10,23 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class NfsfwRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(NfsfwRipper.class); + private static final String DOMAIN = "nfsfw.com", HOST = "nfsfw"; - private int index = 0; private String currentDir = ""; private List subalbumURLs = new ArrayList<>(); @@ -59,14 +62,14 @@ public Document getNextPage(Document page) throws IOException { } else if (!subalbumURLs.isEmpty()){ // Get next sub-album nextURL = subalbumURLs.remove(0); - LOGGER.info("Detected subalbum URL at:" + nextURL); + logger.info("Detected subalbum URL at:" + nextURL); Matcher m = subalbumURLPattern.matcher(nextURL); if (m.matches()) { // Set the new save directory and save images with a new index this.currentDir = m.group(1); this.index = 0; } else { - LOGGER.error("Invalid sub-album URL: " + nextURL); + logger.error("Invalid sub-album URL: " + nextURL); nextURL = null; } } @@ -74,7 +77,7 @@ public Document getNextPage(Document page) throws IOException { try { Thread.sleep(2000); } catch (InterruptedException e) { - LOGGER.error("Interrupted while waiting to load next page", e); + logger.error("Interrupted while waiting to load next page", e); } if (nextURL == null){ throw new IOException("No more pages"); @@ -156,7 +159,7 @@ public boolean pageContainsAlbums(URL url) { List subalbumURLs = getSubalbumURLs(fstPage); return imageURLs.isEmpty() && !subalbumURLs.isEmpty(); } catch (IOException | URISyntaxException e) { - LOGGER.error("Unable to load " + url, e); + logger.error("Unable to load " + url, e); return false; } } @@ -215,7 +218,7 @@ public void run() { .get(); Elements images = doc.select(".gbBlock img"); if (images.isEmpty()) { - LOGGER.error("Failed to find image at " + this.url); + logger.error("Failed to find image at " + this.url); return; } String file = images.first().attr("src"); @@ -224,8 +227,8 @@ public void run() { } addURLToDownload(new URI(file).toURL(), getPrefix(index), this.subdir); } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); + logger.error("[!] Exception while loading/parsing " + this.url, e); } } } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index fe50f1f16..41693a3ee 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -1,15 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.utils.Http; -import com.rarchives.ripme.utils.RipUtils; -import com.rarchives.ripme.utils.Utils; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -18,9 +8,23 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ripper.DownloadThreadPool; +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.RipUtils; +import com.rarchives.ripme.utils.Utils; + public class NhentaiRipper extends AbstractHTMLRipper { - private String albumTitle; + private static final Logger logger = LogManager.getLogger(NhentaiRipper.class); + private Document firstPage; // Thread pool for finding direct image links from "image" pages (html) @@ -87,7 +91,7 @@ public List getTags(Document doc) { List tags = new ArrayList<>(); for (Element tag : doc.select("a.tag")) { String tagString = tag.attr("href").replaceAll("/tag/", "").replaceAll("/", ""); - LOGGER.info("Found tag: " + tagString); + logger.info("Found tag: " + tagString); tags.add(tagString); } return tags; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java index 097fe2c05..59e897f91 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PhotobucketRipper.java @@ -11,6 +11,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.Connection; @@ -24,6 +26,8 @@ // http://s732.photobucket.com/user/doublesix66/library/WARZONE?sort=3&page=1 public class PhotobucketRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(PhotobucketRipper.class); + private static final String DOMAIN = "photobucket.com", HOST = "photobucket"; private static final int ITEMS_PER_PAGE = 24; @@ -85,7 +89,7 @@ public String getHost() { @Override public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException { - LOGGER.info(url); + logger.info(url); String u = url.toExternalForm(); if (u.contains("?")) { // strip options from URL @@ -130,7 +134,7 @@ public String getGID(URL url) throws MalformedURLException, URISyntaxException { public Document getFirstPage() throws IOException { if (this.currAlbum == null) { this.albums = getAlbumMetadata(this.url.toExternalForm()); - LOGGER.info("Detected " + albums.size() + " albums in total"); + logger.info("Detected " + albums.size() + " albums in total"); } this.currAlbum = this.albums.remove(0); // NOTE: Why not just get media count in the metadata json? @@ -165,13 +169,13 @@ public Document getNextPage(Document page) throws IOException { try { Thread.sleep(WAIT_BEFORE_NEXT_PAGE); } catch (InterruptedException e) { - LOGGER.info("Interrupted while waiting before getting next page"); + logger.info("Interrupted while waiting before getting next page"); } if (endOfAlbum){ - LOGGER.info("Turning to next album " + albums.get(0).baseURL); + logger.info("Turning to next album " + albums.get(0).baseURL); return getFirstPage(); } else { - LOGGER.info("Turning to page " + currAlbum.pageIndex + + logger.info("Turning to page " + currAlbum.pageIndex + " of album " + currAlbum.baseURL); Connection.Response resp = Http.url(currAlbum.getCurrPageURL()).response(); currAlbum.cookies = resp.cookies(); @@ -190,7 +194,7 @@ public Document getNextPage(Document page) throws IOException { protected List getURLsFromPage(Document page) { JSONObject collectionData = getCollectionData(page); if (collectionData == null) { - LOGGER.error("Unable to find JSON data at URL: " + page.location()); + logger.error("Unable to find JSON data at URL: " + page.location()); // probably better than returning null, as the ripper will display // that nothing was found instead of a NullPointerException return new ArrayList<>(); @@ -253,7 +257,7 @@ private List getAlbumMetadata(String albumURL) metadata.add(new AlbumMetadata(sub)); } } - LOGGER.info("Succesfully retrieved and parsed metadata"); + logger.info("Succesfully retrieved and parsed metadata"); return metadata; } @@ -273,7 +277,7 @@ private JSONObject getAlbumMetadataJSON(String albumURL) String apiURL = String.format("http://%s.photobucket.com/api/user/" + "%s/album/%s/get?subAlbums=%d&json=1", subdomain, user, albumTitle, ITEMS_PER_PAGE); - LOGGER.info("Loading " + apiURL); + logger.info("Loading " + apiURL); JSONObject data = Http.url(apiURL).getJSON().getJSONObject("data"); if (data.has("subAlbums")) { int count = data.getInt("subAlbumCount"); @@ -298,4 +302,4 @@ private List getSubAlbumJSONs(JSONObject data) { } return subalbumJSONs; } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java index 65d43d397..6fd0b905a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PicstatioRipper.java @@ -8,6 +8,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -16,13 +18,15 @@ public class PicstatioRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(PicstatioRipper.class); + public PicstatioRipper(URL url) throws IOException { super(url); } private String getFullSizedImageFromURL(String fileName) { try { - LOGGER.info("https://www.picstatio.com/wallpaper/" + fileName + "/download"); + logger.info("https://www.picstatio.com/wallpaper/" + fileName + "/download"); return Http.url("https://www.picstatio.com/wallpaper/" + fileName + "/download").get().select("p.text-center > span > a").attr("href"); } catch (IOException e) { e.printStackTrace(); @@ -64,7 +68,7 @@ public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); for (Element e : doc.select("img.img")) { String imageName = e.parent().attr("href"); - LOGGER.info(getFullSizedImageFromURL(imageName.split("/")[2])); + logger.info(getFullSizedImageFromURL(imageName.split("/")[2])); result.add(getFullSizedImageFromURL(imageName.split("/")[2])); } return result; @@ -74,4 +78,4 @@ public List getURLsFromPage(Document doc) { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index a2ce4a196..481ab1ede 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -11,16 +11,21 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; public class PornhubRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(PornhubRipper.class); + // All sleep times are in milliseconds private static final int IMAGE_SLEEP_TIME = 1000; @@ -81,7 +86,7 @@ protected void downloadURL(URL url, int index) { try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { - LOGGER.warn("Interrupted while waiting to load next image", e); + logger.warn("Interrupted while waiting to load next image", e); } } @@ -153,7 +158,7 @@ private void fetchImage() { Elements images = doc.select("#photoImageSection img"); Element image = images.first(); String imgsrc = image.attr("src"); - LOGGER.info("Found URL " + imgsrc + " via " + images.get(0)); + logger.info("Found URL " + imgsrc + " via " + images.get(0)); // Provide prefix and let the AbstractRipper "guess" the filename String prefix = ""; @@ -165,8 +170,8 @@ private void fetchImage() { addURLToDownload(imgurl, prefix); } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while loading/parsing " + this.url, e); + logger.error("[!] Exception while loading/parsing " + this.url, e); } } } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 5a760744b..2419cd035 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -1,5 +1,18 @@ package com.rarchives.ripme.ripper.rippers; +import static j2html.TagCreator.a; +import static j2html.TagCreator.body; +import static j2html.TagCreator.br; +import static j2html.TagCreator.div; +import static j2html.TagCreator.h1; +import static j2html.TagCreator.head; +import static j2html.TagCreator.iff; +import static j2html.TagCreator.rawHtml; +import static j2html.TagCreator.script; +import static j2html.TagCreator.span; +import static j2html.TagCreator.style; +import static j2html.TagCreator.title; + import java.io.IOException; import java.io.OutputStream; import java.net.MalformedURLException; @@ -13,26 +26,29 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ui.RipStatusMessage; -import j2html.TagCreator; -import j2html.tags.ContainerTag; -import j2html.tags.specialized.DivTag; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONTokener; +import org.jsoup.Jsoup; import com.rarchives.ripme.ripper.AlbumRipper; +import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.UpdateUtils; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; -import org.jsoup.Jsoup; -import static j2html.TagCreator.*; +import j2html.TagCreator; +import j2html.tags.ContainerTag; +import j2html.tags.specialized.DivTag; public class RedditRipper extends AlbumRipper { + private static final Logger logger = LogManager.getLogger(RedditRipper.class); + public RedditRipper(URL url) throws IOException { super(url); } @@ -129,7 +145,7 @@ private URL getAndParseAndReturnNext(URL url) throws IOException, URISyntaxExcep saveText(getJsonArrayFromURL(getJsonURL(selfPostURL))); } } catch (Exception e) { - LOGGER.debug("at index " + i + ", for this data: " + data.toString() + e); + logger.debug("at index " + i + ", for this data: " + data.toString() + e); } } if (data.has("after") && !data.isNull("after")) { @@ -148,7 +164,7 @@ private URL getAndParseAndReturnNext(URL url) throws IOException, URISyntaxExcep try { Thread.sleep(2000); } catch (InterruptedException e) { - LOGGER.warn("Interrupted while sleeping", e); + logger.warn("Interrupted while sleeping", e); } return nextURL; } @@ -166,7 +182,7 @@ private JSONArray getJsonArrayFromURL(URL url) throws IOException { try { Thread.sleep(timeDiff); } catch (InterruptedException e) { - LOGGER.warn("[!] Interrupted while waiting to load next page", e); + logger.warn("[!] Interrupted while waiting to load next page", e); return new JSONArray(); } } @@ -185,7 +201,7 @@ private JSONArray getJsonArrayFromURL(URL url) throws IOException { } else if (jsonObj instanceof JSONArray) { jsonArray = (JSONArray) jsonObj; } else { - LOGGER.warn("[!] Unable to parse JSON: " + jsonString); + logger.warn("[!] Unable to parse JSON: " + jsonString); } return jsonArray; } @@ -304,14 +320,14 @@ private void saveText(JSONArray jsonArray) throws JSONException { out.write(html.getBytes()); out.close(); } catch (IOException e) { - LOGGER.error("[!] Error creating save file path for description '" + url + "':", e); + logger.error("[!] Error creating save file path for description '" + url + "':", e); return; } - LOGGER.debug("Downloading " + url + "'s self post to " + saveFileAs); + logger.debug("Downloading " + url + "'s self post to " + saveFileAs); super.retrievingSource(permalink); if (!Files.exists(saveFileAs.getParent())) { - LOGGER.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); + logger.info("[+] Creating directory: " + Utils.removeCWD(saveFileAs.getParent())); try { Files.createDirectory(saveFileAs.getParent()); } catch (IOException e) { @@ -336,7 +352,7 @@ private ContainerTag getComments(JSONArray comments, String author) { getNestedComments(data, commentDiv, author); commentsDiv.with(commentDiv); } catch (Exception e) { - LOGGER.debug("at index " + i + ", for this data: " + data.toString() + e); + logger.debug("at index " + i + ", for this data: " + data.toString() + e); } } return commentsDiv; @@ -424,7 +440,7 @@ private void handleURL(String theUrl, String id, String title) { savePath += id + "-" + url.split("/")[3] + Utils.filesystemSafe(title) + ".mp4"; URL urlToDownload = parseRedditVideoMPD(urls.get(0).toExternalForm()); if (urlToDownload != null) { - LOGGER.info("url: " + urlToDownload + " file: " + savePath); + logger.info("url: " + urlToDownload + " file: " + savePath); addURLToDownload(urlToDownload, Utils.getPath(savePath)); } } else { @@ -470,7 +486,7 @@ private void handleGallery(JSONArray data, JSONObject metadata, String id, Strin } addURLToDownload(mediaURL, prefix, subdirectory); } catch (MalformedURLException | JSONException | URISyntaxException e) { - LOGGER.error("[!] Unable to parse gallery JSON:\ngallery_data:\n" + data +"\nmedia_metadata:\n" + metadata); + logger.error("[!] Unable to parse gallery JSON:\ngallery_data:\n" + data +"\nmedia_metadata:\n" + metadata); } } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 3fc6d6696..d9657f273 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -22,8 +22,13 @@ import com.rarchives.ripme.ripper.AbstractJSONRipper; import com.rarchives.ripme.utils.Http; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + public class RedgifsRipper extends AbstractJSONRipper { + private static final Logger logger = LogManager.getLogger(RedgifsRipper.class); + private static final String HOST = "redgifs.com"; private static final String HOST_2 = "gifdeliverynetwork.com"; private static final String GIFS_DETAIL_ENDPOINT = "https://api.redgifs.com/v2/gifs/%s"; @@ -159,7 +164,7 @@ public String getGID(URL url) throws MalformedURLException { sText = URLDecoder.decode(sText, StandardCharsets.UTF_8); var list = Arrays.asList(sText.split(",")); if (list.size() > 1) { - LOGGER.warn("Url with multiple tags found. \nThey will be sorted alphabetically for folder name."); + logger.warn("Url with multiple tags found. \nThey will be sorted alphabetically for folder name."); } Collections.sort(list); var gid = list.stream().reduce("", (acc, val) -> acc.concat("_" + val)); @@ -256,7 +261,7 @@ private static List getURLsForGallery(String galleryID, String gifID) { list.add(hdURL); } } catch (IOException e) { - LOGGER.error(String.format("Error fetching gallery %s for gif %s", galleryID, gifID), e); + logger.error(String.format("Error fetching gallery %s for gif %s", galleryID, gifID), e); } return list; } @@ -270,7 +275,7 @@ private static List getURLsForGallery(String galleryID, String gifID) { * @throws IOException */ public static String getVideoURL(URL url) throws IOException, URISyntaxException { - LOGGER.info("Retrieving " + url.toExternalForm()); + logger.info("Retrieving " + url.toExternalForm()); var m = SINGLETON_PATTERN.matcher(url.toExternalForm()); if (!m.matches()) { throw new IOException(String.format("Cannot fetch redgif url %s", url.toExternalForm())); @@ -298,7 +303,7 @@ private static void fetchAuthToken() throws IOException { var json = Http.url(TEMPORARY_AUTH_ENDPOINT).getJSON(); var token = json.getString("token"); authToken = token; - LOGGER.info("Incase of redgif 401 errors, please restart the app to refresh the auth token"); + logger.info("Incase of redgif 401 errors, please restart the app to refresh the auth token"); } /** @@ -328,7 +333,7 @@ private URL getSearchOrTagsURL() throws IOException, URISyntaxException { switch (value) { case "gifs" -> endpointQueryParams.put("type", "g"); case "images" -> endpointQueryParams.put("type", "i"); - default -> LOGGER.warn(String.format("Unsupported tab for tags url %s", value)); + default -> logger.warn(String.format("Unsupported tab for tags url %s", value)); } break; case "verified": @@ -346,7 +351,7 @@ private URL getSearchOrTagsURL() throws IOException, URISyntaxException { case "viewMode": break; default: - LOGGER.warn(String.format("Unexpected query param %s for search url. Skipping.", name)); + logger.warn(String.format("Unexpected query param %s for search url. Skipping.", name)); } } @@ -360,7 +365,7 @@ private URL getSearchOrTagsURL() throws IOException, URISyntaxException { } // Check if it is the main tags page with all gifs, images, creator etc if (!endpointQueryParams.containsKey("type")) { - LOGGER.warn("No tab selected, defaulting to gifs"); + logger.warn("No tab selected, defaulting to gifs"); endpointQueryParams.put("type", "g"); } uri = new URIBuilder(TAGS_ENDPOINT); @@ -371,8 +376,8 @@ private URL getSearchOrTagsURL() throws IOException, URISyntaxException { switch (subpaths[subpaths.length - 1]) { case "gifs" -> tabType = "gifs"; case "images" -> tabType = "images"; - case "search" -> LOGGER.warn("No tab selected, defaulting to gifs"); - default -> LOGGER.warn(String.format("Unsupported search tab %s, defaulting to gifs", + case "search" -> logger.warn("No tab selected, defaulting to gifs"); + default -> logger.warn(String.format("Unsupported search tab %s, defaulting to gifs", subpaths[subpaths.length - 1])); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java index 6d14dc30e..7e6899da1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SankakuComplexRipper.java @@ -12,6 +12,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.Connection.Response; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -20,6 +22,9 @@ import com.rarchives.ripme.utils.Http; public class SankakuComplexRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(SankakuComplexRipper.class); + private Document albumDoc = null; private Map cookies = new HashMap<>(); @@ -89,10 +94,10 @@ public List getURLsFromPage(Document doc) { String siteURL = "https://" + subDomain + "sankakucomplex.com"; // Get the page the full sized image is on Document subPage = Http.url(siteURL + postLink).get(); - LOGGER.info("Checking page " + siteURL + postLink); + logger.info("Checking page " + siteURL + postLink); imageURLs.add("https:" + subPage.select("div[id=stats] > ul > li > a[id=highres]").attr("href")); } catch (IOException e) { - LOGGER.warn("Error while loading page " + postLink, e); + logger.warn("Error while loading page " + postLink, e); } } return imageURLs; @@ -112,7 +117,7 @@ public Document getNextPage(Document doc) throws IOException { // Only logged in users can see past page 25 // Trying to rip page 26 will throw a no images found error if (!nextPage.contains("page=26")) { - LOGGER.info("Getting next page: " + pagination.attr("abs:next-page-url")); + logger.info("Getting next page: " + pagination.attr("abs:next-page-url")); return Http.url(pagination.attr("abs:next-page-url")).cookies(cookies).get(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java index 2df6ab2c4..b428cf6b6 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -4,17 +4,22 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; -import java.net.*; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.java_websocket.client.WebSocketClient; - import org.apache.http.NameValuePair; import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.java_websocket.client.WebSocketClient; import org.java_websocket.handshake.ServerHandshake; import org.json.JSONArray; import org.json.JSONException; @@ -24,6 +29,8 @@ public class ScrolllerRipper extends AbstractJSONRipper { + private static final Logger logger = LogManager.getLogger(ScrolllerRipper.class); + public ScrolllerRipper(URL url) throws IOException { super(url); } @@ -92,7 +99,7 @@ public String convertFilterString(String filterParameter) { case "": return "NOFILTER"; default: - LOGGER.error(String.format("Invalid filter %s using no filter",filterParameter)); + logger.error(String.format("Invalid filter %s using no filter",filterParameter)); return ""; } } @@ -182,7 +189,7 @@ public void onClose(int i, String s, boolean b) { @Override public void onError(Exception e) { - LOGGER.error(String.format("WebSocket error, server reported %s", e.getMessage())); + logger.error(String.format("WebSocket error, server reported %s", e.getMessage())); } }; wsc.connect(); @@ -260,7 +267,7 @@ protected JSONObject getFirstPage() throws IOException { try { return prepareQuery(null, this.getGID(url), getParameter(url,"sort")); } catch (URISyntaxException e) { - LOGGER.error(String.format("Error obtaining first page: %s", e.getMessage())); + logger.error(String.format("Error obtaining first page: %s", e.getMessage())); return null; } } @@ -283,11 +290,11 @@ public JSONObject getNextPage(JSONObject source) throws IOException { try { return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); } catch (URISyntaxException e) { - LOGGER.error(String.format("Error changing page: %s", e.getMessage())); + logger.error(String.format("Error changing page: %s", e.getMessage())); return null; } } else { return null; } } -} \ No newline at end of file +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java index f3a216f45..47d52d530 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SinfestRipper.java @@ -8,6 +8,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -16,6 +18,8 @@ public class SinfestRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(SinfestRipper.class); + public SinfestRipper(URL url) throws IOException { super(url); } @@ -44,7 +48,7 @@ public String getGID(URL url) throws MalformedURLException { @Override public Document getNextPage(Document doc) throws IOException { Element elem = doc.select("td.style5 > a > img").last(); - LOGGER.info(elem.parent().attr("href")); + logger.info(elem.parent().attr("href")); if (elem == null || elem.parent().attr("href").equals("view.php?date=")) { throw new IOException("No more pages"); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/SpankbangRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/SpankbangRipper.java index 9ea1a130a..b56ba3cea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/SpankbangRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/SpankbangRipper.java @@ -8,15 +8,17 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.rarchives.ripme.ripper.AbstractSingleFileRipper; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; -import com.rarchives.ripme.ripper.VideoRipper; -import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.ripper.AbstractSingleFileRipper; public class SpankbangRipper extends AbstractSingleFileRipper { + private static final Logger logger = LogManager.getLogger(SpankbangRipper.class); + private static final String HOST = "spankbang"; public SpankbangRipper(URL url) throws IOException { @@ -33,7 +35,7 @@ public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); Elements videos = doc.select(".video-js > source"); if (videos.isEmpty()) { - LOGGER.error("Could not find Embed code at " + url); + logger.error("Could not find Embed code at " + url); return null; } result.add(videos.attr("src")); diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java index ac7414dd0..d6faae9fc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/StaRipper.java @@ -12,6 +12,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -22,6 +24,8 @@ public class StaRipper extends AbstractHTMLRipper { + private static final Logger logger = LogManager.getLogger(StaRipper.class); + public StaRipper(URL url) throws IOException { super(url); } @@ -61,9 +65,9 @@ public List getURLsFromPage(Document doc) { cookies.putAll(resp.cookies()); thumbPage = resp.parse(); } catch (MalformedURLException | URISyntaxException e) { - LOGGER.info(thumbPageURL + " is a malformed URL"); + logger.info(thumbPageURL + " is a malformed URL"); } catch (IOException e) { - LOGGER.info(e.getMessage()); + logger.info(e.getMessage()); } String imageDownloadUrl = thumbPage.select("a.dev-page-download").attr("href"); if (imageDownloadUrl != null && !imageDownloadUrl.equals("")) { @@ -93,10 +97,10 @@ private String getImageLinkFromDLLink(String url) { .followRedirects(false) .execute(); String imageURL = response.header("Location"); - LOGGER.info(imageURL); + logger.info(imageURL); return imageURL; } catch (IOException e) { - LOGGER.info("Got error message " + e.getMessage() + " trying to download " + url); + logger.info("Got error message " + e.getMessage() + " trying to download " + url); return null; } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java index d514c1e62..0d53ecea2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TapasticRipper.java @@ -10,6 +10,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.jsoup.nodes.Document; @@ -20,19 +22,20 @@ import com.rarchives.ripme.utils.Utils; class TapasticEpisode { - int id; - String filename; + public int id; + public String filename; + public TapasticEpisode(int index, int id, String title) { - int index1 = index; - this.id = id; - String title1 = title; + this.id = id; this.filename = Utils.filesystemSafe(title); } } public class TapasticRipper extends AbstractHTMLRipper { - private List episodes= new ArrayList<>(); + private static final Logger logger = LogManager.getLogger(TapasticRipper.class); + + private List episodes = new ArrayList<>(); public TapasticRipper(URL url) throws IOException { super(url); @@ -53,7 +56,7 @@ public List getURLsFromPage(Document page) { List urls = new ArrayList<>(); String html = page.data(); if (!html.contains("episodeList : ")) { - LOGGER.error("No 'episodeList' found at " + this.url); + logger.error("No 'episodeList' found at " + this.url); return urls; } String jsonString = Utils.between(html, "episodeList : ", ",\n").get(0); @@ -90,9 +93,8 @@ public void downloadURL(URL url, int index) { } } } catch (IOException | URISyntaxException e) { - LOGGER.error("[!] Exception while downloading " + url, e); + logger.error("[!] Exception while downloading " + url, e); } - } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java index 8105fe73f..d072c31b4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java @@ -1,8 +1,5 @@ package com.rarchives.ripme.ripper.rippers; -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.utils.Http; - import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -13,16 +10,24 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.jsoup.Jsoup; import org.jsoup.Connection.Response; +import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.utils.Http; + public class ThechiveRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(ThechiveRipper.class); + private Pattern p1 = Pattern.compile("^https?://thechive.com/[0-9]*/[0-9]*/[0-9]*/([a-zA-Z0-9_\\-]*)/?$"); private Pattern imagePattern = Pattern.compile(""); @@ -130,10 +135,10 @@ private List getUrlsFromThechive(Document doc) { /* * The image urls are stored in a