diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index bd373a4e0..7c0b26314 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -1,5 +1,6 @@ package com.rarchives.ripme; +import com.oracle.truffle.regex.tregex.util.json.JsonArray; import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ui.History; import com.rarchives.ripme.ui.HistoryEntry; @@ -17,6 +18,8 @@ import org.apache.commons.lang.SystemUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; import javax.swing.*; import java.awt.*; @@ -354,8 +357,23 @@ private static void loadHistory() throws IOException { HISTORY.clear(); if (Files.exists(historyFile)) { try { - logger.info("Loading history from " + historyFile); + logger.info("Loading history from file: " + historyFile); HISTORY.fromFile(historyFile.toString()); +// JSONArray json = HISTORY.toJSON(); +// JSONArray rebuildArray = new JSONArray(); +// for (Object currentJsonItem : json) { +// JSONObject jsonData = (JSONObject) currentJsonItem; +// if(jsonData.has("url")){ +// String url = jsonData.getString("url"); +// if(url.contains("reddit")){ +// rebuildArray.put(jsonData); +// } +// } +// } +// HISTORY.clear(); +// HISTORY.fromJSON(rebuildArray); +// HISTORY.toFile(historyFile.toString()); + } catch (IOException e) { logger.error("Failed to load history from file " + historyFile, e); logger.warn( diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 8ccff6481..4ca7cdc6e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -354,16 +354,18 @@ protected boolean addURLToDownload(URL url, String subdirectory, String referrer return false; } - if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { - logger.info("Writing " + url.toExternalForm() + " to file"); - try { - writeDownloadedURL(url.toExternalForm() + "\n"); - } catch (IOException e) { - logger.debug("Unable to write URL history file"); + boolean fileDownloadStatus = addURLToDownload(url, saveAs, referrer, cookies, getFileExtFromMIME); + if(fileDownloadStatus){ + if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) { + logger.info("Writing " + url.toExternalForm() + " to file"); + try { + writeDownloadedURL(url.toExternalForm() + "\n"); + } catch (IOException e) { + logger.debug("Unable to write URL history file"); + } } } - - return addURLToDownload(url, saveAs, referrer, cookies, getFileExtFromMIME); + return fileDownloadStatus; } protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index e9c6f2427..58e2b8396 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -7,9 +7,13 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; import javax.net.ssl.HttpsURLConnection; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jsoup.HttpStatusException; @@ -101,172 +105,186 @@ public void run() { observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); // Setup HTTP request - HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { - huc = (HttpsURLConnection) urlToDownload.openConnection(); - } else { - huc = (HttpURLConnection) urlToDownload.openConnection(); - } - huc.setInstanceFollowRedirects(true); - // It is important to set both ConnectTimeout and ReadTimeout. If you don't then - // ripme will wait forever - // for the server to send data after connecting. - huc.setConnectTimeout(TIMEOUT); - huc.setReadTimeout(TIMEOUT); - huc.setRequestProperty("accept", "*/*"); - if (!referrer.equals("")) { - huc.setRequestProperty("Referer", referrer); // Sic - } - huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - StringBuilder cookie = new StringBuilder(); - for (String key : cookies.keySet()) { - if (!cookie.toString().equals("")) { - cookie.append("; "); - } - cookie.append(key).append("=").append(cookies.get(key)); - } - huc.setRequestProperty("Cookie", cookie.toString()); - if (observer.tryResumeDownload()) { - if (fileSize != 0) { - huc.setRequestProperty("Range", "bytes=" + fileSize + "-"); - } - } - logger.debug(Utils.getLocalizedString("request.properties") + ": " + huc.getRequestProperties()); - huc.connect(); + OkHttpClient client = new OkHttpClient.Builder() + .followRedirects(true) + .connectTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .readTimeout(TIMEOUT, TimeUnit.MILLISECONDS) + .build(); - int statusCode = huc.getResponseCode(); - logger.debug("Status code: " + statusCode); - // If the server doesn't allow resuming downloads error out - if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { - // TODO find a better way to handle servers that don't support resuming - // downloads then just erroring out - throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); - } - if (statusCode / 100 == 3) { // 3xx Redirect - if (!redirected) { - // Don't increment retries on the first redirect - tries--; - redirected = true; + // Build cookies string + StringBuilder cookieBuilder = new StringBuilder(); + for (Map.Entry entry : cookies.entrySet()) { + if (!cookieBuilder.isEmpty()) { + cookieBuilder.append("; "); } - String location = huc.getHeaderField("Location"); - urlToDownload = new URI(location).toURL(); - // Throw exception so download can be retried - throw new IOException("Redirect status code " + statusCode + " - redirect to " + location); - } - if (statusCode / 100 == 4) { // 4xx errors - logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode - + " while downloading from " + url); - observer.downloadErrored(url, Utils.getLocalizedString("nonretriable.status.code") + " " - + statusCode + " while downloading " + url.toExternalForm()); - return; // Not retriable, drop out. + cookieBuilder.append(entry.getKey()).append("=").append(entry.getValue()); } - if (statusCode / 100 == 5) { // 5xx errors - observer.downloadErrored(url, Utils.getLocalizedString("retriable.status.code") + " " + statusCode - + " while downloading " + url.toExternalForm()); - // Throw exception so download can be retried - throw new IOException(Utils.getLocalizedString("retriable.status.code") + " " + statusCode); + + Request.Builder requestBuilder = new Request.Builder() + .url(urlToDownload) + .header("Accept", "*/*") + .header("User-Agent", AbstractRipper.USER_AGENT); + + if (!referrer.isEmpty()) { + requestBuilder.header("Referer", referrer); } - if (huc.getContentLength() == 503 && urlToDownload.getHost().endsWith("imgur.com")) { - // Imgur image with 503 bytes is "404" - logger.error("[!] Imgur image is 404 (503 bytes long): " + url); - observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); - return; + + if (!cookieBuilder.isEmpty()) { + requestBuilder.header("Cookie", cookieBuilder.toString()); } - // If the ripper is using the bytes progress bar set bytesTotal to - // huc.getContentLength() - if (observer.useByteProgessBar()) { - bytesTotal = huc.getContentLength(); - observer.setBytesTotal(bytesTotal); - observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + // Handle partial download + if (observer.tryResumeDownload() && fileSize != 0) { + requestBuilder.header("Range", "bytes=" + fileSize + "-"); } - // Save file - InputStream bis; - bis = new BufferedInputStream(huc.getInputStream()); + Request request = requestBuilder.build(); - // Check if we should get the file ext from the MIME type - if (getFileExtFromMIME) { - String fileExt = URLConnection.guessContentTypeFromStream(bis); - if (fileExt != null) { - fileExt = fileExt.replaceAll("image/", ""); - saveAs = new File(saveAs.toString() + "." + fileExt); - } else { - logger.error("Was unable to get content type from stream"); - // Try to get the file type from the magic number - byte[] magicBytes = new byte[8]; - bis.read(magicBytes, 0, 5); - bis.reset(); - fileExt = Utils.getEXTFromMagic(magicBytes); - if (fileExt != null) { - saveAs = new File(saveAs.toString() + "." + fileExt); - } else { - logger.error(Utils.getLocalizedString("was.unable.to.get.content.type.using.magic.number")); - logger.error( - Utils.getLocalizedString("magic.number.was") + ": " + Arrays.toString(magicBytes)); - } + try (Response httpCallResponse = client.newCall(request).execute()) { + + + int statusCode = httpCallResponse.code(); + logger.debug("Status code: " + statusCode); + // If the server doesn't allow resuming downloads error out + if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { + // TODO find a better way to handle servers that don't support resuming + // downloads then just erroring out + throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); } - } - // If we're resuming a download we append data to the existing file - OutputStream fos = null; - if (statusCode == 206) { - fos = new FileOutputStream(saveAs, true); - } else { - try { - fos = new FileOutputStream(saveAs); - } catch (FileNotFoundException e) { - // We do this because some filesystems have a max name length - if (e.getMessage().contains("File name too long")) { - logger.error("The filename " + saveAs.getName() - + " is to long to be saved on this file system."); - logger.info("Shortening filename"); - String[] saveAsSplit = saveAs.getName().split("\\."); - // Get the file extension so when we shorten the file name we don't cut off the - // file extension - String fileExt = saveAsSplit[saveAsSplit.length - 1]; - // The max limit for filenames on Linux with Ext3/4 is 255 bytes - logger.info(saveAs.getName().substring(0, 254 - fileExt.length()) + fileExt); - String filename = saveAs.getName().substring(0, 254 - fileExt.length()) + "." + fileExt; - // We can't just use the new file name as the saveAs because the file name - // doesn't include the - // users save path, so we get the user save path from the old saveAs - saveAs = new File(saveAs.getParentFile().getAbsolutePath() + File.separator + filename); - fos = new FileOutputStream(saveAs); - } else if (saveAs.getAbsolutePath().length() > 259 && Utils.isWindows()) { - // This if is for when the file path has gone above 260 chars which windows does - // not allow - fos = Files.newOutputStream( - Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName())); - assert fos != null: "After shortenSaveAsWindows: " + saveAs.getAbsolutePath(); + if (statusCode / 100 == 3) { // 3xx Redirect + if (!redirected) { + // Don't increment retries on the first redirect + tries--; + redirected = true; } - assert fos != null: e.getStackTrace(); + String location = httpCallResponse.header("Location"); + urlToDownload = new URI(location).toURL(); + // Throw exception so download can be retried + throw new IOException("Redirect status code " + statusCode + " - redirect to " + location); } - } - byte[] data = new byte[1024 * 256]; - int bytesRead; - boolean shouldSkipFileDownload = huc.getContentLength() / 1000000 >= 10 && AbstractRipper.isThisATest(); - // If this is a test rip we skip large downloads - if (shouldSkipFileDownload) { - logger.debug("Not downloading whole file because it is over 10mb and this is a test"); - } else { - while ((bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); - return; + if (statusCode / 100 == 4) { // 4xx errors + logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + + " while downloading from " + url); + observer.downloadErrored(url, Utils.getLocalizedString("nonretriable.status.code") + " " + + statusCode + " while downloading " + url.toExternalForm()); + return; // Not retriable, drop out. + } + if (statusCode / 100 == 5) { // 5xx errors + observer.downloadErrored(url, Utils.getLocalizedString("retriable.status.code") + " " + statusCode + + " while downloading " + url.toExternalForm()); + // Throw exception so download can be retried + throw new IOException(Utils.getLocalizedString("retriable.status.code") + " " + statusCode); + } + if (httpCallResponse.body() != null && httpCallResponse.body().contentLength() == 503 && urlToDownload.getHost().endsWith("imgur.com")) { + // Imgur image with 503 bytes is "404" + logger.error("[!] Imgur image is 404 (503 bytes long): " + url); + observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); + return; + } + + // If the ripper is using the bytes progress bar set bytesTotal to + // huc.getContentLength() + if (observer.useByteProgessBar()) { + bytesTotal = (int) (httpCallResponse.body() != null ? httpCallResponse.body().contentLength() : 0); + observer.setBytesTotal(bytesTotal); + observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); + logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + } + + // Save file + InputStream byteInputDataStream; + if (httpCallResponse.body() != null) { + byteInputDataStream = new BufferedInputStream(httpCallResponse.body().byteStream()); + + // Check if we should get the file ext from the MIME type + if (getFileExtFromMIME) { + String fileExt = URLConnection.guessContentTypeFromStream(byteInputDataStream); + if (fileExt != null) { + fileExt = fileExt.replaceAll("image/", ""); + saveAs = new File(saveAs.toString() + "." + fileExt); + } else { + logger.error("Was unable to get content type from stream"); + // Try to get the file type from the magic number + byte[] magicBytes = new byte[8]; + byteInputDataStream.read(magicBytes, 0, 5); + byteInputDataStream.reset(); + fileExt = Utils.getEXTFromMagic(magicBytes); + if (fileExt != null) { + saveAs = new File(saveAs.toString() + "." + fileExt); + } else { + logger.error(Utils.getLocalizedString("was.unable.to.get.content.type.using.magic.number")); + logger.error( + Utils.getLocalizedString("magic.number.was") + ": " + Arrays.toString(magicBytes)); + } + } } - fos.write(data, 0, bytesRead); - if (observer.useByteProgessBar()) { - bytesDownloaded += bytesRead; - observer.setBytesCompleted(bytesDownloaded); - observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + // If we're resuming a download we append data to the existing file + OutputStream fileOutputDataStream = null; + if (statusCode == 206) { + fileOutputDataStream = new FileOutputStream(saveAs, true); + } else { + try { + fileOutputDataStream = new FileOutputStream(saveAs); + } catch (FileNotFoundException e) { + // We do this because some filesystems have a max name length + if (e.getMessage().contains("File name too long")) { + logger.error("The filename " + saveAs.getName() + + " is to long to be saved on this file system."); + logger.info("Shortening filename"); + String[] saveAsSplit = saveAs.getName().split("\\."); + // Get the file extension so when we shorten the file name we don't cut off the + // file extension + String fileExt = saveAsSplit[saveAsSplit.length - 1]; + // The max limit for filenames on Linux with Ext3/4 is 255 bytes + logger.info(saveAs.getName().substring(0, 254 - fileExt.length()) + fileExt); + String filename = saveAs.getName().substring(0, 254 - fileExt.length()) + "." + fileExt; + // We can't just use the new file name as the saveAs because the file name + // doesn't include the + // users save path, so we get the user save path from the old saveAs + saveAs = new File(saveAs.getParentFile().getAbsolutePath() + File.separator + filename); + fileOutputDataStream = new FileOutputStream(saveAs); + } else if (saveAs.getAbsolutePath().length() > 259 && Utils.isWindows()) { + // This if is for when the file path has gone above 260 chars which windows does + // not allow + fileOutputDataStream = Files.newOutputStream( + Utils.shortenSaveAsWindows(saveAs.getParentFile().getPath(), saveAs.getName())); + assert fileOutputDataStream != null: "After shortenSaveAsWindows: " + saveAs.getAbsolutePath(); + } + assert fileOutputDataStream != null: e.getStackTrace(); + } } + byte[] data = new byte[1024 * 256]; + int bytesRead; + boolean shouldSkipFileDownload = httpCallResponse.body().contentLength() / 1000000 >= 10 && AbstractRipper.isThisATest(); + // If this is a test rip we skip large downloads + if (shouldSkipFileDownload) { + logger.debug("Not downloading whole file because it is over 10mb and this is a test"); + } else { + while ((bytesRead = byteInputDataStream.read(data)) != -1) { + try { + observer.stopCheck(); + } catch (IOException e) { + observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + return; + } + fileOutputDataStream.write(data, 0, bytesRead); + if (observer.useByteProgessBar()) { + bytesDownloaded += bytesRead; + observer.setBytesCompleted(bytesDownloaded); + observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + } + } + } + byteInputDataStream.close(); + fileOutputDataStream.close(); } + + + // You can now access headers, body, etc. + logger.debug("Request headers: " + httpCallResponse.request().headers()); + // Example: read httpCallResponse body + // InputStream in = httpCallResponse.body().byteStream(); } - bis.close(); - fos.close(); break; // Download successful: break out of infinite loop } catch (SocketTimeoutException timeoutEx) { // Handle the timeout diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java index 765edea43..6f0ba8a49 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DanbooruRipper.java @@ -6,7 +6,9 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -24,6 +26,8 @@ import okhttp3.Request; import okhttp3.Response; +import static com.rarchives.ripme.utils.RipUtils.getCookiesFromString; + public class DanbooruRipper extends AbstractJSONRipper { private static final Logger logger = LogManager.getLogger(DanbooruRipper.class); @@ -35,6 +39,8 @@ public class DanbooruRipper extends AbstractJSONRipper { private Pattern gidPattern = null; private int currentPageNum = 1; + private Map cookies = new HashMap<>(); + public DanbooruRipper(URL url) throws IOException { super(url); @@ -44,6 +50,15 @@ public DanbooruRipper(URL url) throws IOException { .build(); } + private void setCookies(String currentCookie) { + if (Utils.getConfigBoolean("danbooru.login", true)) { + logger.info("Logging in using cookies"); +// String faCookies = Utils.getConfigString("danbooru.cookies", currentCookie); + String[] cookieInfo = currentCookie.split("="); + cookies.put(cookieInfo[0], cookieInfo[1]); + } + } + @Override protected String getDomain() { return DOMAIN; @@ -68,6 +83,43 @@ protected JSONObject getNextPage(JSONObject doc) throws IOException { return getCurrentPage(); } + private String getSubUrlFromUrl(String url){ + Request request = new Request.Builder() + .url(url) + .header("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1") + .header("Accept", "application/json,text/javascript,*/*;q=0.01") + .header("Accept-Language", "en-US,en;q=0.9") + .header("Sec-Fetch-Dest", "empty") + .header("Sec-Fetch-Mode", "cors") + .header("Sec-Fetch-Site", "same-origin") + .header("Referer", "https://danbooru.donmai.us/") + .header("X-Requested-With", "XMLHttpRequest") + .header("Connection", "keep-alive") + .build(); + Response response = null; + try { + response = client.newCall(request).execute(); + if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); + + String responseData = response.body().string(); + + JSONObject json = new JSONObject(responseData); + if(json.has("file_url")){ + return json.getString("file_url"); + }else{ + return null; + } + + } catch (Exception e) { + e.printStackTrace(); + } finally { + if(response !=null) { + response.body().close(); + } + } + return null; + } + @Nullable private JSONObject getCurrentPage() throws MalformedURLException { Request request = new Request.Builder() @@ -87,14 +139,27 @@ private JSONObject getCurrentPage() throws MalformedURLException { try { response = client.newCall(request).execute(); if (!response.isSuccessful()) throw new IOException("Unexpected code " + response); - + if(response.headers().get("set-cookie") != null) { + setCookies(response.headers().get("set-cookie")); + } String responseData = response.body().string(); JSONArray jsonArray = new JSONArray(responseData); if(!jsonArray.isEmpty()){ - String newCompatibleJSON = "{ \"resources\":" + jsonArray + " }"; + JSONArray resultsToUse = new JSONArray(); + for(Object currentObject : jsonArray){ + JSONObject currentJSONObject = (JSONObject) currentObject; + //tag_string_artist + //https://danbooru.donmai.us/posts/6677739?q=jagerctm1 + JSONObject objectToAdd = new JSONObject(); + objectToAdd.put("file_url", getSubUrlFromUrl("https://danbooru.donmai.us/posts/" + currentJSONObject.get("id") + "?q="+ currentJSONObject.getString("tag_string_artist") + "&z=1")); + //"https://danbooru.donmai.us/posts/" + currentJSONObject.get("id") + "?q="+ currentJSONObject.getString("tag_string_artist") + "&z=1" + + resultsToUse.put(objectToAdd); + } + String newCompatibleJSON = "{ \"resources\":" + resultsToUse + " }"; return new JSONObject(newCompatibleJSON); } - } catch (IOException e) { + } catch (Exception e) { e.printStackTrace(); } finally { if(response !=null) { @@ -129,7 +194,8 @@ public String getGID(URL url) throws MalformedURLException { @Override protected void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); +// addURLToDownload(url, getPrefix(index)); + addURLToDownload(url, getPrefix(index), "", "https://danbooru.donmai.us", cookies); } private String getTag(URL url) throws MalformedURLException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FileFinishCallBack.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FileFinishCallBack.java new file mode 100644 index 000000000..bdde0d970 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FileFinishCallBack.java @@ -0,0 +1,9 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.net.URL; + +public interface FileFinishCallBack { + void onComplete(String result, URL currentUrl); + + void onFail(String result, URL currentUrl); +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/KemonoPartyRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/KemonoPartyRipper.java new file mode 100644 index 000000000..889656a95 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/KemonoPartyRipper.java @@ -0,0 +1,210 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractJSONRipper; +import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.Utils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * See this link for the API schema. + */ +public class KemonoPartyRipper extends AbstractJSONRipper { + private static final Logger LOGGER = LogManager.getLogger(KemonoPartyRipper.class); + private static final String IMG_URL_BASE = "https://c3.kemono.su/data"; + private static final String VID_URL_BASE = "https://c1.kemono.su/data"; + private static final Pattern IMG_PATTERN = Pattern.compile("^.*\\.(jpg|jpeg|png|gif|apng|webp|tif|tiff)$", Pattern.CASE_INSENSITIVE); + private static final Pattern VID_PATTERN = Pattern.compile("^.*\\.(webm|mp4|m4v)$", Pattern.CASE_INSENSITIVE); + + // just so we can return a JSONObject from getFirstPage + private static final String KEY_WRAPPER_JSON_ARRAY = "array"; + + private static final String KEY_FILE = "file"; + private static final String KEY_PATH = "path"; + private static final String KEY_ATTACHMENTS = "attachments"; + + // One of "onlyfans" or "fansly", but might have others in future? + private final String service; + + // Username of the page to be ripped + private final String user; + private JSONObject current_page_data; + private int internalFileLimit; + + public KemonoPartyRipper(URL url) throws IOException { + super(url); + List pathElements = Arrays.stream(url.getPath().split("/")) + .filter(element -> !element.isBlank()) + .collect(Collectors.toList()); + + service = pathElements.get(0); + user = pathElements.get(2); + + if (service == null || user == null || service.isBlank() || user.isBlank()) { + LOGGER.warn("service=" + service + ", user=" + user); + throw new MalformedURLException("Invalid kemono.party URL: " + url); + } + LOGGER.debug("Parsed service=" + service + " and user=" + user + " from " + url); + } + + @Override + protected String getDomain() { + return "kemono.party"; + } + + @Override + public String getHost() { + return "kemono.party"; + } + + @Override + public boolean canRip(URL url) { + String host = url.getHost(); + return host.endsWith("kemono.party") || host.endsWith("kemono.su") || host.endsWith("kemono.cr"); + } + + @Override + public String getGID(URL url) { + return Utils.filesystemSafe(String.format("%s_%s", service, user)); + } + + @Override + protected JSONObject getFirstPage() throws IOException { + String apiUrl = String.format("https://kemono.su/api/v1/%s/user/%s/posts", service, user); + String jsonArrayString = Http.url(apiUrl) + .ignoreContentType() + .header("Accept", "text/css") + .response() + .body(); + JSONArray jsonArray = new JSONArray(jsonArrayString); + internalFileLimit += 50; + // Ideally we'd just return the JSONArray from here, but we have to wrap it in a JSONObject + JSONObject wrapperObject = new JSONObject(); + wrapperObject.put(KEY_WRAPPER_JSON_ARRAY, jsonArray); + return wrapperObject; + } + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + String title; + try { + //Gets artist name + title = getHost() + "_" + getGID(url) + "_" + Http.url(url).get().select("meta[name=artist_name][content]").attr("content"); + }catch (Exception e){ + LOGGER.info("Failed to get album title, using id."); + title = getGID(url); + } + return title; + } + + @Override + protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxException { + String apiUrl = String.format("https://kemono.su/api/v1/%s/user/%s/posts?o=%s", service, user, internalFileLimit); + String jsonArrayString = Http.url(apiUrl) + .ignoreContentType() + .header("Accept", "text/css") + .response() + .body(); + JSONArray jsonArray = new JSONArray(jsonArrayString); + + // Ideally we'd just return the JSONArray from here, but we have to wrap it in a JSONObject + JSONObject wrapperObject = new JSONObject(); + wrapperObject.put(KEY_WRAPPER_JSON_ARRAY, jsonArray); + internalFileLimit += 50; + if(jsonArray.isEmpty()){ + return null; + } + return wrapperObject; + } + + @Override + protected List getURLsFromJSON(JSONObject json) { + // extract the array from our wrapper JSONObject + JSONArray posts = json.getJSONArray(KEY_WRAPPER_JSON_ARRAY); + ArrayList urls = new ArrayList<>(); + for (int i = 0; i < posts.length(); i++) { + JSONObject post = posts.getJSONObject(i); + pullFileUrl(post, urls); + pullAttachmentUrls(post, urls); + } + LOGGER.debug("Pulled " + urls.size() + " URLs from " + posts.length() + " posts"); + return urls; + } + + @Override + protected void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index)); + } + + /** + * Retrieves the URL of a file from a given JSONObject and adds it to the provided list. + * + * @param post The JSONObject containing information about the file. + * @param results The list to which the URL of the file will be added. + */ + private void pullFileUrl(JSONObject post, ArrayList results) { + try { + JSONObject file = post.getJSONObject(KEY_FILE); + String path = file.getString(KEY_PATH); + if (isImage(path)) { + String url = IMG_URL_BASE + path; + results.add(url); + } else if (isVideo(path)) { + String url = VID_URL_BASE + path; + results.add(url); + } else { + LOGGER.error("Unknown extension for kemono.su path: " + path); + } + } catch (JSONException e) { + /* No-op */ + } + } + + /** + * Retrieves the URLs of attachments from a given JSONObject and adds them to the provided list. + * + * @param post The JSONObject containing information about the post. + * @param results The list to which the URLs of the attachments will be added. + */ + private void pullAttachmentUrls(JSONObject post, ArrayList results) { + try { + JSONArray attachments = post.getJSONArray(KEY_ATTACHMENTS); + for (int i = 0; i < attachments.length(); i++) { + JSONObject attachment = attachments.getJSONObject(i); + pullFileUrl(attachment, results); + } + } catch (JSONException e) { + /* No-op */ + } + } + + /** + * Checks if the given path represents an image file. + * + * @param path The path of the file to be checked. + * @return True if the file is an image, false otherwise. + */ + private boolean isImage(String path) { + Matcher matcher = IMG_PATTERN.matcher(path); + return matcher.matches(); + } + + private boolean isVideo(String path) { + Matcher matcher = VID_PATTERN.matcher(path); + return matcher.matches(); + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/test.json b/src/main/java/com/rarchives/ripme/ripper/rippers/test.json new file mode 100644 index 000000000..ea2eede58 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/test.json @@ -0,0 +1,99 @@ +{ + "id": 7695707, + "created_at": "2024-06-10T07:34:42.266-04:00", + "uploader_id": 923344, + "score": 24, + "source": "https://i.pximg.net/img-original/img/2023/08/26/23/21/34/111185876_p0.jpg", + "md5": "3e9a0edcabcb9eda6d33ab5a1b1956b8", + "last_comment_bumped_at": null, + "rating": "g", + "image_width": 939, + "image_height": 1127, + "tag_string": "1girl bankai black_hair black_kimono bleach bleach:_sennen_kessen-hen blue_eyes colored_eyelashes commentary_request expressionless from_side hair_between_eyes hakka_no_togame_(bankai) hara_hikaru ice ice_crystal ice_flower japanese_clothes kimono kuchiki_rukia looking_up shihakusho shinigami short_hair solo white_background", + "fav_count": 20, + "file_ext": "jpg", + "last_noted_at": null, + "parent_id": null, + "has_children": false, + "approver_id": null, + "tag_count_general": 21, + "tag_count_artist": 1, + "tag_count_character": 1, + "tag_count_copyright": 2, + "file_size": 626956, + "up_score": 24, + "down_score": 0, + "is_pending": false, + "is_flagged": false, + "is_deleted": false, + "tag_count": 26, + "updated_at": "2024-06-10T07:34:55.158-04:00", + "is_banned": false, + "pixiv_id": 111185876, + "last_commented_at": null, + "has_active_children": false, + "bit_flags": 0, + "tag_count_meta": 1, + "has_large": true, + "has_visible_children": false, + "media_asset": { + "id": 21386805, + "created_at": "2024-06-10T07:33:08.944-04:00", + "updated_at": "2024-06-10T07:33:10.406-04:00", + "md5": "3e9a0edcabcb9eda6d33ab5a1b1956b8", + "file_ext": "jpg", + "file_size": 626956, + "image_width": 939, + "image_height": 1127, + "duration": null, + "status": "active", + "file_key": "EhecRBrsv", + "is_public": true, + "pixel_hash": "fb40ecb11db5e990438f223ba0314ba7", + "variants": [ + { + "type": "180x180", + "url": "https://cdn.donmai.us/180x180/3e/9a/3e9a0edcabcb9eda6d33ab5a1b1956b8.jpg", + "width": 150, + "height": 180, + "file_ext": "jpg" + }, + { + "type": "360x360", + "url": "https://cdn.donmai.us/360x360/3e/9a/3e9a0edcabcb9eda6d33ab5a1b1956b8.jpg", + "width": 300, + "height": 360, + "file_ext": "jpg" + }, + { + "type": "720x720", + "url": "https://cdn.donmai.us/720x720/3e/9a/3e9a0edcabcb9eda6d33ab5a1b1956b8.webp", + "width": 600, + "height": 720, + "file_ext": "webp" + }, + { + "type": "sample", + "url": "https://cdn.donmai.us/sample/3e/9a/sample-3e9a0edcabcb9eda6d33ab5a1b1956b8.jpg", + "width": 850, + "height": 1020, + "file_ext": "jpg" + }, + { + "type": "original", + "url": "https://cdn.donmai.us/original/3e/9a/3e9a0edcabcb9eda6d33ab5a1b1956b8.jpg", + "width": 939, + "height": 1127, + "file_ext": "jpg" + } + ] + }, + "tag_string_general": "1girl bankai black_hair black_kimono blue_eyes colored_eyelashes expressionless from_side hair_between_eyes hakka_no_togame_(bankai) ice ice_crystal ice_flower japanese_clothes kimono looking_up shihakusho shinigami short_hair solo white_background", + "tag_string_character": "kuchiki_rukia", + "tag_string_copyright": "bleach bleach:_sennen_kessen-hen", + "tag_string_artist": "hara_hikaru", + "tag_string_meta": "commentary_request", + "file_url": "https://cdn.donmai.us/original/3e/9a/3e9a0edcabcb9eda6d33ab5a1b1956b8.jpg", + "large_file_url": "https://cdn.donmai.us/sample/3e/9a/sample-3e9a0edcabcb9eda6d33ab5a1b1956b8.jpg", + "preview_file_url": "https://cdn.donmai.us/180x180/3e/9a/3e9a0edcabcb9eda6d33ab5a1b1956b8.jpg" +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/test2.json b/src/main/java/com/rarchives/ripme/ripper/rippers/test2.json new file mode 100644 index 000000000..199212c97 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/test2.json @@ -0,0 +1,92 @@ +{ + "file_url": "https://cdn.donmai.us/original/6c/42/6c42e7758e9d884ceb5d1301e66486a3.jpg", + "uploader_id": 1052903, + "media_asset": { + "created_at": "2025-03-16T05:40:50.241-04:00", + "image_width": 752, + "pixel_hash": "12f077cb2fc2ef012b083b283c7f30d5", + "variants": [ + { + "file_ext": "jpg", + "width": 106, + "type": "180x180", + "url": "https://cdn.donmai.us/180x180/6c/42/6c42e7758e9d884ceb5d1301e66486a3.jpg", + "height": 180 + }, + { + "file_ext": "jpg", + "width": 212, + "type": "360x360", + "url": "https://cdn.donmai.us/360x360/6c/42/6c42e7758e9d884ceb5d1301e66486a3.jpg", + "height": 360 + }, + { + "file_ext": "webp", + "width": 423, + "type": "720x720", + "url": "https://cdn.donmai.us/720x720/6c/42/6c42e7758e9d884ceb5d1301e66486a3.webp", + "height": 720 + }, + { + "file_ext": "jpg", + "width": 752, + "type": "original", + "url": "https://cdn.donmai.us/original/6c/42/6c42e7758e9d884ceb5d1301e66486a3.jpg", + "height": 1280 + } + ], + "file_size": 157074, + "duration": null, + "file_key": "1iTvjZpV6", + "file_ext": "jpg", + "image_height": 1280, + "updated_at": "2025-03-16T05:40:52.442-04:00", + "is_public": true, + "id": 25223325, + "md5": "6c42e7758e9d884ceb5d1301e66486a3", + "status": "active" + }, + "tag_string": "1girl aqua_marine_(aqua000marine) blue_eyes breasts crossed_legs duel_monster gloves grey_hair hat highres long_hair sideboob silent_magician simple_background solo tabard white_gloves white_hat white_tabard wizard_hat yu-gi-oh!", + "rating": "s", + "tag_string_artist": "aqua_marine_(aqua000marine)", + "created_at": "2025-03-16T05:59:13.093-04:00", + "last_comment_bumped_at": null, + "last_commented_at": null, + "source": "https://pawoo.net/@rokomoko0/101675723688106698", + "score": 19, + "has_large": false, + "fav_count": 18, + "large_file_url": "https://cdn.donmai.us/original/6c/42/6c42e7758e9d884ceb5d1301e66486a3.jpg", + "image_height": 1280, + "is_deleted": false, + "tag_count": 21, + "updated_at": "2025-08-03T19:24:08.070-04:00", + "pixiv_id": null, + "bit_flags": 0, + "tag_count_character": 1, + "id": 9004519, + "tag_count_artist": 1, + "tag_string_meta": "highres", + "down_score": 0, + "tag_string_general": "1girl blue_eyes breasts crossed_legs duel_monster gloves grey_hair hat long_hair sideboob simple_background solo tabard white_gloves white_hat white_tabard wizard_hat", + "has_children": false, + "tag_count_copyright": 1, + "approver_id": null, + "tag_string_character": "silent_magician", + "image_width": 752, + "last_noted_at": null, + "up_score": 19, + "tag_string_copyright": "yu-gi-oh!", + "file_size": 157074, + "has_visible_children": false, + "file_ext": "jpg", + "has_active_children": false, + "parent_id": null, + "tag_count_general": 17, + "preview_file_url": "https://cdn.donmai.us/180x180/6c/42/6c42e7758e9d884ceb5d1301e66486a3.jpg", + "tag_count_meta": 1, + "is_banned": false, + "is_flagged": false, + "md5": "6c42e7758e9d884ceb5d1301e66486a3", + "is_pending": false +} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ui/History.java b/src/main/java/com/rarchives/ripme/ui/History.java index 190eeeb8e..0a54804c5 100644 --- a/src/main/java/com/rarchives/ripme/ui/History.java +++ b/src/main/java/com/rarchives/ripme/ui/History.java @@ -90,7 +90,7 @@ public HistoryEntry getEntryByURL(String url) { throw new RuntimeException("Could not find URL " + url + " in History"); } - private void fromJSON(JSONArray jsonArray) { + public void fromJSON(JSONArray jsonArray) { JSONObject json; for (int i = 0; i < jsonArray.length(); i++) { json = jsonArray.getJSONObject(i); @@ -116,7 +116,7 @@ public void fromList(List stringList) { } } - private JSONArray toJSON() { + public JSONArray toJSON() { JSONArray jsonArray = new JSONArray(); for (HistoryEntry entry : list) { jsonArray.put(entry.toJSON()); diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 35357830c..a493a0ee0 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -41,6 +41,8 @@ import com.rarchives.ripme.uiUtils.ContextActionProtections; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; +import org.json.JSONArray; +import org.json.JSONObject; /** * Everything UI-related starts and ends here. diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FreeComicOnlineRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FreeComicOnlineRipperTest.java index e33b9de25..fe2465d8e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FreeComicOnlineRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FreeComicOnlineRipperTest.java @@ -4,12 +4,15 @@ import java.net.URI; import java.net.URISyntaxException; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import com.rarchives.ripme.ripper.rippers.FreeComicOnlineRipper; public class FreeComicOnlineRipperTest extends RippersTest { + //Seems broken from the website side. @Test + @Disabled public void testFreeComicOnlineChapterAlbum() throws IOException, URISyntaxException { FreeComicOnlineRipper ripper = new FreeComicOnlineRipper( new URI("https://freecomiconline.me/comic/perfect-half-hentai0003/chapter-01/").toURL());