From d770b933ab2e203b48ed402885bfe294d6afe9e7 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:44:53 -0400 Subject: [PATCH 01/78] Upgrade gradle --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index df97d72b8..d4081da47 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From ff6297c7eac77010f2e0ba9a83a11cef64e12f86 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:51:30 -0400 Subject: [PATCH 02/78] Upgrade vulnerable json lib --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 2ac8aed1b..4bd0bf097 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -23,7 +23,7 @@ dependencies { implementation("com.lmax:disruptor:3.4.4") implementation("org.java-websocket:Java-WebSocket:1.5.3") implementation("org.jsoup:jsoup:1.16.1") - implementation("org.json:json:20211205") + implementation("org.json:json:20231013") implementation("com.j2html:j2html:1.6.0") implementation("commons-configuration:commons-configuration:1.10") implementation("commons-cli:commons-cli:1.5.0") From 85c0f5181303a928868aef38cafd3ec9dab89c67 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:52:00 -0400 Subject: [PATCH 03/78] Upgrade vulnerable commons-io lib --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4bd0bf097..7f0b89a1b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -27,7 +27,7 @@ dependencies { implementation("com.j2html:j2html:1.6.0") implementation("commons-configuration:commons-configuration:1.10") implementation("commons-cli:commons-cli:1.5.0") - implementation("commons-io:commons-io:2.13.0") + implementation("commons-io:commons-io:2.14.0") implementation("org.apache.httpcomponents:httpclient:4.5.14") implementation("org.apache.httpcomponents:httpmime:4.5.14") implementation("org.apache.logging.log4j:log4j-api:2.20.0") From 0be9a978cd25d3dfe693eb0170aaac0e095e0e05 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Fri, 18 Jul 2025 23:57:45 -0400 Subject: [PATCH 04/78] Fix log4j2 "Unrecognized format specifier" --- build.gradle.kts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index 7f0b89a1b..b8b4a613e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -12,6 +12,7 @@ plugins { id("jacoco") id("java") id("maven-publish") + id("com.gradleup.shadow") version "9.0.0-rc1" } repositories { @@ -80,6 +81,10 @@ tasks.withType { }) } +tasks.shadowJar { + transform() +} + publishing { publications { create("maven") { From 168c499ab0cd2b79711c54b9ce0772f3b91dcbdb Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sat, 19 Jul 2025 01:54:15 -0400 Subject: [PATCH 05/78] Limit log pane lines --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 35357830c..7d815ae92 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -15,8 +15,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collections; -import java.util.Date; +import java.util.*; import java.util.List; import java.util.stream.Stream; @@ -69,6 +68,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JButton optionLog; private static JPanel logPanel; private static JTextPane logText; + private static final Queue logLineLengths = new LinkedList<>(); + private static final int MAX_LOG_PANE_LINES = 1000; // History private static JButton optionHistory; @@ -1247,7 +1248,11 @@ private void appendLog(final String text, final Color color) { StyledDocument sd = logText.getStyledDocument(); try { synchronized (this) { + if (logLineLengths.size() > MAX_LOG_PANE_LINES) { + sd.remove(0, logLineLengths.remove()); + } sd.insertString(sd.getLength(), text + "\n", sas); + logLineLengths.add(text.length() + 1); } } catch (BadLocationException e) { LOGGER.warn(e.getMessage()); From d3af1793a26d0dc526ea230198a619ff499c5764 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sun, 20 Jul 2025 02:59:41 -0400 Subject: [PATCH 06/78] Add graceful stop --- .../ripme/ripper/AbstractRipper.java | 22 ++++- .../ripme/ripper/DownloadFileThread.java | 18 ++--- .../ripme/ripper/DownloadVideoThread.java | 5 +- .../com/rarchives/ripme/ui/MainWindow.java | 81 ++++++++++++------- 4 files changed, 82 insertions(+), 44 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index f1ed45600..ee06e7806 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -56,7 +56,7 @@ public abstract class AbstractRipper DownloadThreadPool threadPool; RipStatusHandler observer = null; - private boolean completed = true; + private final AtomicBoolean completed = new AtomicBoolean(false); public abstract void rip() throws IOException, URISyntaxException; @@ -71,6 +71,7 @@ public boolean hasASAPRipping() { // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; private final AtomicBoolean shouldStop = new AtomicBoolean(false); + private final AtomicBoolean shouldPanic = new AtomicBoolean(false); private static boolean thisIsATest = false; public void stop() { @@ -78,10 +79,24 @@ public void stop() { shouldStop.set(true); } + public void panic() { + logger.trace("panic()"); + shouldStop.set(true); + shouldPanic.set(true); + } + + public boolean isPanicked() { + return shouldPanic.get(); + } + public boolean isStopped() { return shouldStop.get(); } + public boolean isCompleted() { + return completed.get(); + } + protected void stopCheck() throws IOException { if (shouldStop.get()) { throw new IOException("Ripping interrupted"); @@ -480,7 +495,7 @@ public static String getFileName(URL url, String prefix, String fileName, String */ protected void waitForThreads() { logger.debug("Waiting for threads to finish"); - completed = false; + completed.set(false); threadPool.waitForThreads(); checkIfComplete(); } @@ -532,8 +547,7 @@ void checkIfComplete() { return; } - if (!completed) { - completed = true; + if (!completed.getAndSet(true)) { logger.info(" Rip completed!"); RipStatusComplete rsc = new RipStatusComplete(workingDir.toPath(), getCount()); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index e9c6f2427..417c4370b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -63,6 +63,13 @@ public void setCookies(Map cookies) { */ @Override public void run() { + + if (observer.isStopped()) { + // TODO add handler for graceful stop + observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + return; + } + // First thing we make sure the file name doesn't have any illegal chars in it saveAs = new File( saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); @@ -72,12 +79,7 @@ public void run() { if (saveAs.exists() && observer.tryResumeDownload()) { fileSize = saveAs.length(); } - try { - observer.stopCheck(); - } catch (IOException e) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); - return; - } + if (saveAs.exists() && !observer.tryResumeDownload() && !getFileExtFromMIME || Utils.fuzzyExists(Paths.get(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) { @@ -251,9 +253,7 @@ public void run() { logger.debug("Not downloading whole file because it is over 10mb and this is a test"); } else { while ((bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { + if (observer.isPanicked()) { observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); return; } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 9430adce3..9f164ecef 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -48,6 +48,7 @@ public void run() { try { observer.stopCheck(); } catch (IOException e) { + // TODO create status for gracefully-stopped download observer.downloadErrored(url, "Download interrupted"); return; } @@ -107,9 +108,7 @@ public void run() { bis = new BufferedInputStream(huc.getInputStream()); fos = Files.newOutputStream(saveAs); while ( (bytesRead = bis.read(data)) != -1) { - try { - observer.stopCheck(); - } catch (IOException e) { + if (observer.isPanicked()) { observer.downloadErrored(url, "Download interrupted"); return; } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 7d815ae92..959e5162a 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -17,6 +17,7 @@ import java.nio.file.Paths; import java.util.*; import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Stream; import javax.imageio.ImageIO; @@ -48,13 +49,11 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final Logger LOGGER = LogManager.getLogger(MainWindow.class); - /* not static! */ - private boolean isRipping = false; // Flag to indicate if we're ripping something - private static JFrame mainFrame; private static JTextField ripTextfield; private static JButton ripButton, stopButton; + private static JButton panicButton; private static JLabel statusLabel; private static JButton openButton; @@ -129,6 +128,9 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static AbstractRipper ripper; + private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. + private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. + private void updateQueue(DefaultListModel model) { if (model == null) model = queueListModel; @@ -327,6 +329,8 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib ripButton = new JButton("Rip", ripIcon); stopButton = new JButton("Stop"); stopButton.setEnabled(false); + panicButton = new JButton("Panic!"); + panicButton.setEnabled(false); try { Image stopIcon = ImageIO.read(getClass().getClassLoader().getResource("stop.png")); stopButton.setIcon(new ImageIcon(stopIcon)); @@ -349,6 +353,8 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib ripPanel.add(ripButton, gbc); gbc.gridx = 3; ripPanel.add(stopButton, gbc); + gbc.gridx = 4; + ripPanel.add(panicButton, gbc); gbc.weightx = 1; statusLabel = new JLabel(Utils.getLocalizedString("inactive")); @@ -802,13 +808,32 @@ private void update() { stopButton.addActionListener(event -> { if (ripper != null) { ripper.stop(); - isRipping = false; + gracefulStop.set(true); + queueListModel.add(0, ripper.getURL().toString()); + stopButton.setEnabled(false); + statusProgress.setValue(0); + statusProgress.setVisible(false); + pack(); + statusProgress.setValue(0); + //status(Utils.getLocalizedString("download.interrupted")); + status("Rip gracefully stopping"); + appendLog("Download interrupted", Color.RED); + } + }); + + panicButton.addActionListener(event -> { + if (ripper != null) { + ripper.stop(); + ripper.panic(); + panicStop.set(true); + queueListModel.add(0, ripper.getURL().toString()); stopButton.setEnabled(false); + panicButton.setEnabled(false); statusProgress.setValue(0); statusProgress.setVisible(false); pack(); statusProgress.setValue(0); - status(Utils.getLocalizedString("download.interrupted")); + status("Rip interrupted"); // TODO localize appendLog("Download interrupted", Color.RED); } }); @@ -1061,10 +1086,7 @@ public void mouseClicked(MouseEvent e) { @Override public void intervalAdded(ListDataEvent arg0) { updateQueue(); - - if (!isRipping) { - ripNextAlbum(); - } + ripNextAlbum(); } @Override @@ -1324,14 +1346,20 @@ private void saveHistory() { } private void ripNextAlbum() { - isRipping = true; - // Save current state of queue to configuration. Utils.setConfigList("queue", queueListModel.elements()); + boolean wasGracefulStop = gracefulStop.getAndSet(false); + boolean wasPanicStop = gracefulStop.getAndSet(false); + if (wasGracefulStop || wasPanicStop) { + // Stop requested + ripFinishCleanup(); + return; + } + if (queueListModel.isEmpty()) { // End of queue - isRipping = false; + ripFinishCleanup(); return; } @@ -1353,6 +1381,13 @@ private void ripNextAlbum() { } } + private void ripFinishCleanup() { + stopButton.setEnabled(false); + panicButton.setEnabled(false); + statusProgress.setValue(0); + statusProgress.setVisible(false); + } + private Thread ripAlbum(String urlString) { if (!logPanel.isVisible()) { optionLog.doClick(); @@ -1373,6 +1408,7 @@ private Thread ripAlbum(String urlString) { return null; } stopButton.setEnabled(true); + panicButton.setEnabled(true); statusProgress.setValue(100); openButton.setVisible(false); statusLabel.setVisible(true); @@ -1482,9 +1518,7 @@ public void actionPerformed(ActionEvent event) { mainWindow.statusWithColor("This URL is already in queue: " + url, Color.ORANGE); ripTextfield.setText(""); } - else if(!mainWindow.isRipping){ - mainWindow.ripNextAlbum(); - } + mainWindow.ripNextAlbum(); } } @@ -1503,10 +1537,6 @@ public void run() { } private synchronized void handleEvent(StatusEvent evt) { - if (ripper.isStopped()) { - return; - } - RipStatusMessage msg = evt.msg; int completedPercent = evt.ripper.getCompletionPercentage(); @@ -1552,9 +1582,7 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + ripFinishCleanup(); openButton.setVisible(false); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); @@ -1585,9 +1613,8 @@ private synchronized void handleEvent(StatusEvent evt) { Utils.playSound("camera.wav"); } saveHistory(); - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + Utils.saveConfig(); + ripFinishCleanup(); openButton.setVisible(true); Path f = rsc.dir; String prettyFile = Utils.shortenPath(f); @@ -1657,9 +1684,7 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + ripFinishCleanup(); openButton.setVisible(false); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); From 7436b3dc5a82affb45732d21885ef551473673a5 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 9 Sep 2025 03:37:01 -0400 Subject: [PATCH 07/78] Fix method name --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 2 +- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 4 +++- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 2 +- src/main/java/com/rarchives/ripme/ripper/VideoRipper.java | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 0740f62c4..aed3959a8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -475,7 +475,7 @@ protected void checkIfComplete() { return; } if (itemsPending.isEmpty()) { - super.checkIfComplete(); + notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index a49084c63..94cd12bf1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -277,7 +277,7 @@ protected void checkIfComplete() { return; } if (itemsPending.isEmpty()) { - super.checkIfComplete(); + notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index ee06e7806..a00d160cb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -538,10 +538,12 @@ int getCount() { return 1; } + protected abstract void checkIfComplete(); + /** * Notifies observers and updates state if all files have been ripped. */ - void checkIfComplete() { + protected void notifyComplete() { if (observer == null) { logger.debug("observer is null"); return; diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index bda3bf6fb..870143ac7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -187,7 +187,7 @@ protected void checkIfComplete() { return; } if (itemsPending.isEmpty()) { - super.checkIfComplete(); + notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 785f3d92b..67515e789 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -207,7 +207,7 @@ protected void checkIfComplete() { } if (bytesCompleted >= bytesTotal) { - super.checkIfComplete(); + notifyComplete(); } } From 078b04267cf842bb9b0799dc05b8a4b96eec4e0a Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sun, 20 Jul 2025 02:03:33 -0400 Subject: [PATCH 08/78] Add transfer rate monitor --- .../ripme/ripper/DownloadFileThread.java | 1 + .../ripme/ripper/DownloadVideoThread.java | 1 + .../com/rarchives/ripme/ui/MainWindow.java | 57 +++++++++++++- .../rarchives/ripme/ui/RipStatusMessage.java | 1 + .../rarchives/ripme/utils/TransferRate.java | 77 +++++++++++++++++++ 5 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/rarchives/ripme/utils/TransferRate.java diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 417c4370b..a8a2f0589 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -258,6 +258,7 @@ public void run() { return; } fos.write(data, 0, bytesRead); + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesRead); if (observer.useByteProgessBar()) { bytesDownloaded += bytesRead; observer.setBytesCompleted(bytesDownloaded); diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 9f164ecef..b825598f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -113,6 +113,7 @@ public void run() { return; } fos.write(data, 0, bytesRead); + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesRead); bytesDownloaded += bytesRead; observer.setBytesCompleted(bytesDownloaded); observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 959e5162a..4825fd6ee 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -18,6 +18,10 @@ import java.util.*; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import javax.imageio.ImageIO; @@ -40,6 +44,7 @@ import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.uiUtils.ContextActionProtections; import com.rarchives.ripme.utils.RipUtils; +import com.rarchives.ripme.utils.TransferRate; import com.rarchives.ripme.utils.Utils; /** @@ -56,6 +61,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JButton panicButton; private static JLabel statusLabel; + private static final JLabel transferRateLabel = new JLabel(); private static JButton openButton; private static JProgressBar statusProgress; @@ -131,6 +137,30 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. + public static final int TRANSFER_RATE_REFRESH_RATE = 200; + private static final TransferRate transferRate = new TransferRate(); + + private static final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); + private Future rateRefresherFuture = null; + private final Runnable rateRefresher = () -> { + if (isHalted()) { + if (rateRefresherFuture != null) { + rateRefresherFuture.cancel(true); + rateRefresherFuture = null; + } + transferRateLabel.setText(""); + return; + } + transferRateLabel.setText(transferRate.formatHumanTransferRate()); + }; + + /** + * @return true if fully halted/panic button pressed + */ + public static boolean isHalted() { + return ripper == null || ripper.isPanicked() || ripper.isCompleted(); + } + private void updateQueue(DefaultListModel model) { if (model == null) model = queueListModel; @@ -283,6 +313,8 @@ private void createUI(Container pane) { LOGGER.error("[!] Exception setting system theme:", e); } + Font monospaced = new Font(Font.MONOSPACED, Font.PLAIN, mainFrame.getContentPane().getFont().getSize()); + ripTextfield = new JTextField("", 20); ripTextfield.addMouseListener(new ContextMenuMouseListener(ripTextfield)); @@ -359,16 +391,26 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib statusLabel = new JLabel(Utils.getLocalizedString("inactive")); statusLabel.setHorizontalAlignment(JLabel.CENTER); + transferRateLabel.setHorizontalAlignment(JLabel.RIGHT); + transferRateLabel.setFont(monospaced); openButton = new JButton(); openButton.setVisible(false); JPanel statusPanel = new JPanel(new GridBagLayout()); statusPanel.setBorder(emptyBorder); gbc.gridx = 0; + gbc.weightx = 1; statusPanel.add(statusLabel, gbc); + gbc.gridx = 1; + gbc.weightx = 0; + statusPanel.add(transferRateLabel, gbc); + gbc.gridx = 0; + gbc.weightx = 1; + gbc.gridwidth = 2; gbc.gridy = 1; statusPanel.add(openButton, gbc); gbc.gridy = 0; + gbc.gridwidth = 1; JPanel progressPanel = new JPanel(new GridBagLayout()); progressPanel.setBorder(emptyBorder); @@ -1363,6 +1405,10 @@ private void ripNextAlbum() { return; } + if (rateRefresherFuture == null || rateRefresherFuture.isDone()) { + rateRefresherFuture = executor.scheduleAtFixedRate(rateRefresher, 0, TRANSFER_RATE_REFRESH_RATE, TimeUnit.MILLISECONDS); + } + String nextAlbum = (String) queueListModel.remove(0); updateQueue(); @@ -1412,6 +1458,7 @@ private Thread ripAlbum(String urlString) { statusProgress.setValue(100); openButton.setVisible(false); statusLabel.setVisible(true); + transferRateLabel.setVisible(true); pack(); boolean failed = false; try { @@ -1538,13 +1585,21 @@ public void run() { private synchronized void handleEvent(StatusEvent evt) { RipStatusMessage msg = evt.msg; + RipStatusMessage.STATUS status = msg.getStatus(); + + // CHUNK_BYTES is noisy, so handle it before any other computation + if (status == RipStatusMessage.STATUS.CHUNK_BYTES) { + transferRate.addChunk((Long) msg.getObject()); + transferRateLabel.setText(transferRate.formatHumanTransferRate()); + return; + } int completedPercent = evt.ripper.getCompletionPercentage(); statusProgress.setValue(completedPercent); statusProgress.setVisible(true); status(evt.ripper.getStatusText()); - switch (msg.getStatus()) { + switch (status) { case LOADING_RESOURCE: case DOWNLOAD_STARTED: if (LOGGER.isEnabled(Level.INFO)) { diff --git a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java index f589e9dbb..3d4c1644f 100644 --- a/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java +++ b/src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java @@ -16,6 +16,7 @@ public enum STATUS { DOWNLOAD_SKIP("Download Skipped"), TOTAL_BYTES("Total bytes"), COMPLETED_BYTES("Completed bytes"), + CHUNK_BYTES("Transferred bytes in last chunk"), RIP_ERRORED("Rip Errored"), NO_ALBUM_OR_USER("No album or user"); diff --git a/src/main/java/com/rarchives/ripme/utils/TransferRate.java b/src/main/java/com/rarchives/ripme/utils/TransferRate.java new file mode 100644 index 000000000..9227160f3 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/TransferRate.java @@ -0,0 +1,77 @@ +package com.rarchives.ripme.utils; + +import java.util.Deque; +import java.util.concurrent.ConcurrentLinkedDeque; + +public class TransferRate { + private final Deque transferQueue = new ConcurrentLinkedDeque<>(); + private int windowDurationMs = 10000; + + public void addChunk(long bytes) { + long now = System.currentTimeMillis(); + transferQueue.addFirst(new ChunkStamp(now, bytes)); + removeOldChunks(now); + } + + public double calculateBytesPerSecond() { + if (transferQueue.isEmpty()) { + return 0; + } + long totalBytes = 0; + ChunkStamp oldest = transferQueue.getLast(); + long now = System.currentTimeMillis(); + removeOldChunks(now); + if (transferQueue.isEmpty()) { + return 0; + } + for (ChunkStamp chunkStamp : transferQueue) { + totalBytes += chunkStamp.bytes; + } + long elapsedMs = now - oldest.timestampMs; + double elapsedSeconds = (double) elapsedMs / 1000; + if (elapsedSeconds <= 0) { + return 0; + } + return totalBytes / elapsedSeconds; + } + + public String formatHumanTransferRate() { + double bps = calculateBytesPerSecond(); + int giB = 1024 * 1024 * 1024; + int miB = 1024 * 1024; + int kiB = 1024; + if (bps >= giB) { + return String.format("%.2f GiB/s", bps / giB); + } else if (bps >= miB) { + return String.format("%.2f MiB/s", bps / miB); + } else if (bps >= kiB) { + return String.format("%.2f KiB/s", bps / kiB); + } else { + return String.format("%.2f B/s", bps); + } + } + + public void setWindowDurationMs(int windowDurationMs) { + this.windowDurationMs = windowDurationMs; + } + + public int getWindowDurationMs() { + return windowDurationMs; + } + + private void removeOldChunks(long now) { + while (!transferQueue.isEmpty() && transferQueue.getLast().timestampMs < now - windowDurationMs) { + transferQueue.removeLast(); + } + } + + private static class ChunkStamp { + long bytes; + long timestampMs; // millis + + ChunkStamp(long timestampMs, long bytes) { + this.bytes = bytes; + this.timestampMs = timestampMs; + } + } +} From 98c15f4b18d9c1b087b5ce79c46927199a6b0c7a Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 11 Sep 2025 19:12:40 -0400 Subject: [PATCH 09/78] Throttle progress update --- .../ripme/ripper/DownloadFileThread.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index a8a2f0589..991873f4c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -252,17 +252,25 @@ public void run() { if (shouldSkipFileDownload) { logger.debug("Not downloading whole file because it is over 10mb and this is a test"); } else { + long lastProgressUpdate = 0; + long bytesSinceLastProgressUpdate = 0; while ((bytesRead = bis.read(data)) != -1) { if (observer.isPanicked()) { observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); return; } fos.write(data, 0, bytesRead); - observer.sendUpdate(STATUS.CHUNK_BYTES, bytesRead); - if (observer.useByteProgessBar()) { - bytesDownloaded += bytesRead; - observer.setBytesCompleted(bytesDownloaded); - observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + bytesSinceLastProgressUpdate += bytesRead; + long now = System.currentTimeMillis(); + if (now > lastProgressUpdate + 200) { + lastProgressUpdate = now; + observer.sendUpdate(STATUS.CHUNK_BYTES, bytesSinceLastProgressUpdate); + bytesSinceLastProgressUpdate = 0; + if (observer.useByteProgessBar()) { + bytesDownloaded += bytesRead; + observer.setBytesCompleted(bytesDownloaded); + observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); + } } } } From 8f99769a27411d9bdeca3124cbb403e81ad880ae Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sun, 20 Jul 2025 02:37:38 -0400 Subject: [PATCH 10/78] Handle no space left on device --- .../rarchives/ripme/ripper/DownloadFileThread.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 991873f4c..1e36a8ded 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -290,7 +290,16 @@ public void run() { "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); return; } - } catch (IOException | URISyntaxException e) { + } catch (IOException e) { + if (e.getMessage().matches("No space left on device")) { + logger.debug("IOException", e); + observer.downloadErrored(url, "No space left on device"); // TODO localize; TODO cancel all rips + return; + } + logger.debug("IOException", e); + logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + + e.getMessage()); + } catch (URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); From 306a4a0e6b396f80fa6ffff9d3f34b108f3cbcd6 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sun, 20 Jul 2025 02:37:49 -0400 Subject: [PATCH 11/78] Fix sorter usage warning --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 4825fd6ee..55e8ef081 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1370,6 +1370,10 @@ private void loadHistory() throws IOException { }); } } + if (!HISTORY.isEmpty()) { + // Fix "WARNING: row index is bigger than sorter's row count. Most likely this is a wrong sorter usage" + historyTableModel.fireTableDataChanged(); + } } private void saveHistory() { From 6305b5ea3fc617914b4aab6bf285ba78465dc40d Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Sun, 20 Jul 2025 02:38:39 -0400 Subject: [PATCH 12/78] Set spammy log message to trace --- src/main/java/com/rarchives/ripme/utils/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 6edc83785..20218bcc6 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -798,7 +798,7 @@ public static String[] getSupportedLanguages() { } public static String getLocalizedString(String key) { - LOGGER.debug(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), + LOGGER.trace(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), resourceBundle.getString(key))); return resourceBundle.getString(key); } From 732632e7ea3698155a93ef2ad87e895fe05caa12 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 00:37:33 -0400 Subject: [PATCH 13/78] Clarify download try # log message --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- .../java/com/rarchives/ripme/ripper/DownloadVideoThread.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 1e36a8ded..b5902670f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -99,7 +99,7 @@ public void run() { do { tries += 1; try { - logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Retry #" + tries : "")); + logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Try #" + tries : "")); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); // Setup HTTP request diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index b825598f1..3d070e8c2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -85,7 +85,7 @@ public void run() { byte[] data = new byte[1024 * 256]; int bytesRead; try { - logger.info(" Downloading file: " + url + (tries > 0 ? " Retry #" + tries : "")); + logger.info(" Downloading file: " + url + (tries > 0 ? " Try #" + tries+1 : "")); observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); // Setup HTTP request From aeabca280e9a642f05c417ce034e6fb6bdd4e8a3 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 00:42:13 -0400 Subject: [PATCH 14/78] Simplify expression --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index b5902670f..0ff7b751d 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -104,7 +104,7 @@ public void run() { // Setup HTTP request HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { + if (url.getProtocol().equals("https")) { huc = (HttpsURLConnection) urlToDownload.openConnection(); } else { huc = (HttpURLConnection) urlToDownload.openConnection(); From 25e715a97f4e434a671771ddafa7afde0cf5fcf9 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 00:51:48 -0400 Subject: [PATCH 15/78] Fix download error handling --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 0ff7b751d..fbd4b27ad 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -299,10 +299,14 @@ public void run() { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); + observer.downloadErrored(url, e.getMessage()); + return; } catch (URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); + observer.downloadErrored(url, Utils.getLocalizedString("exception.while.downloading.file")); + return; } catch (NullPointerException npe){ logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); From e0af7e605314630cb74b0c448818528d4eadeef6 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 13 Aug 2025 14:18:04 -0400 Subject: [PATCH 16/78] Retry with continue --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index fbd4b27ad..7fb541e63 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -152,8 +152,8 @@ public void run() { } String location = huc.getHeaderField("Location"); urlToDownload = new URI(location).toURL(); - // Throw exception so download can be retried - throw new IOException("Redirect status code " + statusCode + " - redirect to " + location); + logger.debug("Redirect status code {} - redirect to {}", statusCode, location); + continue; // retry } if (statusCode / 100 == 4) { // 4xx errors logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode From d1f67e67e5887bed2cb50004df8dd390f8836178 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 12:35:23 -0400 Subject: [PATCH 17/78] Support fetching media from single-use token URLs --- .../ripme/ripper/AbstractHTMLRipper.java | 81 ++++++------ .../ripme/ripper/AbstractJSONRipper.java | 76 +++++------ .../ripme/ripper/AbstractRipper.java | 32 +++-- .../rarchives/ripme/ripper/AlbumRipper.java | 72 ++++++----- .../ripme/ripper/DownloadFileThread.java | 122 +++++++++++++----- .../ripme/ripper/DownloadVideoThread.java | 104 ++++++++++----- .../com/rarchives/ripme/ripper/RipUrlId.java | 97 ++++++++++++++ .../ripme/ripper/TokenedUrlGetter.java | 14 ++ .../rarchives/ripme/ripper/VideoRipper.java | 55 ++++---- 9 files changed, 448 insertions(+), 205 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ripper/RipUrlId.java create mode 100644 src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index aed3959a8..40a3e0a56 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -13,11 +13,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -36,9 +32,9 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class); - private final Map itemsPending = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); Document cachedFirstPage; protected AbstractHTMLRipper(URL url) throws IOException { @@ -346,11 +342,10 @@ public int getCount() { return itemsCompleted.size() + itemsErrored.size(); } - @Override /* Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -358,31 +353,42 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); + private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); protected AbstractJSONRipper(URL url) throws IOException { super(url); @@ -152,7 +149,7 @@ public int getCount() { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -160,31 +157,41 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, - Boolean getFileExtFromMIME); + public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + TokenedUrlGetter tug = () -> url; + RipUrlId ripUrlId = new RipUrlId(getClass(), getHost(), url); + Path directory = saveAs.getParent(); + String filename = saveAs.getFileName().toString(); + return addURLToDownload(tug, ripUrlId, directory, filename, referrer, cookies, getFileExtFromMIME); + } + + protected boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String referrer, Map cookies, Boolean getFileExtFromMIME) { + return addURLToDownload(tug, ripUrlId, directory, null, referrer, cookies, getFileExtFromMIME); + } + + public abstract boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME); + /** * Queues image to be downloaded and saved. @@ -515,21 +529,21 @@ public void retrievingSource(String url) { /** * Notifies observers that a file download has completed. * - * @param url URL that was completed. - * @param saveAs Where the downloaded file is stored. + * @param ripUrlId The RipUrlId that was completed. + * @param saveAs Where the downloaded file is stored. */ - public abstract void downloadCompleted(URL url, Path saveAs); + public abstract void downloadCompleted(RipUrlId ripUrlId, Path saveAs); /** * Notifies observers that a file could not be downloaded (includes a reason). */ - public abstract void downloadErrored(URL url, String reason); + public abstract void downloadErrored(RipUrlId ripUrlId, String reason); /** * Notify observers that a download could not be completed, * but was not technically an "error". */ - public abstract void downloadExists(URL url, Path file); + public abstract void downloadExists(RipUrlId ripUrlId, Path file); /** * @return Number of files downloaded. @@ -805,7 +819,7 @@ protected boolean tryResumeDownload() { return false; } - protected boolean shouldIgnoreURL(URL url) { + protected static boolean shouldIgnoreURL(URL url) { // TODO change to "shouldIgnoreExtension" final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions"); if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 870143ac7..83762f3b7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -10,9 +10,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -32,9 +30,9 @@ public abstract class AlbumRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AlbumRipper.class); - private Map itemsPending = Collections.synchronizedMap(new HashMap()); - private Map itemsCompleted = Collections.synchronizedMap(new HashMap()); - private Map itemsErrored = Collections.synchronizedMap(new HashMap()); + private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); protected AlbumRipper(URL url) throws IOException { super(url); @@ -62,7 +60,7 @@ public int getCount() { /** * Queues multiple URLs of single images to download from a single Album URL */ - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { // Only download one file if this is a test. if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { stop(); @@ -70,31 +68,42 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies = new HashMap<>(); - private final URL url; - private File saveAs; - private final String prettySaveAs; + private final Path directory; + private String filename; private final AbstractRipper observer; private final int retries; private final Boolean getFileExtFromMIME; @@ -37,11 +39,13 @@ class DownloadFileThread implements Runnable { private final int TIMEOUT; private final int retrySleep; - public DownloadFileThread(URL url, File saveAs, AbstractRipper observer, Boolean getFileExtFromMIME) { + + public DownloadFileThread(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, AbstractRipper observer, Boolean getFileExtFromMIME) { super(); - this.url = url; - this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs.toPath()); + this.tokenedUrlGetter = tug; + this.ripUrlId = ripUrlId; + this.directory = directory; + this.filename = filename; this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000); @@ -66,13 +70,47 @@ public void run() { if (observer.isStopped()) { // TODO add handler for graceful stop - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("download.interrupted")); return; } + URL url = null; + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + if (filename == null) { + // Strip token query parameters + filename = Path.of(url.getPath()).getFileName().toString(); + } // First thing we make sure the file name doesn't have any illegal chars in it - saveAs = new File( - saveAs.getParentFile().getAbsolutePath() + File.separator + Utils.sanitizeSaveAs(saveAs.getName())); + filename = Utils.sanitizeSaveAs(filename); + if (AbstractRipper.shouldIgnoreURL(url)) { + observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return; + } + + if (!Files.exists(directory)) { + logger.info("[+] Creating directory: " + directory); + try { + Files.createDirectories(directory); + } catch (IOException e) { + logger.error("Error creating directory", e); + observer.downloadErrored(ripUrlId, "Error creating directory: " + directory + " ; " + e.getMessage()); + return; + } + } + + File saveAs = directory.resolve(filename).toFile(); + String prettySaveAs = Utils.removeCWD(saveAs.toPath()); + long fileSize = 0; int bytesTotal; int bytesDownloaded = 0; @@ -89,25 +127,26 @@ public void run() { } else { logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " + Utils.getLocalizedString("file.already.exists") + ": " + prettySaveAs); - observer.downloadExists(url, saveAs.toPath()); + observer.downloadExists(ripUrlId, saveAs.toPath()); return; } } - URL urlToDownload = this.url; boolean redirected = false; int tries = 0; // Number of attempts to download do { tries += 1; try { - logger.info(" Downloading file: " + urlToDownload + (tries > 0 ? " Try #" + tries : "")); - observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); + logger.info(" Downloading file: " + url + (tries > 0 ? " Try #" + tries : "")); + + String urlNoQuery = new URI(url.getProtocol(), url.getAuthority(), url.getPath(), null, null).toURL().toExternalForm(); + observer.sendUpdate(STATUS.DOWNLOAD_STARTED, urlNoQuery); // Setup HTTP request HttpURLConnection huc; if (url.getProtocol().equals("https")) { - huc = (HttpsURLConnection) urlToDownload.openConnection(); + huc = (HttpsURLConnection) url.openConnection(); } else { - huc = (HttpURLConnection) urlToDownload.openConnection(); + huc = (HttpURLConnection) url.openConnection(); } huc.setInstanceFollowRedirects(true); // It is important to set both ConnectTimeout and ReadTimeout. If you don't then @@ -142,7 +181,10 @@ public void run() { if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { // TODO find a better way to handle servers that don't support resuming // downloads then just erroring out - throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); + observer.downloadErrored(ripUrlId, "Local file exists, resume attempted, but server does not support resuming downloads: " + + statusCode + " while downloading " + url.toExternalForm()); + //throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); + return; } if (statusCode / 100 == 3) { // 3xx Redirect if (!redirected) { @@ -151,27 +193,27 @@ public void run() { redirected = true; } String location = huc.getHeaderField("Location"); - urlToDownload = new URI(location).toURL(); + url = new URI(location).toURL(); // TODO fix redirect with TokenedUrlGetter logger.debug("Redirect status code {} - redirect to {}", statusCode, location); continue; // retry } if (statusCode / 100 == 4) { // 4xx errors logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + " while downloading from " + url); - observer.downloadErrored(url, Utils.getLocalizedString("nonretriable.status.code") + " " + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + " while downloading " + url.toExternalForm()); return; // Not retriable, drop out. } if (statusCode / 100 == 5) { // 5xx errors - observer.downloadErrored(url, Utils.getLocalizedString("retriable.status.code") + " " + statusCode + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("retriable.status.code") + " " + statusCode + " while downloading " + url.toExternalForm()); // Throw exception so download can be retried throw new IOException(Utils.getLocalizedString("retriable.status.code") + " " + statusCode); } - if (huc.getContentLength() == 503 && urlToDownload.getHost().endsWith("imgur.com")) { + if (huc.getContentLength() == 503 && url.getHost().endsWith("imgur.com")) { // Imgur image with 503 bytes is "404" logger.error("[!] Imgur image is 404 (503 bytes long): " + url); - observer.downloadErrored(url, "Imgur image is 404: " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, "Imgur image is 404: " + url.toExternalForm()); return; } @@ -181,7 +223,7 @@ public void run() { bytesTotal = huc.getContentLength(); observer.setBytesTotal(bytesTotal); observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); } // Save file @@ -256,7 +298,7 @@ public void run() { long bytesSinceLastProgressUpdate = 0; while ((bytesRead = bis.read(data)) != -1) { if (observer.isPanicked()) { - observer.downloadErrored(url, Utils.getLocalizedString("download.interrupted")); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("download.interrupted")); return; } fos.write(data, 0, bytesRead); @@ -284,33 +326,33 @@ public void run() { break; } catch (HttpStatusException hse) { logger.debug(Utils.getLocalizedString("http.status.exception"), hse); - logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + urlToDownload); + logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + hse.getUrl()); if (hse.getStatusCode() == 404 && Utils.getConfigBoolean("errors.skip404", false)) { - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); return; } } catch (IOException e) { if (e.getMessage().matches("No space left on device")) { logger.debug("IOException", e); - observer.downloadErrored(url, "No space left on device"); // TODO localize; TODO cancel all rips + observer.downloadErrored(ripUrlId, "No space left on device"); // TODO localize; TODO cancel all rips return; } logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); - observer.downloadErrored(url, e.getMessage()); + observer.downloadErrored(ripUrlId, e.getMessage()); return; } catch (URISyntaxException e) { logger.debug("IOException", e); logger.error("[!] " + Utils.getLocalizedString("exception.while.downloading.file") + ": " + url + " - " + e.getMessage()); - observer.downloadErrored(url, Utils.getLocalizedString("exception.while.downloading.file")); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("exception.while.downloading.file")); return; } catch (NullPointerException npe){ logger.error("[!] " + Utils.getLocalizedString("failed.to.download") + " for URL " + url); - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; @@ -318,7 +360,7 @@ public void run() { if (tries > this.retries) { logger.error("[!] " + Utils.getLocalizedString("exceeded.maximum.retries") + " (" + this.retries + ") for URL " + url); - observer.downloadErrored(url, + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.download") + " " + url.toExternalForm()); return; } else { @@ -326,9 +368,23 @@ public void run() { Utils.sleep(retrySleep); } } + + // get fresh URL for the next attempt + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + } while (true); - observer.downloadCompleted(url, saveAs.toPath()); - logger.info("[+] Saved " + url + " as " + this.prettySaveAs); + observer.downloadCompleted(ripUrlId, saveAs.toPath()); + logger.info("[+] Saved " + url + " as " + prettySaveAs); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 3d070e8c2..608003528 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -1,10 +1,8 @@ package com.rarchives.ripme.ripper; -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; +import java.io.*; import java.net.HttpURLConnection; +import java.net.URISyntaxException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; @@ -15,6 +13,7 @@ import com.rarchives.ripme.utils.Utils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jsoup.HttpStatusException; /** * Thread for downloading files. @@ -24,17 +23,19 @@ class DownloadVideoThread implements Runnable { private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); - private final URL url; - private final Path saveAs; - private final String prettySaveAs; + private final TokenedUrlGetter tokenedUrlGetter; // Some URLs may be valid for a limited time. This should get a fresh url + private final RipUrlId ripUrlId; + private final Path directory; + private String filename; private final AbstractRipper observer; private final int retries; - public DownloadVideoThread(URL url, Path saveAs, AbstractRipper observer) { + public DownloadVideoThread(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, AbstractRipper observer) { super(); - this.url = url; - this.saveAs = saveAs; - this.prettySaveAs = Utils.removeCWD(saveAs); + this.tokenedUrlGetter = tug; + this.ripUrlId = ripUrlId; + this.directory = directory; + this.filename = filename; this.observer = observer; this.retries = Utils.getConfigInteger("download.retries", 1); } @@ -45,13 +46,44 @@ public DownloadVideoThread(URL url, Path saveAs, AbstractRipper observer) { */ @Override public void run() { - try { - observer.stopCheck(); - } catch (IOException e) { + if (observer.isStopped()) { // TODO create status for gracefully-stopped download - observer.downloadErrored(url, "Download interrupted"); + observer.downloadErrored(ripUrlId, "Download interrupted"); return; } + URL url = null; + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + if (filename == null) { + // Strip token query parameters + filename = Path.of(url.getPath()).getFileName().toString(); + } + if (AbstractRipper.shouldIgnoreURL(url)) { + observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return; + } + if (!Files.exists(directory)) { + logger.info("[+] Creating directory: " + directory); + try { + Files.createDirectories(directory); + } catch (IOException e) { + logger.error("Error creating directory", e); + observer.downloadErrored(ripUrlId, "Error creating directory: " + directory + " ; " + e.getMessage()); + return; + } + } + Path saveAs = directory.resolve(filename); + String prettySaveAs = Utils.removeCWD(saveAs); + if (Files.exists(saveAs)) { if (Utils.getConfigBoolean("file.overwrite", false)) { logger.info("[!] Deleting existing file" + prettySaveAs); @@ -62,22 +94,22 @@ public void run() { } } else { logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); - observer.downloadExists(url, saveAs); + observer.downloadExists(ripUrlId, saveAs); return; } } int bytesTotal, bytesDownloaded = 0; try { - bytesTotal = getTotalBytes(this.url); + bytesTotal = getTotalBytes(url); } catch (IOException e) { - logger.error("Failed to get file size at " + this.url, e); - observer.downloadErrored(this.url, "Failed to get file size of " + this.url); + logger.error("Failed to get file size at " + url, e); + observer.downloadErrored(ripUrlId, "Failed to get file size of " + url); return; } observer.setBytesTotal(bytesTotal); observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + this.url + " = " + bytesTotal + "b"); + logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); int tries = 0; // Number of attempts to download do { @@ -90,16 +122,16 @@ public void run() { // Setup HTTP request HttpURLConnection huc; - if (this.url.toString().startsWith("https")) { - huc = (HttpsURLConnection) this.url.openConnection(); + if (url.getProtocol().equals("https")) { + huc = (HttpsURLConnection) url.openConnection(); } else { - huc = (HttpURLConnection) this.url.openConnection(); + huc = (HttpURLConnection) url.openConnection(); } huc.setInstanceFollowRedirects(true); huc.setConnectTimeout(0); // Never timeout huc.setRequestProperty("accept", "*/*"); - huc.setRequestProperty("Referer", this.url.toExternalForm()); // Sic + huc.setRequestProperty("Referer", url.toExternalForm()); // Sic huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); tries += 1; logger.debug("Request properties: " + huc.getRequestProperties().toString()); @@ -109,7 +141,7 @@ public void run() { fos = Files.newOutputStream(saveAs); while ( (bytesRead = bis.read(data)) != -1) { if (observer.isPanicked()) { - observer.downloadErrored(url, "Download interrupted"); + observer.downloadErrored(ripUrlId, "Download interrupted"); return; } fos.write(data, 0, bytesRead); @@ -134,12 +166,26 @@ public void run() { } if (tries > this.retries) { logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); - observer.downloadErrored(url, "Failed to download " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, "Failed to download " + url.toExternalForm()); return; } + + // get fresh URL for the next attempt + try { + url = tokenedUrlGetter.getTokenedUrl(); + } catch (HttpStatusException e) { + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + logger.error("[!] Failed to get URL for " + ripUrlId); + return; // do not retry + } catch (IOException | URISyntaxException e) { + logger.error("[!] Failed to get URL for " + ripUrlId, e); + observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + return; // do not retry + } + } while (true); - observer.downloadCompleted(url, saveAs); - logger.info("[+] Saved " + url + " as " + this.prettySaveAs); + observer.downloadCompleted(ripUrlId, saveAs); + logger.info("[+] Saved " + url + " as " + prettySaveAs); } /** @@ -152,7 +198,7 @@ private int getTotalBytes(URL url) throws IOException { HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("HEAD"); conn.setRequestProperty("accept", "*/*"); - conn.setRequestProperty("Referer", this.url.toExternalForm()); // Sic + conn.setRequestProperty("Referer", url.toExternalForm()); // Sic conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); return conn.getContentLength(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java b/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java new file mode 100644 index 000000000..9e376bb2e --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/RipUrlId.java @@ -0,0 +1,97 @@ +package com.rarchives.ripme.ripper; + +import java.net.URL; +import java.util.Objects; + +/** + * RipUrlId represents a unique file on a host. + * Necessary because some files may be accessible from multiple URLs, for example: + * - a file in multiple albums, or + * - a file only accessible with a tokened URL. + */ +public class RipUrlId { + Class ripper; + String ripperHost; + String ripUrlId; + URL url; + + /** + * @param ripper The ripper associated with the id + * @param ripperHost The ripper's getHost(), because a ripper may support multiple hosts + * @param ripUrlId The unique identifier of the file fetchable by the ripper + */ + public RipUrlId(Class ripper, String ripperHost, String ripUrlId) { + if (ripper == null) { + throw new IllegalArgumentException("ripper cannot be null"); + } + if (ripperHost == null) { + throw new IllegalArgumentException("ripperHost cannot be null"); + } + if (ripUrlId == null) { + throw new IllegalArgumentException("ripUrlId cannot be null"); + } + this.ripper = ripper; + this.ripperHost = ripperHost; + this.ripUrlId = ripUrlId; + } + + /** + * Transitionary constructor for rippers that do not yet create an id + * + * @param ripper The ripper associated with the id + * @param ripperHost The ripper's getHost(), because a ripper may support multiple hosts + * @param url A URL fetchable by the ripper + * @deprecated The other constructor is preferable + */ + @Deprecated + public RipUrlId(Class ripper, String ripperHost, URL url) { + if (ripper == null) { + throw new IllegalArgumentException("ripper cannot be null"); + } + if (ripperHost == null) { + throw new IllegalArgumentException("ripperHost cannot be null"); + } + if (url == null) { + throw new IllegalArgumentException("url cannot be null"); + } + this.ripper = ripper; + this.ripperHost = ripperHost; + this.url = url; + } + + public Class getRipper() { + return ripper; + } + + public String getRipperHost() { + return ripperHost; + } + + public String getRipUrlId() { + return ripUrlId; + } + + public URL getUrl() { + return url; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + RipUrlId ripUrlId1 = (RipUrlId) o; + return Objects.equals(ripper, ripUrlId1.ripper) && Objects.equals(ripperHost, ripUrlId1.ripperHost) && Objects.equals(ripUrlId, ripUrlId1.ripUrlId) && Objects.equals(url, ripUrlId1.url); + } + + @Override + public int hashCode() { + return Objects.hash(ripper, ripperHost, ripUrlId, url); + } + + @Override + public String toString() { + if (url != null) { + return url.toString(); + } + return ripper.getSimpleName() + ": " + ripperHost + ": " + ripUrlId; + } +} diff --git a/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java b/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java new file mode 100644 index 000000000..706c7bc72 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/TokenedUrlGetter.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.ripper; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; + +public interface TokenedUrlGetter { + /** + * @return The URL of the file to fetch + * @throws IOException May be thrown if a tokened URI can't be fetched + * @throws URISyntaxException May be thrown if a URI can't be constructed + */ + URL getTokenedUrl() throws IOException, URISyntaxException; +} diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 67515e789..8fe4faeaf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -49,10 +49,21 @@ public String getAlbumTitle(URL url) { } @Override - public boolean addURLToDownload(URL url, Path saveAs) { + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { if (Utils.getConfigBoolean("urls_only.save", false)) { // Output URL to file String urlFile = this.workingDir + "/urls.txt"; + URL url = null; + try { + url = tug.getTokenedUrl(); + } catch (IOException | URISyntaxException e) { + logger.error("Unable to get URL for {}", ripUrlId, e); + return false; + } + if (AbstractRipper.shouldIgnoreURL(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return false; + } try (FileWriter fw = new FileWriter(urlFile, true)) { fw.write(url.toExternalForm()); @@ -64,28 +75,25 @@ public boolean addURLToDownload(URL url, Path saveAs) { logger.error("Error while writing to " + urlFile, e); return false; } + return true; } else { if (isThisATest()) { // Tests shouldn't download the whole video // Just change this.url to the download URL so the test knows we found it. logger.debug("Test rip, found URL: " + url); - this.url = url; + try { + this.url = tug.getTokenedUrl(); + } catch (IOException | URISyntaxException e) { + throw new RuntimeException(e); + } return true; } - if (shouldIgnoreURL(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - threadPool.addThread(new DownloadVideoThread(url, saveAs, this)); + + threadPool.addThread(new DownloadVideoThread(tug, ripUrlId, directory, filename, this)); } return true; } - @Override - public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { - return addURLToDownload(url, saveAs); - } - /** * Creates & sets working directory based on URL. * @@ -123,11 +131,11 @@ public int getCompletionPercentage() { /** * Runs if download successfully completed. * - * @param url Target URL - * @param saveAs Path to file, including filename. + * @param ripUrlId Target URL ID + * @param saveAs Path to file, including filename. */ @Override - public void downloadCompleted(URL url, Path saveAs) { + public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { if (observer == null) { return; } @@ -146,31 +154,32 @@ public void downloadCompleted(URL url, Path saveAs) { /** * Runs if the download errored somewhere. * - * @param url Target URL - * @param reason Reason why the download failed. + * @param ripUrlId Target URL ID + * @param reason Reason why the download failed. */ @Override - public void downloadErrored(URL url, String reason) { + public void downloadErrored(RipUrlId ripUrlId, String reason) { if (observer == null) { return; } - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason)); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); checkIfComplete(); } /** * Runs if user tries to redownload an already existing File. - * @param url Target URL - * @param file Existing file + * + * @param ripUrlId Target URL ID + * @param file Existing file */ @Override - public void downloadExists(URL url, Path file) { + public void downloadExists(RipUrlId ripUrlId, Path file) { if (observer == null) { return; } - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); checkIfComplete(); } From c9edecda0573a43e45c10ba37ab48d8761b2faa7 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 12:54:34 -0400 Subject: [PATCH 18/78] Use more specific method name --- .../java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/AbstractJSONRipper.java | 2 +- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 2 +- src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java | 2 +- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- .../java/com/rarchives/ripme/ripper/DownloadVideoThread.java | 2 +- src/main/java/com/rarchives/ripme/ripper/VideoRipper.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 40a3e0a56..0c0b72731 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -373,7 +373,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di itemsErrored.put(ripUrlId, e.getMessage()); return false; } - if (AbstractRipper.shouldIgnoreURL(url)) { + if (AbstractRipper.shouldIgnoreExtension(url)) { sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return false; } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 9146375e9..7910bae77 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -176,7 +176,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di itemsErrored.put(ripUrlId, e.getMessage()); return false; } - if (shouldIgnoreURL(url)) { + if (AbstractRipper.shouldIgnoreExtension(url)) { sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return false; } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 976717de0..e988b0088 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -819,7 +819,7 @@ protected boolean tryResumeDownload() { return false; } - protected static boolean shouldIgnoreURL(URL url) { // TODO change to "shouldIgnoreExtension" + protected static boolean shouldIgnoreExtension(URL url) { final String[] ignoredExtensions = Utils.getConfigStringArray("download.ignore_extensions"); if (ignoredExtensions == null || ignoredExtensions.length == 0) return false; // nothing ignored diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 83762f3b7..6df442424 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -88,7 +88,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di itemsErrored.put(ripUrlId, e.getMessage()); return false; } - if (shouldIgnoreURL(url)) { + if (AbstractRipper.shouldIgnoreExtension(url)) { sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return false; } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 5e7d3878b..b134412c5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -92,7 +92,7 @@ public void run() { } // First thing we make sure the file name doesn't have any illegal chars in it filename = Utils.sanitizeSaveAs(filename); - if (AbstractRipper.shouldIgnoreURL(url)) { + if (AbstractRipper.shouldIgnoreExtension(url)) { observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return; } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java index 608003528..9a3f6f7cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java @@ -67,7 +67,7 @@ public void run() { // Strip token query parameters filename = Path.of(url.getPath()).getFileName().toString(); } - if (AbstractRipper.shouldIgnoreURL(url)) { + if (AbstractRipper.shouldIgnoreExtension(url)) { observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return; } diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 8fe4faeaf..d717ef851 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -60,7 +60,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di logger.error("Unable to get URL for {}", ripUrlId, e); return false; } - if (AbstractRipper.shouldIgnoreURL(url)) { + if (AbstractRipper.shouldIgnoreExtension(url)) { sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); return false; } From a1d063489c6ecb81c55cab06219fb26ae8cd6e27 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:09:20 -0400 Subject: [PATCH 19/78] Remove dead code --- .../com/rarchives/ripme/ripper/AlbumRipper.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 6df442424..1cbae7568 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -116,19 +116,6 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di return true; } - /** - * Queues image to be downloaded and saved. - * Uses filename from URL to decide filename. - * @param url - * URL to download - * @return - * True on success - */ - protected boolean addURLToDownload(URL url) { - // Use empty prefix and empty subdirectory - return addURLToDownload(url, "", ""); - } - @Override /** * Cleans up & tells user about successful download From 95bd2cd76ed30d2e9cc63d67cdd72ed828dc9c0d Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:46:59 -0400 Subject: [PATCH 20/78] Extract duplicate code --- .../ripme/ripper/AbstractHTMLRipper.java | 77 ------------------ .../ripme/ripper/AbstractJSONRipper.java | 78 ------------------- .../ripme/ripper/AbstractRipper.java | 65 +++++++++++++--- .../rarchives/ripme/ripper/AlbumRipper.java | 78 ------------------- .../rarchives/ripme/ripper/VideoRipper.java | 70 +---------------- 5 files changed, 54 insertions(+), 314 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 0c0b72731..4bee19e12 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -32,9 +32,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class); - private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); Document cachedFirstPage; protected AbstractHTMLRipper(URL url) throws IOException { @@ -334,14 +331,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /* - Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - /* Queues multiple URLs of single images to download from a single Album URL */ @@ -414,72 +403,6 @@ protected boolean addURLToDownload(URL url) { return addURLToDownload(url, "", ""); } - /* - Cleans up & tells user about successful download - */ - @Override - public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - /* - * Cleans up & tells user about failed download. - */ - @Override - public void downloadErrored(RipUrlId ripUrlId, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(ripUrlId); - itemsErrored.put(ripUrlId, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - - checkIfComplete(); - } - - /* - Tells user that a single file in the album they wish to download has - already been downloaded in the past. - */ - @Override - public void downloadExists(RipUrlId ripUrlId, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - notifyComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 7910bae77..3935fcc63 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -28,10 +28,6 @@ public abstract class AbstractJSONRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AbstractJSONRipper.class); - private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); - protected AbstractJSONRipper(URL url) throws IOException { super(url); } @@ -137,14 +133,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - @Override /** * Queues multiple URLs of single images to download from a single Album URL @@ -217,72 +205,6 @@ protected boolean addURLToDownload(URL url) { return addURLToDownload(url, "", ""); } - /** - * Cleans up & tells user about successful download - */ - @Override - public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - /** - * Cleans up & tells user about failed download. - */ - @Override - public void downloadErrored(RipUrlId ripUrlId, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(ripUrlId); - itemsErrored.put(ripUrlId, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - - checkIfComplete(); - } - - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - @Override - public void downloadExists(RipUrlId ripUrlId, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - notifyComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index e988b0088..709cb95ab 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -14,12 +14,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Observable; -import java.util.Random; -import java.util.Scanner; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.LogManager; @@ -45,6 +40,11 @@ public abstract class AbstractRipper implements RipperInterface, Runnable { private static final Logger logger = LogManager.getLogger(AbstractRipper.class); + + protected final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); + protected final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); + protected final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + private final String URLHistoryFile = Utils.getURLHistoryFile(); public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"; @@ -532,27 +532,68 @@ public void retrievingSource(String url) { * @param ripUrlId The RipUrlId that was completed. * @param saveAs Where the downloaded file is stored. */ - public abstract void downloadCompleted(RipUrlId ripUrlId, Path saveAs); + protected void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { + if (observer == null) { + return; + } + try { + String path = Utils.removeCWD(saveAs); + RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, saveAs); + observer.update(this, msg); + + checkIfComplete(); + } catch (Exception e) { + logger.error("Exception while updating observer: ", e); + } + } /** * Notifies observers that a file could not be downloaded (includes a reason). */ - public abstract void downloadErrored(RipUrlId ripUrlId, String reason); + protected void downloadErrored(RipUrlId ripUrlId, String reason) { + if (observer == null) { + return; + } + itemsPending.remove(ripUrlId); + itemsErrored.put(ripUrlId, reason); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); + + checkIfComplete(); + } /** * Notify observers that a download could not be completed, * but was not technically an "error". */ - public abstract void downloadExists(RipUrlId ripUrlId, Path file); + protected void downloadExists(RipUrlId ripUrlId, Path file) { + if (observer == null) { + return; + } + + itemsPending.remove(ripUrlId); + itemsCompleted.put(ripUrlId, file); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); + + checkIfComplete(); + } /** * @return Number of files downloaded. */ - int getCount() { - return 1; + public int getCount() { + return itemsCompleted.size() + itemsErrored.size(); } - protected abstract void checkIfComplete(); + /** + * Checks if complete and notifies observers if complete + */ + protected void checkIfComplete() { + if (itemsPending.isEmpty()) { + notifyComplete(); + } + } /** * Notifies observers and updates state if all files have been ripped. diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 1cbae7568..8b40a82bf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -30,10 +30,6 @@ public abstract class AlbumRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(AlbumRipper.class); - private final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); - private final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); - private final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); - protected AlbumRipper(URL url) throws IOException { super(url); } @@ -48,14 +44,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Returns total amount of files attempted. - */ - public int getCount() { - return itemsCompleted.size() + itemsErrored.size(); - } - @Override /** * Queues multiple URLs of single images to download from a single Album URL @@ -116,72 +104,6 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di return true; } - @Override - /** - * Cleans up & tells user about successful download - */ - public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { - if (observer == null) { - return; - } - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, saveAs); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - @Override - /** - * Cleans up & tells user about failed download. - */ - public void downloadErrored(RipUrlId ripUrlId, String reason) { - if (observer == null) { - return; - } - itemsPending.remove(ripUrlId); - itemsErrored.put(ripUrlId, reason); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - - checkIfComplete(); - } - - @Override - /** - * Tells user that a single file in the album they wish to download has - * already been downloaded in the past. - */ - public void downloadExists(RipUrlId ripUrlId, Path file) { - if (observer == null) { - return; - } - - itemsPending.remove(ripUrlId); - itemsCompleted.put(ripUrlId, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); - - checkIfComplete(); - } - - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - if (itemsPending.isEmpty()) { - notifyComplete(); - } - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index d717ef851..5279c66f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -89,6 +89,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di return true; } + itemsPending.add(ripUrlId); threadPool.addThread(new DownloadVideoThread(tug, ripUrlId, directory, filename, this)); } return true; @@ -128,61 +129,6 @@ public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - /** - * Runs if download successfully completed. - * - * @param ripUrlId Target URL ID - * @param saveAs Path to file, including filename. - */ - @Override - public void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { - if (observer == null) { - return; - } - - try { - String path = Utils.removeCWD(saveAs); - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path); - observer.update(this, msg); - - checkIfComplete(); - } catch (Exception e) { - logger.error("Exception while updating observer: ", e); - } - } - - /** - * Runs if the download errored somewhere. - * - * @param ripUrlId Target URL ID - * @param reason Reason why the download failed. - */ - @Override - public void downloadErrored(RipUrlId ripUrlId, String reason) { - if (observer == null) { - return; - } - - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - checkIfComplete(); - } - - /** - * Runs if user tries to redownload an already existing File. - * - * @param ripUrlId Target URL ID - * @param file Existing file - */ - @Override - public void downloadExists(RipUrlId ripUrlId, Path file) { - if (observer == null) { - return; - } - - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); - checkIfComplete(); - } - /** * Gets the status and changes it to a human-readable form. * @@ -206,18 +152,4 @@ public URL sanitizeURL(URL url) throws MalformedURLException { return url; } - /** - * Notifies observers and updates state if all files have been ripped. - */ - @Override - protected void checkIfComplete() { - if (observer == null) { - return; - } - - if (bytesCompleted >= bytesTotal) { - notifyComplete(); - } - } - } From 860ff7a1a4f79bcf39ae616b913d5b74801521b9 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:57:08 -0400 Subject: [PATCH 21/78] Replace DownloadVideoThread with DownloadFileThread --- .../ripme/ripper/DownloadVideoThread.java | 206 ------------------ .../rarchives/ripme/ripper/VideoRipper.java | 7 +- 2 files changed, 6 insertions(+), 207 deletions(-) delete mode 100644 src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java deleted file mode 100644 index 9a3f6f7cf..000000000 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadVideoThread.java +++ /dev/null @@ -1,206 +0,0 @@ -package com.rarchives.ripme.ripper; - -import java.io.*; -import java.net.HttpURLConnection; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; - -import javax.net.ssl.HttpsURLConnection; - -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; -import com.rarchives.ripme.utils.Utils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jsoup.HttpStatusException; - -/** - * Thread for downloading files. - * Includes retry logic, observer notifications, and other goodies. - */ -class DownloadVideoThread implements Runnable { - - private static final Logger logger = LogManager.getLogger(DownloadVideoThread.class); - - private final TokenedUrlGetter tokenedUrlGetter; // Some URLs may be valid for a limited time. This should get a fresh url - private final RipUrlId ripUrlId; - private final Path directory; - private String filename; - private final AbstractRipper observer; - private final int retries; - - public DownloadVideoThread(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, AbstractRipper observer) { - super(); - this.tokenedUrlGetter = tug; - this.ripUrlId = ripUrlId; - this.directory = directory; - this.filename = filename; - this.observer = observer; - this.retries = Utils.getConfigInteger("download.retries", 1); - } - - /** - * Attempts to download the file. Retries as needed. - * Notifies observers upon completion/error/warn. - */ - @Override - public void run() { - if (observer.isStopped()) { - // TODO create status for gracefully-stopped download - observer.downloadErrored(ripUrlId, "Download interrupted"); - return; - } - URL url = null; - try { - url = tokenedUrlGetter.getTokenedUrl(); - } catch (HttpStatusException e) { - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); - logger.error("[!] Failed to get URL for " + ripUrlId); - return; // do not retry - } catch (IOException | URISyntaxException e) { - logger.error("[!] Failed to get URL for " + ripUrlId, e); - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); - return; // do not retry - } - if (filename == null) { - // Strip token query parameters - filename = Path.of(url.getPath()).getFileName().toString(); - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return; - } - if (!Files.exists(directory)) { - logger.info("[+] Creating directory: " + directory); - try { - Files.createDirectories(directory); - } catch (IOException e) { - logger.error("Error creating directory", e); - observer.downloadErrored(ripUrlId, "Error creating directory: " + directory + " ; " + e.getMessage()); - return; - } - } - Path saveAs = directory.resolve(filename); - String prettySaveAs = Utils.removeCWD(saveAs); - - if (Files.exists(saveAs)) { - if (Utils.getConfigBoolean("file.overwrite", false)) { - logger.info("[!] Deleting existing file" + prettySaveAs); - try { - Files.delete(saveAs); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - logger.info("[!] Skipping " + url + " -- file already exists: " + prettySaveAs); - observer.downloadExists(ripUrlId, saveAs); - return; - } - } - - int bytesTotal, bytesDownloaded = 0; - try { - bytesTotal = getTotalBytes(url); - } catch (IOException e) { - logger.error("Failed to get file size at " + url, e); - observer.downloadErrored(ripUrlId, "Failed to get file size of " + url); - return; - } - observer.setBytesTotal(bytesTotal); - observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); - logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); - - int tries = 0; // Number of attempts to download - do { - InputStream bis = null; OutputStream fos = null; - byte[] data = new byte[1024 * 256]; - int bytesRead; - try { - logger.info(" Downloading file: " + url + (tries > 0 ? " Try #" + tries+1 : "")); - observer.sendUpdate(STATUS.DOWNLOAD_STARTED, url.toExternalForm()); - - // Setup HTTP request - HttpURLConnection huc; - if (url.getProtocol().equals("https")) { - huc = (HttpsURLConnection) url.openConnection(); - } - else { - huc = (HttpURLConnection) url.openConnection(); - } - huc.setInstanceFollowRedirects(true); - huc.setConnectTimeout(0); // Never timeout - huc.setRequestProperty("accept", "*/*"); - huc.setRequestProperty("Referer", url.toExternalForm()); // Sic - huc.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - tries += 1; - logger.debug("Request properties: " + huc.getRequestProperties().toString()); - huc.connect(); - // Check status code - bis = new BufferedInputStream(huc.getInputStream()); - fos = Files.newOutputStream(saveAs); - while ( (bytesRead = bis.read(data)) != -1) { - if (observer.isPanicked()) { - observer.downloadErrored(ripUrlId, "Download interrupted"); - return; - } - fos.write(data, 0, bytesRead); - observer.sendUpdate(STATUS.CHUNK_BYTES, bytesRead); - bytesDownloaded += bytesRead; - observer.setBytesCompleted(bytesDownloaded); - observer.sendUpdate(STATUS.COMPLETED_BYTES, bytesDownloaded); - } - bis.close(); - fos.close(); - break; // Download successful: break out of infinite loop - } catch (IOException e) { - logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e); - } finally { - // Close any open streams - try { - if (bis != null) { bis.close(); } - } catch (IOException ignored) { } - try { - if (fos != null) { fos.close(); } - } catch (IOException ignored) { } - } - if (tries > this.retries) { - logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url); - observer.downloadErrored(ripUrlId, "Failed to download " + url.toExternalForm()); - return; - } - - // get fresh URL for the next attempt - try { - url = tokenedUrlGetter.getTokenedUrl(); - } catch (HttpStatusException e) { - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); - logger.error("[!] Failed to get URL for " + ripUrlId); - return; // do not retry - } catch (IOException | URISyntaxException e) { - logger.error("[!] Failed to get URL for " + ripUrlId, e); - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); - return; // do not retry - } - - } while (true); - observer.downloadCompleted(ripUrlId, saveAs); - logger.info("[+] Saved " + url + " as " + prettySaveAs); - } - - /** - * @param url - * Target URL - * @return - * Returns connection length - */ - private int getTotalBytes(URL url) throws IOException { - HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestMethod("HEAD"); - conn.setRequestProperty("accept", "*/*"); - conn.setRequestProperty("Referer", url.toExternalForm()); // Sic - conn.setRequestProperty("User-agent", AbstractRipper.USER_AGENT); - return conn.getContentLength(); - } - -} \ No newline at end of file diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 5279c66f2..2852fd973 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -48,6 +48,11 @@ public String getAlbumTitle(URL url) { return "videos"; } + @Override + protected boolean useByteProgessBar() { + return true; + } + @Override public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { if (Utils.getConfigBoolean("urls_only.save", false)) { @@ -90,7 +95,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di } itemsPending.add(ripUrlId); - threadPool.addThread(new DownloadVideoThread(tug, ripUrlId, directory, filename, this)); + threadPool.addThread(new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME)); } return true; } From ee37c7452892e2249a5829566e297c35d34f452d Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:40:23 -0400 Subject: [PATCH 22/78] Extract duplicate code --- .../ripme/ripper/AbstractHTMLRipper.java | 59 ---------------- .../ripme/ripper/AbstractJSONRipper.java | 59 ---------------- .../ripme/ripper/AbstractRipper.java | 67 ++++++++++++++++++- .../rarchives/ripme/ripper/AlbumRipper.java | 60 ----------------- .../rarchives/ripme/ripper/VideoRipper.java | 48 +------------ 5 files changed, 67 insertions(+), 226 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 4bee19e12..a29a318cb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -331,65 +331,6 @@ protected boolean allowDuplicates() { return false; } - /* - Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.contains(ripUrlId) - || itemsCompleted.containsKey(ripUrlId) - || itemsErrored.containsKey(ripUrlId) )) { - // Item is already downloaded/downloading, skip it. - // TODO print path if in itemsCompleted or itemsErrored - logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); - return false; - } - - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - URL url = null; - try { - url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to get URL for {}", ripUrlId, e); - itemsErrored.put(ripUrlId, e.getMessage()); - return false; - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(ripUrlId, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - return true; - } - else { - itemsPending.add(ripUrlId); - DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - /** * Queues image to be downloaded and saved. * Uses filename from URL to decide filename. diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 3935fcc63..fb31aaa55 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -133,65 +133,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.contains(ripUrlId) - || itemsCompleted.containsKey(ripUrlId) - || itemsErrored.containsKey(ripUrlId) )) { - // Item is already downloaded/downloading, skip it. - // TODO print path if in itemsCompleted or itemsErrored - logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); - return false; - } - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - URL url = null; - try { - url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to get URL for {}", ripUrlId, e); - itemsErrored.put(ripUrlId, e.getMessage()); - return false; - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(ripUrlId, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - return true; - } - else { - itemsPending.add(ripUrlId); - DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - /** * Queues image to be downloaded and saved. * Uses filename from URL to decide filename. diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 709cb95ab..640199c83 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -11,9 +11,11 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; @@ -64,6 +66,8 @@ public abstract class AbstractRipper public abstract String getGID(URL url) throws MalformedURLException, URISyntaxException; + protected abstract boolean allowDuplicates(); + public boolean hasASAPRipping() { return false; } @@ -271,7 +275,66 @@ protected boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path return addURLToDownload(tug, ripUrlId, directory, null, referrer, cookies, getFileExtFromMIME); } - public abstract boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME); + /** + * Queues multiple URLs of single images to download from a single Album URL + */ + public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { + // Only download one file if this is a test. + if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { + stop(); + itemsPending.clear(); + return false; + } + if (!allowDuplicates() + && ( itemsPending.contains(ripUrlId) + || itemsCompleted.containsKey(ripUrlId) + || itemsErrored.containsKey(ripUrlId) )) { + // Item is already downloaded/downloading, skip it. + // TODO print path if in itemsCompleted or itemsErrored + logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); + return false; + } + + if (Utils.getConfigBoolean("urls_only.save", false)) { + // Output URL to file + Path urlFile = Paths.get(this.workingDir + "/urls.txt"); + URL url = null; + try { + url = tug.getTokenedUrl(); + } catch (IOException | URISyntaxException e) { + logger.error("Unable to get URL for {}", ripUrlId, e); + itemsErrored.put(ripUrlId, e.getMessage()); + return false; + } + if (AbstractRipper.shouldIgnoreExtension(url)) { + sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + return false; + } + String text = url.toExternalForm() + System.lineSeparator(); + try { + Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); + itemsCompleted.put(ripUrlId, urlFile); + } catch (IOException e) { + logger.error("Error while writing to " + urlFile, e); + return false; + } + return true; + } + else { + itemsPending.add(ripUrlId); + DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); + if (referrer != null) { + dft.setReferrer(referrer); + } + if (cookies != null) { + dft.setCookies(cookies); + } + threadPool.addThread(dft); + } + + return true; + } + /** @@ -590,7 +653,7 @@ public int getCount() { * Checks if complete and notifies observers if complete */ protected void checkIfComplete() { - if (itemsPending.isEmpty()) { + if (itemsPending.isEmpty()) { // TODO add itemsActive for current transfers notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 8b40a82bf..1570e11b0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -44,66 +44,6 @@ protected boolean allowDuplicates() { return false; } - @Override - /** - * Queues multiple URLs of single images to download from a single Album URL - */ - public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { - // Only download one file if this is a test. - if (super.isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) { - stop(); - itemsPending.clear(); - return false; - } - if (!allowDuplicates() - && ( itemsPending.contains(ripUrlId) - || itemsCompleted.containsKey(ripUrlId) - || itemsErrored.containsKey(ripUrlId) )) { - // Item is already downloaded/downloading, skip it. - // TODO print path if in itemsCompleted or itemsErrored - logger.info("[!] Skipping " + ripUrlId + " -- already attempted: " + Utils.removeCWD(directory)); - return false; - } - - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - Path urlFile = Paths.get(this.workingDir + "/urls.txt"); - URL url = null; - try { - url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to get URL for {}", ripUrlId, e); - itemsErrored.put(ripUrlId, e.getMessage()); - return false; - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - String text = url.toExternalForm() + System.lineSeparator(); - try { - Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND); - itemsCompleted.put(ripUrlId, urlFile); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - } - return true; - } - else { - itemsPending.add(ripUrlId); - DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - threadPool.addThread(dft); - } - - return true; - } - /** * Sets directory to save all ripped files to. * @param url diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index 2852fd973..d2904feb2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -12,8 +12,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.rarchives.ripme.ui.RipStatusMessage; -import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Utils; public abstract class VideoRipper extends AbstractRipper { @@ -54,50 +52,8 @@ protected boolean useByteProgessBar() { } @Override - public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path directory, String filename, String referrer, Map cookies, Boolean getFileExtFromMIME) { - if (Utils.getConfigBoolean("urls_only.save", false)) { - // Output URL to file - String urlFile = this.workingDir + "/urls.txt"; - URL url = null; - try { - url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - logger.error("Unable to get URL for {}", ripUrlId, e); - return false; - } - if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); - return false; - } - - try (FileWriter fw = new FileWriter(urlFile, true)) { - fw.write(url.toExternalForm()); - fw.write("\n"); - - RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, urlFile); - observer.update(this, msg); - } catch (IOException e) { - logger.error("Error while writing to " + urlFile, e); - return false; - } - return true; - } else { - if (isThisATest()) { - // Tests shouldn't download the whole video - // Just change this.url to the download URL so the test knows we found it. - logger.debug("Test rip, found URL: " + url); - try { - this.url = tug.getTokenedUrl(); - } catch (IOException | URISyntaxException e) { - throw new RuntimeException(e); - } - return true; - } - - itemsPending.add(ripUrlId); - threadPool.addThread(new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME)); - } - return true; + protected boolean allowDuplicates() { + return false; } /** From 19c0061daa0266b770bb226b6aa487bfba2d3251 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 1 Sep 2025 01:51:09 -0400 Subject: [PATCH 23/78] Unify thread pools --- .../rarchives/ripme/ripper/AbstractHTMLRipper.java | 4 ---- .../rarchives/ripme/ripper/AbstractJSONRipper.java | 3 --- .../com/rarchives/ripme/ripper/AbstractRipper.java | 8 ++++++-- .../rarchives/ripme/ripper/DownloadThreadPool.java | 12 ++++++++---- .../ripme/ripper/rippers/DeviantartRipper.java | 8 +------- .../rarchives/ripme/ripper/rippers/E621Ripper.java | 9 +-------- .../ripme/ripper/rippers/EHentaiRipper.java | 9 +-------- .../rarchives/ripme/ripper/rippers/FlickrRipper.java | 8 -------- .../ripme/ripper/rippers/FuraffinityRipper.java | 9 --------- .../ripme/ripper/rippers/HqpornerRipper.java | 8 +------- .../ripme/ripper/rippers/ImagebamRipper.java | 9 +-------- .../ripme/ripper/rippers/ImagevenueRipper.java | 9 +-------- .../rarchives/ripme/ripper/rippers/ListalRipper.java | 9 +-------- .../ripme/ripper/rippers/MotherlessRipper.java | 10 +--------- .../rarchives/ripme/ripper/rippers/NfsfwRipper.java | 11 +---------- .../ripme/ripper/rippers/NhentaiRipper.java | 8 -------- .../ripme/ripper/rippers/PornhubRipper.java | 10 +--------- 17 files changed, 24 insertions(+), 120 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index a29a318cb..86214c3f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -69,10 +69,6 @@ protected List getDescriptionsFromPage(Document doc) throws IOException protected abstract void downloadURL(URL url, int index); - protected DownloadThreadPool getThreadPool() { - return null; - } - protected boolean keepSortOrder() { return true; } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index fb31aaa55..7143025d9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -42,9 +42,6 @@ protected JSONObject getNextPage(JSONObject doc) throws IOException, URISyntaxEx } protected abstract List getURLsFromJSON(JSONObject json); protected abstract void downloadURL(URL url, int index); - private DownloadThreadPool getThreadPool() { - return null; - } protected boolean keepSortOrder() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 640199c83..211792939 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -107,6 +107,10 @@ protected void stopCheck() throws IOException { } } + protected DownloadThreadPool getThreadPool() { + return threadPool; + } + /** * Adds a URL to the url history file * @@ -234,7 +238,7 @@ public void setup() throws IOException, URISyntaxException { // ctx.reconfigure(); // ctx.updateLoggers(); - this.threadPool = new DownloadThreadPool(); + this.threadPool = new DownloadThreadPool(getClass().getSimpleName() + "_" + getGID(url)); } public void setObserver(RipStatusHandler obs) { @@ -653,7 +657,7 @@ public int getCount() { * Checks if complete and notifies observers if complete */ protected void checkIfComplete() { - if (itemsPending.isEmpty()) { // TODO add itemsActive for current transfers + if (itemsPending.isEmpty() && threadPool.getActiveThreadCount() == 0 && threadPool.getPendingThreadCount() == 0) { notifyComplete(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 8ae43743f..353996237 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -16,10 +16,6 @@ public class DownloadThreadPool { private static final Logger logger = LogManager.getLogger(DownloadThreadPool.class); private ThreadPoolExecutor threadPool = null; - public DownloadThreadPool() { - initialize("Main"); - } - public DownloadThreadPool(String threadPoolName) { initialize(threadPoolName); } @@ -53,4 +49,12 @@ public void waitForThreads() { logger.error("[!] Interrupted while waiting for threads to finish: ", e); } } + + public int getPendingThreadCount() { + return threadPool.getQueue().size(); + } + + public int getActiveThreadCount() { + return threadPool.getActiveCount(); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 98510250c..e2db757f1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -94,7 +94,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { private boolean usingCatPath = false; private int downloadCount = 0; private Map cookies = new HashMap(); - private DownloadThreadPool deviantartThreadPool = new DownloadThreadPool("deviantart"); private ArrayList names = new ArrayList(); List allowedCookies = Arrays.asList("agegate_state", "userinfo", "auth", "auth_secure"); @@ -106,11 +105,6 @@ public class DeviantartRipper extends AbstractHTMLRipper { private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0"; private final String utilsKey = "DeviantartLogin.cookies"; //for config file - @Override - public DownloadThreadPool getThreadPool() { - return deviantartThreadPool; - } - public DeviantartRipper(URL url) throws IOException { super(url); } @@ -304,7 +298,7 @@ protected void downloadURL(URL url, int index) { // Start Thread and add to pool. DeviantartImageThread t = new DeviantartImageThread(url); - deviantartThreadPool.addThread(t); + getThreadPool().addThread(t); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index 9b40f0542..d18cb9ec8 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -37,8 +37,6 @@ public class E621Ripper extends AbstractHTMLRipper { private static Pattern gidPatternNew = null; private static Pattern gidPatternPoolNew = null; - private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); - private Map cookies = new HashMap(); private String userAgent = USER_AGENT; @@ -78,11 +76,6 @@ private Document getDocument(String url) throws IOException { return getDocument(url, 1); } - @Override - public DownloadThreadPool getThreadPool() { - return e621ThreadPool; - } - @Override public String getDomain() { return "e621.net"; @@ -136,7 +129,7 @@ public void downloadURL(final URL url, int index) { // rate limit sleep(3000); // addURLToDownload(url, getPrefix(index)); - e621ThreadPool.addThread(new E621FileThread(url, getPrefix(index))); + getThreadPool().addThread(new E621FileThread(url, getPrefix(index))); } private String getTerm(URL url) throws MalformedURLException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index 5349f55c7..a85fdd6ff 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -44,8 +44,6 @@ public class EHentaiRipper extends AbstractHTMLRipper { } private String lastURL = null; - // Thread pool for finding direct image links from "image" pages (html) - private final DownloadThreadPool ehentaiThreadPool = new DownloadThreadPool("ehentai"); // Current HTML document private Document albumDoc = null; @@ -53,11 +51,6 @@ public EHentaiRipper(URL url) throws IOException { super(url); } - @Override - public DownloadThreadPool getThreadPool() { - return ehentaiThreadPool; - } - @Override public String getHost() { return "e-hentai"; @@ -194,7 +187,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { EHentaiImageThread t = new EHentaiImageThread(url, index, this.workingDir.toPath()); - ehentaiThreadPool.addThread(t); + getThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 1f1954207..37acde4ac 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -28,8 +28,6 @@ public class FlickrRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(FlickrRipper.class); - private final DownloadThreadPool flickrThreadPool; - private enum UrlType { USER, PHOTOSET @@ -45,11 +43,6 @@ private class Album { } } - @Override - public DownloadThreadPool getThreadPool() { - return flickrThreadPool; - } - @Override public boolean hasASAPRipping() { return true; @@ -57,7 +50,6 @@ public boolean hasASAPRipping() { public FlickrRipper(URL url) throws IOException { super(url); - flickrThreadPool = new DownloadThreadPool(); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java index ad9075d0d..cb7c49479 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FuraffinityRipper.java @@ -54,15 +54,6 @@ private void warnAboutSharedAccount(String loginCookies) { } } - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool furaffinityThreadPool - = new DownloadThreadPool( "furaffinity"); - - @Override - public DownloadThreadPool getThreadPool() { - return furaffinityThreadPool; - } - public FuraffinityRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index 7183f1d79..b36d48923 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -30,7 +30,6 @@ public class HqpornerRipper extends AbstractHTMLRipper { private Pattern p1 = Pattern.compile("https?://hqporner.com/hdporn/([a-zA-Z0-9_-]*).html/?$"); // video pattern. private Pattern p2 = Pattern.compile("https://hqporner.com/([a-zA-Z0-9/_-]+)"); // category/top/actress/studio pattern. private Pattern p3 = Pattern.compile("https?://[A-Za-z0-9/.-_]+\\.mp4"); // to match links ending with .mp4 - private DownloadThreadPool hqpornerThreadPool = new DownloadThreadPool("hqpornerThreadPool"); private String subdirectory = ""; public HqpornerRipper(URL url) throws IOException { @@ -111,7 +110,7 @@ public boolean tryResumeDownload() { @Override public void downloadURL(URL url, int index) { - hqpornerThreadPool.addThread(new HqpornerDownloadThread(url, index, subdirectory)); + getThreadPool().addThread(new HqpornerDownloadThread(url, index, subdirectory)); } @Override @@ -123,11 +122,6 @@ public Document getNextPage(Document doc) throws IOException { throw new IOException("No next page found."); } - @Override - public DownloadThreadPool getThreadPool() { - return hqpornerThreadPool; - } - @Override public boolean useByteProgessBar() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index d3e5b2ce8..52d6b4e1a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -29,13 +29,6 @@ public class ImagebamRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(ImagebamRipper.class); - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool imagebamThreadPool = new DownloadThreadPool("imagebam"); - @Override - public DownloadThreadPool getThreadPool() { - return imagebamThreadPool; - } - public ImagebamRipper(URL url) throws IOException { super(url); } @@ -90,7 +83,7 @@ public List getURLsFromPage(Document doc) { @Override public void downloadURL(URL url, int index) { ImagebamImageThread t = new ImagebamImageThread(url, index); - imagebamThreadPool.addThread(t); + getThreadPool().addThread(t); sleep(500); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index 5421e14dd..2be15a05f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -25,13 +25,6 @@ public class ImagevenueRipper extends AbstractHTMLRipper { private static final Logger logger = LogManager.getLogger(ImagevenueRipper.class); - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool imagevenueThreadPool = new DownloadThreadPool("imagevenue"); - @Override - public DownloadThreadPool getThreadPool() { - return imagevenueThreadPool; - } - public ImagevenueRipper(URL url) throws IOException { super(url); } @@ -72,7 +65,7 @@ public List getURLsFromPage(Document doc) { public void downloadURL(URL url, int index) { ImagevenueImageThread t = new ImagevenueImageThread(url, index); - imagevenueThreadPool.addThread(t); + getThreadPool().addThread(t); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index 7157e49dd..9124a1495 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -38,8 +38,6 @@ public class ListalRipper extends AbstractHTMLRipper { private String listId = null; // listId to get more images via POST. private UrlType urlType = UrlType.UNKNOWN; - private DownloadThreadPool listalThreadPool = new DownloadThreadPool("listalThreadPool"); - public ListalRipper(URL url) throws IOException { super(url); } @@ -77,7 +75,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { - listalThreadPool.addThread(new ListalImageDownloadThread(url, index)); + getThreadPool().addThread(new ListalImageDownloadThread(url, index)); } @Override @@ -137,11 +135,6 @@ public Document getNextPage(Document page) throws IOException, URISyntaxExceptio } - @Override - public DownloadThreadPool getThreadPool() { - return listalThreadPool; - } - /** * Returns the image urls for UrlType LIST. */ diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index 955e85a34..ce46e9f3b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -31,11 +31,8 @@ public class MotherlessRipper extends AbstractHTMLRipper { private static final String DOMAIN = "motherless.com", HOST = "motherless"; - private DownloadThreadPool motherlessThreadPool; - public MotherlessRipper(URL url) throws IOException { super(url); - motherlessThreadPool = new DownloadThreadPool(); } @Override @@ -117,7 +114,7 @@ protected List getURLsFromPage(Document page) { protected void downloadURL(URL url, int index) { // Create thread for finding image at "url" page MotherlessImageRunnable mit = new MotherlessImageRunnable(url, index); - motherlessThreadPool.addThread(mit); + getThreadPool().addThread(mit); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -155,11 +152,6 @@ public String getGID(URL url) throws MalformedURLException { throw new MalformedURLException("Expected URL format: https://motherless.com/GIXXXXXXX, got: " + url); } - @Override - protected DownloadThreadPool getThreadPool() { - return motherlessThreadPool; - } - /** * Helper class to find and download images found on "image" pages */ diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index d6b17b02f..41540afee 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -34,12 +34,8 @@ public class NfsfwRipper extends AbstractHTMLRipper { "https?://[wm.]*nfsfw.com/gallery/v/[^/]+/(.+)$" ); - // threads pool for downloading images from image pages - private DownloadThreadPool nfsfwThreadPool; - public NfsfwRipper(URL url) throws IOException { super(url); - nfsfwThreadPool = new DownloadThreadPool("NFSFW"); } @Override @@ -105,7 +101,7 @@ protected void downloadURL(URL url, int index) { index = ++this.index; } NfsfwImageThread t = new NfsfwImageThread(url, currentDir, index); - nfsfwThreadPool.addThread(t); + getThreadPool().addThread(t); } @Override @@ -141,11 +137,6 @@ public String getGID(URL url) throws MalformedURLException { + " Got: " + url); } - @Override - public DownloadThreadPool getThreadPool() { - return nfsfwThreadPool; - } - @Override public boolean hasQueueSupport() { return true; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java index 41693a3ee..13df2aa74 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NhentaiRipper.java @@ -27,9 +27,6 @@ public class NhentaiRipper extends AbstractHTMLRipper { private Document firstPage; - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool nhentaiThreadPool = new DownloadThreadPool("nhentai"); - @Override public boolean hasQueueSupport() { return true; @@ -51,11 +48,6 @@ public List getAlbumsToQueue(Document doc) { return urlsToAddToQueue; } - @Override - public DownloadThreadPool getThreadPool() { - return nhentaiThreadPool; - } - public NhentaiRipper(URL url) throws IOException { super(url); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index 481ab1ede..7db21cb15 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -31,9 +31,6 @@ public class PornhubRipper extends AbstractHTMLRipper { private static final String DOMAIN = "pornhub.com", HOST = "Pornhub"; - // Thread pool for finding direct image links from "image" pages (html) - private DownloadThreadPool pornhubThreadPool = new DownloadThreadPool("pornhub"); - public PornhubRipper(URL url) throws IOException { super(url); } @@ -82,7 +79,7 @@ protected List getURLsFromPage(Document page) { @Override protected void downloadURL(URL url, int index) { PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir.toPath()); - pornhubThreadPool.addThread(t); + getThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { @@ -119,11 +116,6 @@ public String getGID(URL url) throws MalformedURLException { + " Got: " + url); } - @Override - public DownloadThreadPool getThreadPool(){ - return pornhubThreadPool; - } - public boolean canRip(URL url) { return url.getHost().endsWith(DOMAIN) && url.getPath().startsWith("/album"); } From ba88600a4ee126da09b29d8175ce6507c02a359d Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 9 Sep 2025 04:15:41 -0400 Subject: [PATCH 24/78] Fix race condition, handle dupes in albums If all downloads on a page are skipped, then checkIfComplete() would have said the rip is complete, letting the next ripper run while the last ripper continues to fetch the next pages Checking if complete depends on checking the number of crawled items, which needs to be done with a new counter, because Map and Set won't track duplicate files in an album The thread pools have also been given distinct names, to make the purpose more obvious, and because using the wrong thread can cause a hang --- .../ripme/ripper/AbstractHTMLRipper.java | 25 ++-- .../ripme/ripper/AbstractJSONRipper.java | 23 ++-- .../ripme/ripper/AbstractRipper.java | 129 ++++++++++++++---- .../ripme/ripper/DownloadThreadPool.java | 72 ++++++++-- .../ripper/rippers/DeviantartRipper.java | 3 +- .../ripme/ripper/rippers/E621Ripper.java | 3 +- .../ripme/ripper/rippers/EHentaiRipper.java | 3 +- .../ripme/ripper/rippers/HqpornerRipper.java | 3 +- .../ripme/ripper/rippers/ImagebamRipper.java | 3 +- .../ripper/rippers/ImagevenueRipper.java | 3 +- .../ripme/ripper/rippers/ImgurRipper.java | 2 +- .../ripme/ripper/rippers/ListalRipper.java | 3 +- .../ripper/rippers/MotherlessRipper.java | 2 +- .../ripme/ripper/rippers/NfsfwRipper.java | 3 +- .../ripme/ripper/rippers/PornhubRipper.java | 3 +- .../ripme/ripper/rippers/RedditRipper.java | 2 +- .../ripme/ripper/rippers/TumblrRipper.java | 2 +- .../ripme/ripper/rippers/VkRipper.java | 2 +- .../rippers/video/CliphunterRipper.java | 2 +- .../rippers/video/MotherlessVideoRipper.java | 2 +- .../ripper/rippers/video/PornhubRipper.java | 2 +- .../rippers/video/TwitchVideoRipper.java | 2 +- .../ripper/rippers/video/ViddmeRipper.java | 2 +- .../ripper/rippers/video/VidearnRipper.java | 2 +- .../ripme/ripper/rippers/video/VkRipper.java | 2 +- .../ripper/rippers/video/YuvutuRipper.java | 2 +- .../com/rarchives/ripme/ui/MainWindow.java | 31 +++-- 27 files changed, 232 insertions(+), 101 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 86214c3f1..2ba66554e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -110,7 +110,7 @@ protected boolean pageContainsAlbums(URL url) { @Override public void rip() throws IOException, URISyntaxException { - int index = 0; + int imageIndex = 0; int textindex = 0; logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); @@ -165,9 +165,9 @@ public void rip() throws IOException, URISyntaxException { } for (String imageURL : imageURLs) { - index += 1; - logger.debug("Found image url #" + index + ": '" + imageURL + "'"); - downloadURL(new URI(imageURL).toURL(), index); + imageIndex += 1; + logger.debug("Found image url #" + imageIndex + ": '" + imageURL + "'"); + downloadURL(new URI(imageURL).toURL(), imageIndex); if (isStopped() || isThisATest()) { break; } @@ -195,7 +195,7 @@ public void rip() throws IOException, URISyntaxException { workingDir.getCanonicalPath() + "" + File.separator - + getPrefix(index) + + getPrefix(imageIndex) + (tempDesc.length > 1 ? tempDesc[1] : filename) + ".txt").exists(); @@ -225,12 +225,17 @@ public void rip() throws IOException, URISyntaxException { } } - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); - getThreadPool().waitForThreads(); + logger.info("All items queued; total items: {}; url: {}", imageIndex, url); + + // Final total item count is now known + setItemsTotal(imageIndex); + + if (getCrawlerThreadPool() != null) { + logger.debug("Waiting for crawler threadpool: {}", url); + getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url); } - waitForThreads(); + + waitForRipperThreads(); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 7143025d9..5ed3c637c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -59,7 +59,7 @@ public URL sanitizeURL(URL url) throws MalformedURLException, URISyntaxException @Override public void rip() throws IOException, URISyntaxException { - int index = 0; + int imageIndex = 0; logger.info("Retrieving " + this.url); sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm()); JSONObject json = getFirstPage(); @@ -88,9 +88,9 @@ public void rip() throws IOException, URISyntaxException { break; } - index += 1; - logger.debug("Found image url #" + index+ ": " + imageURL); - downloadURL(new URI(imageURL).toURL(), index); + imageIndex += 1; + logger.debug("Found image url #" + imageIndex+ ": " + imageURL); + downloadURL(new URI(imageURL).toURL(), imageIndex); } if (isStopped() || isThisATest()) { @@ -106,12 +106,17 @@ public void rip() throws IOException, URISyntaxException { } } - // If they're using a thread pool, wait for it. - if (getThreadPool() != null) { - logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName()); - getThreadPool().waitForThreads(); + logger.info("All items queued; total items: {}; url: {}", imageIndex, url); + + // Final total item count is now known + setItemsTotal(imageIndex); + + if (getCrawlerThreadPool() != null) { + logger.debug("Waiting for crawler threadpool: {}", url); + getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url); } - waitForThreads(); + + waitForRipperThreads(); } protected String getPrefix(int index) { diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 211792939..823c97eb4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -18,6 +18,7 @@ import java.nio.file.StandardOpenOption; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,6 +47,23 @@ public abstract class AbstractRipper protected final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); protected final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); protected final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); + protected final Map itemsSkipped = Collections.synchronizedMap(new HashMap<>()); + + /** + * Rippers should set itemsTotal to the best known number of total items, + * if known at the start of the rip, e.g. in getFirstPage(). + * The best known number might be indicated on the album page, + * or calculated by the number of pages and the number of items per page. + * Once the last item is seen by the HTML or JSON crawler, the final value is set. + */ + private final AtomicInteger itemsTotal = new AtomicInteger(0); + + /** + * If an album has a duplicate RipUrlId (e.g. the same image linked twice), + * duplicates can't be counted by itemsPending, but {@link #waitForRipperThreads()} needs + * to know that the ripper has seen each link crawled. + */ + private final AtomicInteger itemsSeen = new AtomicInteger(0); private final String URLHistoryFile = Utils.getURLHistoryFile(); @@ -55,7 +73,8 @@ public abstract class AbstractRipper protected URL url; protected File workingDir; - DownloadThreadPool threadPool; + private DownloadThreadPool ripperThreadPool; + private DownloadThreadPool crawlerThreadPool; RipStatusHandler observer = null; private final AtomicBoolean completed = new AtomicBoolean(false); @@ -74,8 +93,8 @@ public boolean hasASAPRipping() { // Everytime addUrlToDownload skips a already downloaded url this increases by 1 public int alreadyDownloadedUrls = 0; - private final AtomicBoolean shouldStop = new AtomicBoolean(false); - private final AtomicBoolean shouldPanic = new AtomicBoolean(false); + protected final AtomicBoolean shouldStop = new AtomicBoolean(false); + protected final AtomicBoolean shouldPanic = new AtomicBoolean(false); private static boolean thisIsATest = false; public void stop() { @@ -97,18 +116,27 @@ public boolean isStopped() { return shouldStop.get(); } - public boolean isCompleted() { - return completed.get(); - } - protected void stopCheck() throws IOException { if (shouldStop.get()) { throw new IOException("Ripping interrupted"); } } - protected DownloadThreadPool getThreadPool() { - return threadPool; + /** + * Used for file downloads. Used by {@link #addURLToDownload(TokenedUrlGetter, RipUrlId, Path, String, String, Map, Boolean)} + */ + protected DownloadThreadPool getRipperThreadPool() { + return ripperThreadPool; + } + + /** + * Used by Rippers to crawl file pages.
+ * After the last file page is crawled and all threads are queued to the ripper thread pool, + * {@link #rip()} terminates the crawler thread pool.
+ * After the crawler thread pool is finished, {@link #rip()} terminates the ripper thread pool. + */ + protected DownloadThreadPool getCrawlerThreadPool() { + return crawlerThreadPool; } /** @@ -238,7 +266,8 @@ public void setup() throws IOException, URISyntaxException { // ctx.reconfigure(); // ctx.updateLoggers(); - this.threadPool = new DownloadThreadPool(getClass().getSimpleName() + "_" + getGID(url)); + this.ripperThreadPool = new DownloadThreadPool(getClass().getSimpleName() + "-ripper-" + getGID(url)); + this.crawlerThreadPool = new DownloadThreadPool(getClass().getSimpleName() + "-crawler"); } public void setObserver(RipStatusHandler obs) { @@ -289,6 +318,8 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di itemsPending.clear(); return false; } + itemsSeen.incrementAndGet(); + if (!allowDuplicates() && ( itemsPending.contains(ripUrlId) || itemsCompleted.containsKey(ripUrlId) @@ -333,7 +364,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di if (cookies != null) { dft.setCookies(cookies); } - threadPool.addThread(dft); + getRipperThreadPool().addThread(dft); } return true; @@ -574,11 +605,24 @@ public static String getFileName(URL url, String prefix, String fileName, String /** * Waits for downloading threads to complete. */ - protected void waitForThreads() { - logger.debug("Waiting for threads to finish"); - completed.set(false); - threadPool.waitForThreads(); - checkIfComplete(); + protected void waitForRipperThreads() { + waitForRipperThreads(true); + } + + protected void waitForRipperThreads(boolean notifyComplete) { + logger.debug("Waiting for threads to finish; url: {}", url); + if (!notifyComplete) { + setItemsTotal(0); + } + ripperThreadPool.waitForThreads(() -> { + boolean finished = shouldStop.get() || (itemsSeen.get() >= itemsTotal.get() && itemsPending.isEmpty()); + logger.trace("ripperThreadPool: are threads finished? {} url: {} shouldStop: {}; itemsPending.size(): {}; itemsCompleted.size(): {}; itemsErrored.size(): {}; itemsSkipped.size(): {}; itemsTotal: {}; itemsSeen: {}", + finished, url, shouldStop, itemsPending.size(), itemsCompleted.size(), itemsErrored.size(), itemsSkipped.size(), itemsTotal, itemsSeen); + return finished; + }, url); + if (notifyComplete) { + notifyComplete(); + } } /** @@ -610,7 +654,7 @@ protected void downloadCompleted(RipUrlId ripUrlId, Path saveAs) { itemsCompleted.put(ripUrlId, saveAs); observer.update(this, msg); - checkIfComplete(); + //checkIfComplete(); } catch (Exception e) { logger.error("Exception while updating observer: ", e); } @@ -627,7 +671,22 @@ protected void downloadErrored(RipUrlId ripUrlId, String reason) { itemsErrored.put(ripUrlId, reason); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, ripUrlId + " : " + reason)); - checkIfComplete(); + //checkIfComplete(); + } + + /** + * Notifies observers that a file could not be downloaded (includes a reason). + */ + protected void downloadSkipped(RipUrlId ripUrlId, String reason) { + if (observer == null) { + return; + } + itemsPending.remove(ripUrlId); + //itemsSkipped.put(ripUrlId, reason); + itemsCompleted.put(ripUrlId, null); // TODO use itemsSkipped and make the progress bar display it as completed + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_SKIP, ripUrlId + " : " + reason)); + + //checkIfComplete(); } /** @@ -643,7 +702,7 @@ protected void downloadExists(RipUrlId ripUrlId, Path file) { itemsCompleted.put(ripUrlId, file); observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); - checkIfComplete(); + //checkIfComplete(); } /** @@ -653,15 +712,6 @@ public int getCount() { return itemsCompleted.size() + itemsErrored.size(); } - /** - * Checks if complete and notifies observers if complete - */ - protected void checkIfComplete() { - if (itemsPending.isEmpty() && threadPool.getActiveThreadCount() == 0 && threadPool.getPendingThreadCount() == 0) { - notifyComplete(); - } - } - /** * Notifies observers and updates state if all files have been ripped. */ @@ -676,6 +726,7 @@ protected void notifyComplete() { RipStatusComplete rsc = new RipStatusComplete(workingDir.toPath(), getCount()); RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, rsc); + logger.debug("Sending RIP_COMPLETE: url: {}", getURL()); observer.update(this, msg); // we do not care if the rollingfileappender is active, @@ -809,11 +860,11 @@ public void run() { rip(); } catch (HttpStatusException e) { logger.error("Got exception while running ripper:", e); - waitForThreads(); + waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, "HTTP status code " + e.getStatusCode() + " for URL " + e.getUrl()); } catch (Exception e) { logger.error("Got exception while running ripper:", e); - waitForThreads(); + waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, e.getMessage()); } finally { cleanup(); @@ -942,4 +993,22 @@ protected static boolean shouldIgnoreExtension(URL url) { } return false; } + + /** + * Gets the asserted number of total items, or 0 if unknown. + * Possibly useful in rippers. + */ + protected int getItemsTotal() { + return itemsTotal.get(); + } + + /** + * For use in rippers to update the best estimate of total items. + */ + protected void setItemsTotal(int itemsTotal) { + if (itemsTotal < 0) { + throw new IllegalArgumentException("itemsTotal cannot be negative. Use 0 for unknown."); + } + this.itemsTotal.set(itemsTotal); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 353996237..2de0f1e80 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -1,8 +1,12 @@ package com.rarchives.ripme.ripper; +import java.net.URL; import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import com.rarchives.ripme.utils.Utils; import org.apache.logging.log4j.LogManager; @@ -15,36 +19,75 @@ public class DownloadThreadPool { private static final Logger logger = LogManager.getLogger(DownloadThreadPool.class); private ThreadPoolExecutor threadPool = null; + private final AtomicLong scheduledThreadCount = new AtomicLong(0); + private final String name; public DownloadThreadPool(String threadPoolName) { - initialize(threadPoolName); - } - - /** - * Initializes the threadpool. - * @param threadPoolName Name of the threadpool. - */ - private void initialize(String threadPoolName) { int threads = Utils.getConfigInteger("threads.size", 10); logger.debug("Initializing " + threadPoolName + " thread pool with " + threads + " threads"); - threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); + this.name = threadPoolName; + this.threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); } + /** * For adding threads to execution pool. * @param t * Thread to be added. */ public void addThread(Runnable t) { + logger.trace("addThread called; name: {}, scheduledThreadCount: {}", name, scheduledThreadCount); + scheduledThreadCount.incrementAndGet(); threadPool.execute(t); } /** * Tries to shutdown threadpool. */ - public void waitForThreads() { + public void waitForThreads(Supplier isFinishedQueueing, URL url) { + logger.trace("waitForThreads called; name: {}; url: {}", name, url); + while (!isFinishedQueueing.get()) { + logger.trace("waiting for items to finish queueing; name: {}; url: {}", name, url); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.trace("sleep interrupted; name: {}; url: {}", name, url); + break; + } + } + + logger.trace("about to shutdown thread pool. name: {}; url: {}", name, url); threadPool.shutdown(); + logger.trace("thread pool shutdown. name: {}; url: {}", name, url); try { threadPool.awaitTermination(3600, TimeUnit.SECONDS); + logger.trace("thread pool terminated. name: {}; url: {}", name, url); + } catch (InterruptedException e) { + logger.error("[!] Interrupted while waiting for threads to finish: ", e); + } + } + + /** + * Tries to shutdown threadpool. + */ + public void waitForThreads(int expectedScheduledThreads, AtomicBoolean shouldStop, URL url) { + logger.trace("waitForThreads called; name: {}; url: {}", name, url); + while (getScheduledThreadCount() < expectedScheduledThreads && !shouldStop.get()) { + logger.trace("waiting for scheduled threads to equal expected scheduled threads; name: {}; scheduled: {}; expected: {} url: {}", name, scheduledThreadCount, expectedScheduledThreads, url); + try { + Thread.sleep(200); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + + logger.trace("about to shutdown thread pool. name: {}; url: {}", name, url); + threadPool.shutdown(); + logger.trace("thread pool shutdown. name: {}; url: {}", name, url); + try { + threadPool.awaitTermination(3600, TimeUnit.SECONDS); + logger.trace("thread pool terminated. name: {}; url: {}", name, url); } catch (InterruptedException e) { logger.error("[!] Interrupted while waiting for threads to finish: ", e); } @@ -57,4 +100,13 @@ public int getPendingThreadCount() { public int getActiveThreadCount() { return threadPool.getActiveCount(); } + + public long getCompletedThreadCount() { + return threadPool.getCompletedTaskCount(); + } + + public long getScheduledThreadCount() { + //return threadPool.getTaskCount(); // approximate, bad + return scheduledThreadCount.get(); + } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index e2db757f1..4bb197e77 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -30,7 +30,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -298,7 +297,7 @@ protected void downloadURL(URL url, int index) { // Start Thread and add to pool. DeviantartImageThread t = new DeviantartImageThread(url); - getThreadPool().addThread(t); + getCrawlerThreadPool().addThread(t); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java index d18cb9ec8..d0fbbbb9e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java @@ -19,7 +19,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; @@ -129,7 +128,7 @@ public void downloadURL(final URL url, int index) { // rate limit sleep(3000); // addURLToDownload(url, getPrefix(index)); - getThreadPool().addThread(new E621FileThread(url, getPrefix(index))); + getCrawlerThreadPool().addThread(new E621FileThread(url, getPrefix(index))); } private String getTerm(URL url) throws MalformedURLException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java index a85fdd6ff..d58bcc4c9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/EHentaiRipper.java @@ -21,7 +21,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; @@ -187,7 +186,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { EHentaiImageThread t = new EHentaiImageThread(url, index, this.workingDir.toPath()); - getThreadPool().addThread(t); + getCrawlerThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java index b36d48923..54a98f306 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/HqpornerRipper.java @@ -18,7 +18,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class HqpornerRipper extends AbstractHTMLRipper { @@ -110,7 +109,7 @@ public boolean tryResumeDownload() { @Override public void downloadURL(URL url, int index) { - getThreadPool().addThread(new HqpornerDownloadThread(url, index, subdirectory)); + getCrawlerThreadPool().addThread(new HqpornerDownloadThread(url, index, subdirectory)); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java index 52d6b4e1a..104106301 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagebamRipper.java @@ -1,7 +1,6 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; import java.io.IOException; @@ -83,7 +82,7 @@ public List getURLsFromPage(Document doc) { @Override public void downloadURL(URL url, int index) { ImagebamImageThread t = new ImagebamImageThread(url, index); - getThreadPool().addThread(t); + getCrawlerThreadPool().addThread(t); sleep(500); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java index 2be15a05f..c63bd36e9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImagevenueRipper.java @@ -17,7 +17,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -65,7 +64,7 @@ public List getURLsFromPage(Document doc) { public void downloadURL(URL url, int index) { ImagevenueImageThread t = new ImagevenueImageThread(url, index); - getThreadPool().addThread(t); + getCrawlerThreadPool().addThread(t); } /** diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java index 5bb7b0020..afbab931e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ImgurRipper.java @@ -205,7 +205,7 @@ public void rip() throws IOException { } catch (URISyntaxException e) { throw new IOException("Failed ripping " + this.url, e); } - waitForThreads(); + waitForRipperThreads(); } private void ripSingleImage(URL url) throws IOException, URISyntaxException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java index 9124a1495..2cf37ef6e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ListalRipper.java @@ -19,7 +19,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; /** @@ -75,7 +74,7 @@ public List getURLsFromPage(Document page) { @Override public void downloadURL(URL url, int index) { - getThreadPool().addThread(new ListalImageDownloadThread(url, index)); + getCrawlerThreadPool().addThread(new ListalImageDownloadThread(url, index)); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java index ce46e9f3b..dc1fa3f67 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/MotherlessRipper.java @@ -114,7 +114,7 @@ protected List getURLsFromPage(Document page) { protected void downloadURL(URL url, int index) { // Create thread for finding image at "url" page MotherlessImageRunnable mit = new MotherlessImageRunnable(url, index); - getThreadPool().addThread(mit); + getCrawlerThreadPool().addThread(mit); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java index 41540afee..e69dbed81 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/NfsfwRipper.java @@ -17,7 +17,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; public class NfsfwRipper extends AbstractHTMLRipper { @@ -101,7 +100,7 @@ protected void downloadURL(URL url, int index) { index = ++this.index; } NfsfwImageThread t = new NfsfwImageThread(url, currentDir, index); - getThreadPool().addThread(t); + getCrawlerThreadPool().addThread(t); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java index 7db21cb15..c6279c3e0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/PornhubRipper.java @@ -18,7 +18,6 @@ import org.jsoup.select.Elements; import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; @@ -79,7 +78,7 @@ protected List getURLsFromPage(Document page) { @Override protected void downloadURL(URL url, int index) { PornhubImageThread t = new PornhubImageThread(url, index, this.workingDir.toPath()); - getThreadPool().addThread(t); + getCrawlerThreadPool().addThread(t); try { Thread.sleep(IMAGE_SLEEP_TIME); } catch (InterruptedException e) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 2419cd035..cb6b8eeb7 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -114,7 +114,7 @@ public void rip() throws IOException { } catch (URISyntaxException e) { new IOException(e.getMessage()); } - waitForThreads(); + waitForRipperThreads(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 6bbda8757..7e92b103b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -236,7 +236,7 @@ public void rip() throws IOException { } } - waitForThreads(); + waitForRipperThreads(); } private boolean handleJSON(JSONObject json) { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java index b6b39bc61..5b71081d5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java @@ -158,7 +158,7 @@ public void rip() throws IOException, URISyntaxException { for (int index = 0; index < URLs.size(); index ++) { downloadURL(new URI(URLs.get(index)).toURL(), index); } - waitForThreads(); + waitForRipperThreads(); } else { RIP_TYPE = RipType.IMAGE; diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java index 1e7a48f8e..76527ceea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/CliphunterRipper.java @@ -78,6 +78,6 @@ public void rip() throws IOException, URISyntaxException { } } addURLToDownload(new URI(vidURL).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java index 0f95aaafc..c6cf3f123 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/MotherlessVideoRipper.java @@ -70,6 +70,6 @@ public void rip() throws IOException, URISyntaxException { } String vidUrl = vidUrls.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java index aa8b90541..6f10ec429 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/PornhubRipper.java @@ -154,6 +154,6 @@ public void rip() throws IOException, URISyntaxException { } addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + bestQuality + "p_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java index 076e90ca6..f6249e9d0 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/TwitchVideoRipper.java @@ -75,6 +75,6 @@ public void rip() throws IOException, URISyntaxException { addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + title); } } - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java index a2cff267d..31c127d39 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/ViddmeRipper.java @@ -68,6 +68,6 @@ public void rip() throws IOException, URISyntaxException { String vidUrl = videos.first().attr("content"); vidUrl = vidUrl.replaceAll("&", "&"); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java index 00e77c427..4829cff86 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VidearnRipper.java @@ -63,6 +63,6 @@ public void rip() throws IOException, URISyntaxException { } String vidUrl = mp4s.get(0); addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java index 4a7ea8ccd..1ab5c1ef4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java @@ -61,7 +61,7 @@ public void rip() throws IOException, URISyntaxException { logger.info(" Retrieving " + this.url); String videoURL = getVideoURLAtPage(this.url.toExternalForm()); addURLToDownload(new URI(videoURL).toURL(), HOST + "_" + getGID(this.url)); - waitForThreads(); + waitForRipperThreads(); } public static String getVideoURLAtPage(String url) throws IOException { diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java index 2fe0291e6..61762f3bb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/video/YuvutuRipper.java @@ -77,6 +77,6 @@ public void rip() throws IOException, URISyntaxException { addURLToDownload(new URI(vidUrl).toURL(), HOST + "_" + getGID(this.url)); } } - waitForThreads(); + waitForRipperThreads(); } } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 55e8ef081..7d7084d08 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -136,6 +136,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. + private static final AtomicBoolean isRipperActive = new AtomicBoolean(false); public static final int TRANSFER_RATE_REFRESH_RATE = 200; private static final TransferRate transferRate = new TransferRate(); @@ -143,7 +144,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1); private Future rateRefresherFuture = null; private final Runnable rateRefresher = () -> { - if (isHalted()) { + if (!isRipperActive.get()) { if (rateRefresherFuture != null) { rateRefresherFuture.cancel(true); rateRefresherFuture = null; @@ -154,13 +155,6 @@ public final class MainWindow implements Runnable, RipStatusHandler { transferRateLabel.setText(transferRate.formatHumanTransferRate()); }; - /** - * @return true if fully halted/panic button pressed - */ - public static boolean isHalted() { - return ripper == null || ripper.isPanicked() || ripper.isCompleted(); - } - private void updateQueue(DefaultListModel model) { if (model == null) model = queueListModel; @@ -1392,6 +1386,13 @@ private void saveHistory() { } private void ripNextAlbum() { + LOGGER.debug("ripNextAlbum called"); + if (isRipperActive.getAndSet(true)) { + // Already ripping + LOGGER.debug("already ripping"); + return; + } + // Save current state of queue to configuration. Utils.setConfigList("queue", queueListModel.elements()); @@ -1399,6 +1400,7 @@ private void ripNextAlbum() { boolean wasPanicStop = gracefulStop.getAndSet(false); if (wasGracefulStop || wasPanicStop) { // Stop requested + LOGGER.debug("wasGracefulStop or wasPanicStop"); ripFinishCleanup(); return; } @@ -1417,16 +1419,20 @@ private void ripNextAlbum() { updateQueue(); + LOGGER.debug("calling ripAlbum(\"{}\")", nextAlbum); Thread t = ripAlbum(nextAlbum); if (t == null) { + LOGGER.debug("ripAlbum() returned null"); try { Thread.sleep(500); } catch (InterruptedException ie) { LOGGER.error(Utils.getLocalizedString("interrupted.while.waiting.to.rip.next.album"), ie); } + isRipperActive.set(false); ripNextAlbum(); } else { + LOGGER.debug("Starting new ripper thread"); t.start(); } } @@ -1434,6 +1440,7 @@ private void ripNextAlbum() { private void ripFinishCleanup() { stopButton.setEnabled(false); panicButton.setEnabled(false); + isRipperActive.set(false); statusProgress.setValue(0); statusProgress.setVisible(false); } @@ -1466,6 +1473,7 @@ private Thread ripAlbum(String urlString) { pack(); boolean failed = false; try { + LOGGER.debug("Creating ripper for url {}", url); ripper = AbstractRipper.getRipper(url); ripper.setup(); } catch (Exception e) { @@ -1641,10 +1649,11 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - ripFinishCleanup(); openButton.setVisible(false); + ripFinishCleanup(); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); + ripNextAlbum(); break; case RIP_COMPLETE: @@ -1673,7 +1682,6 @@ private synchronized void handleEvent(StatusEvent evt) { } saveHistory(); Utils.saveConfig(); - ripFinishCleanup(); openButton.setVisible(true); Path f = rsc.dir; String prettyFile = Utils.shortenPath(f); @@ -1730,6 +1738,7 @@ private synchronized void handleEvent(StatusEvent evt) { LOGGER.error(e); } }); + ripFinishCleanup(); pack(); ripNextAlbum(); break; @@ -1743,8 +1752,8 @@ private synchronized void handleEvent(StatusEvent evt) { if (LOGGER.isEnabled(Level.ERROR)) { appendLog((String) msg.getObject(), Color.RED); } - ripFinishCleanup(); openButton.setVisible(false); + ripFinishCleanup(); pack(); statusWithColor("Error: " + msg.getObject(), Color.RED); break; From 174a70c4a1a7b49f1163a9c850e392a81b480b09 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:59:01 -0400 Subject: [PATCH 25/78] Reconfigure logger from new Configuration Log4j2 docs recommend against modifying Appenders, and recommend building new Configuration objects instead --- .../com/rarchives/ripme/ui/MainWindow.java | 24 +++----- .../java/com/rarchives/ripme/utils/Utils.java | 59 ++++++++++++------- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 7d7084d08..95b5ada4d 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1136,24 +1136,14 @@ public void intervalRemoved(ListDataEvent arg0) { } private void setLogLevel(String level) { - // default level is error, set in case something else is given. - Level newLevel = Level.ERROR; level = level.substring(level.lastIndexOf(' ') + 1); - switch (level) { - case "Debug": - newLevel = Level.DEBUG; - break; - case "Info": - newLevel = Level.INFO; - break; - case "Warn": - newLevel = Level.WARN; - } - LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); - loggerConfig.setLevel(newLevel); - ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. + Level newLevel = switch (level) { + case "Debug" -> Level.DEBUG; + case "Info" -> Level.INFO; + case "Warn" -> Level.WARN; + default -> Level.ERROR; + }; + Utils.configureLogger(newLevel); } private void setupTrayIcon() { diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 20218bcc6..f7272914d 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -38,9 +38,11 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.appender.RollingFileAppender; import org.apache.logging.log4j.core.appender.rolling.DefaultRolloverStrategy; import org.apache.logging.log4j.core.appender.rolling.SizeBasedTriggeringPolicy; @@ -49,6 +51,10 @@ import org.apache.logging.log4j.core.config.LoggerConfig; import com.rarchives.ripme.ripper.AbstractRipper; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilder; +import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory; +import org.apache.logging.log4j.core.config.builder.api.RootLoggerComponentBuilder; +import org.apache.logging.log4j.core.config.builder.impl.BuiltConfiguration; /** * Common utility functions used in various places throughout the project. @@ -592,36 +598,45 @@ public static void playSound(String filename) { } } + public static void configureLogger() { + configureLogger(Level.INFO); // default INFO level + } + /** * Configures root logger, either for FILE output or just console. */ - public static void configureLogger() { - LoggerContext ctx = (LoggerContext) LogManager.getContext(false); - Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); + public static void configureLogger(Level level) { + ConfigurationBuilder builder = ConfigurationBuilderFactory.newConfigurationBuilder(); + + //builder.setStatusLevel(Level.DEBUG); + final String consoleAppenderName = "stdout"; + builder.add(builder.newAppender(consoleAppenderName, "CONSOLE") + .addAttribute("target", ConsoleAppender.Target.SYSTEM_OUT) + .add(builder.newLayout("PatternLayout").addAttribute("pattern", "%-5level %c{1}: %msg%n%xEx")) + ); + + RootLoggerComponentBuilder rootLogger = builder.newRootLogger(level); + rootLogger.add(builder.newAppenderRef(consoleAppenderName)); // write to ripme.log file if checked in GUI boolean logSave = getConfigBoolean("log.save", false); if (logSave) { - LOGGER.debug("add rolling appender ripmelog"); - TriggeringPolicy tp = SizeBasedTriggeringPolicy.createPolicy("20M"); - DefaultRolloverStrategy rs = DefaultRolloverStrategy.newBuilder().withMax("2").build(); - RollingFileAppender rolling = RollingFileAppender.newBuilder() - .setName("ripmelog") - .withFileName("ripme.log") - .withFilePattern("%d{yyyy-MM-dd HH:mm:ss} %p %m%n") - .withPolicy(tp) - .withStrategy(rs) - .build(); - loggerConfig.addAppender(rolling, null, null); - } else { - LOGGER.debug("remove rolling appender ripmelog"); - if (config.getAppender("ripmelog") != null) { - config.getAppender("ripmelog").stop(); - } - loggerConfig.removeAppender("ripmelog"); + final String fileAppenderName = "rolling"; + builder.add(builder.newAppender(fileAppenderName, "RollingFile") + .addAttribute("fileName", "ripme.log") + .addAttribute("filePattern", "ripme-%d{yyyy-MM-dd}-%i.log.gz") + .add(builder.newLayout("PatternLayout").addAttribute("pattern", "%d %-5level %c{1}: %msg%n%xEx")) + .addComponent(builder.newComponent("Policies") + .addComponent(builder.newComponent("SizeBasedTriggeringPolicy").addAttribute("size", "20M"))) + ); + rootLogger.add(builder.newAppenderRef(fileAppenderName)); } - ctx.updateLoggers(); // This causes all Loggers to refetch information from their LoggerConfig. + + builder.add(rootLogger); + + Configuration configuration = builder.build(); + LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + ctx.reconfigure(configuration); } /** From 17a98f924c484f075d269b52d01147777a60a9a8 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 03:03:42 -0400 Subject: [PATCH 26/78] Set antialiasing hint --- src/main/java/com/rarchives/ripme/App.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index bd373a4e0..0d120c1c1 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -76,6 +76,9 @@ public static void main(String[] args) throws IOException { if (GraphicsEnvironment.isHeadless() || args.length > 0) { handleArguments(args); } else { + // Antialiasing hint, especially for Linux + System.setProperty("awt.useSystemAAFontSettings", "on"); + if (SystemUtils.IS_OS_MAC_OSX) { System.setProperty("apple.laf.useScreenMenuBar", "true"); System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe"); From 23c17783ec9a376bbbc5e5013994d94fc152da4d Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 03:06:28 -0400 Subject: [PATCH 27/78] Reduce duplicate code --- .../com/rarchives/ripme/ui/MainWindow.java | 49 +------------------ 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 95b5ada4d..a32b22966 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -721,8 +721,8 @@ private void checkAndUpdate() { return field; } - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, - JButton thing2ToAdd) { + private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComponent thing1ToAdd, + JComponent thing2ToAdd) { gbc.gridy = gbcYValue; gbc.gridx = 0; configurationPanel.add(thing1ToAdd, gbc); @@ -730,51 +730,6 @@ private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYV configurationPanel.add(thing2ToAdd, gbc); } - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JLabel thing1ToAdd, - JTextField thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JCheckBox thing1ToAdd, - JCheckBox thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings("rawtypes") - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JCheckBox thing1ToAdd, - JComboBox thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings("rawtypes") - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComboBox thing1ToAdd, - JButton thing2ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - gbc.gridx = 1; - configurationPanel.add(thing2ToAdd, gbc); - } - - @SuppressWarnings({ "unused", "rawtypes" }) - private void addItemToConfigGridBagConstraints(GridBagConstraints gbc, int gbcYValue, JComboBox thing1ToAdd) { - gbc.gridy = gbcYValue; - gbc.gridx = 0; - configurationPanel.add(thing1ToAdd, gbc); - } - private void changeLocale() { statusLabel.setText(Utils.getLocalizedString("inactive")); configUpdateButton.setText(Utils.getLocalizedString("check.for.updates")); From b3de65183dc898acf7f8523c904e1485211fba3d Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 03:44:17 -0400 Subject: [PATCH 28/78] Log error with logger --- .../ripme/uiUtils/ContextActionProtections.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java index 9237fea90..57ea0aa1b 100644 --- a/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java +++ b/src/main/java/com/rarchives/ripme/uiUtils/ContextActionProtections.java @@ -1,6 +1,8 @@ package com.rarchives.ripme.uiUtils; -import javax.swing.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import javax.swing.text.JTextComponent; import java.awt.*; import java.awt.datatransfer.Clipboard; @@ -10,6 +12,8 @@ import java.io.IOException; public class ContextActionProtections { + private static final Logger logger = LogManager.getLogger(ContextActionProtections.class); + public static void pasteFromClipboard(JTextComponent textComponent) { Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard(); Transferable transferable = clipboard.getContents(new Object()); @@ -24,8 +28,8 @@ public static void pasteFromClipboard(JTextComponent textComponent) { // } // Set the text in the JTextField textComponent.setText(clipboardContent); - } catch (UnsupportedFlavorException | IOException unable_to_modify_text_on_paste) { - unable_to_modify_text_on_paste.printStackTrace(); + } catch (UnsupportedFlavorException | IOException e) { + logger.error("Unable to paste from clipboard", e); } } } From d2ae329a35d60a8a9da00696239c81b9cfde5df6 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 03:50:36 -0400 Subject: [PATCH 29/78] Reduce extra borders --- .../java/com/rarchives/ripme/ui/MainWindow.java | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index a32b22966..6cbb61bc8 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -195,7 +195,7 @@ public MainWindow() throws IOException { mainFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); mainFrame.setLayout(new GridBagLayout()); - createUI(mainFrame.getContentPane()); + createUI((JPanel) mainFrame.getContentPane()); pack(); loadHistory(); @@ -281,7 +281,7 @@ private boolean isCollapsed() { && !configurationPanel.isVisible()); } - private void createUI(Container pane) { + private void createUI(JPanel pane) { // If creating the tray icon fails, ignore it. try { setupTrayIcon(); @@ -289,7 +289,7 @@ private void createUI(Container pane) { LOGGER.warn(e.getMessage()); } - EmptyBorder emptyBorder = new EmptyBorder(5, 5, 5, 5); + pane.setBorder(new EmptyBorder(5, 5, 5, 5)); GridBagConstraints gbc = new GridBagConstraints(); gbc.fill = GridBagConstraints.HORIZONTAL; gbc.weightx = 1; @@ -363,7 +363,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib } catch (Exception ignored) { } JPanel ripPanel = new JPanel(new GridBagLayout()); - ripPanel.setBorder(emptyBorder); gbc.fill = GridBagConstraints.BOTH; gbc.weightx = 0; @@ -390,7 +389,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib openButton = new JButton(); openButton.setVisible(false); JPanel statusPanel = new JPanel(new GridBagLayout()); - statusPanel.setBorder(emptyBorder); gbc.gridx = 0; gbc.weightx = 1; @@ -407,12 +405,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridwidth = 1; JPanel progressPanel = new JPanel(new GridBagLayout()); - progressPanel.setBorder(emptyBorder); statusProgress = new JProgressBar(0, 100); progressPanel.add(statusProgress, gbc); JPanel optionsPanel = new JPanel(new GridBagLayout()); - optionsPanel.setBorder(emptyBorder); optionLog = new JButton(Utils.getLocalizedString("Log")); optionHistory = new JButton(Utils.getLocalizedString("History")); optionQueue = new JButton(Utils.getLocalizedString("queue")); @@ -444,7 +440,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib optionsPanel.add(optionConfiguration, gbc); logPanel = new JPanel(new GridBagLayout()); - logPanel.setBorder(emptyBorder); logText = new JTextPane(); logText.setEditable(false); JScrollPane logTextScroll = new JScrollPane(logText); @@ -458,7 +453,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.weighty = 0; historyPanel = new JPanel(new GridBagLayout()); - historyPanel.setBorder(emptyBorder); historyPanel.setVisible(false); historyPanel.setPreferredSize(new Dimension(300, 250)); @@ -540,7 +534,6 @@ public void setValueAt(Object value, int row, int col) { gbc.ipady = 0; JPanel historyButtonPanel = new JPanel(new GridBagLayout()); historyButtonPanel.setSize(new Dimension(300, 10)); - historyButtonPanel.setBorder(emptyBorder); gbc.gridx = 0; historyButtonPanel.add(historyButtonRemove, gbc); gbc.gridx = 1; @@ -554,7 +547,6 @@ public void setValueAt(Object value, int row, int col) { historyPanel.add(historyButtonPanel, gbc); queuePanel = new JPanel(new GridBagLayout()); - queuePanel.setBorder(emptyBorder); queuePanel.setVisible(false); queuePanel.setPreferredSize(new Dimension(300, 250)); queueListModel = new DefaultListModel<>(); @@ -581,7 +573,6 @@ public void setValueAt(Object value, int row, int col) { gbc.ipady = 0; configurationPanel = new JPanel(new GridBagLayout()); - configurationPanel.setBorder(emptyBorder); configurationPanel.setVisible(false); // TODO Configuration components From e5c12b4f39c8390582553ccc6fd23d2283c19ce1 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 03:51:13 -0400 Subject: [PATCH 30/78] Avoid layout shifting --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 6cbb61bc8..056c2421e 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -430,6 +430,13 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib } catch (Exception e) { LOGGER.warn(e.getMessage()); } + + // Prevent button sizes/positions from shifting when text bolds/unbolds + optionLog.setPreferredSize(optionLog.getPreferredSize()); + optionHistory.setPreferredSize(optionHistory.getPreferredSize()); + optionQueue.setPreferredSize(optionQueue.getPreferredSize()); + optionConfiguration.setPreferredSize(optionConfiguration.getPreferredSize()); + gbc.gridx = 0; optionsPanel.add(optionLog, gbc); gbc.gridx = 1; From 3d76fd3658420a85a5a0863962a5ff7cd15698fa Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 04:01:43 -0400 Subject: [PATCH 31/78] Extract duplicate code --- .../ripme/ui/DeselectableButtonGroup.java | 14 +++ .../com/rarchives/ripme/ui/MainWindow.java | 108 ++++++------------ 2 files changed, 50 insertions(+), 72 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ui/DeselectableButtonGroup.java diff --git a/src/main/java/com/rarchives/ripme/ui/DeselectableButtonGroup.java b/src/main/java/com/rarchives/ripme/ui/DeselectableButtonGroup.java new file mode 100644 index 000000000..51ae63c0a --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ui/DeselectableButtonGroup.java @@ -0,0 +1,14 @@ +package com.rarchives.ripme.ui; + +import javax.swing.*; + +public class DeselectableButtonGroup extends ButtonGroup { + @Override + public void setSelected(ButtonModel model, boolean selected) { + if (selected) { + super.setSelected(model, selected); + } else { + clearSelection(); + } + } +} diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 056c2421e..2197e951b 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -53,6 +53,7 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final Logger LOGGER = LogManager.getLogger(MainWindow.class); + private static final String RIPME_PANEL = "ripme.panel"; private static JFrame mainFrame; @@ -69,15 +70,17 @@ public final class MainWindow implements Runnable, RipStatusHandler { // anchored to the top when there is no open lower panel private static JPanel emptyPanel; + private static final ButtonGroup panelButtonGroup = new DeselectableButtonGroup(); + // Log - private static JButton optionLog; + private static JToggleButton optionLog; private static JPanel logPanel; private static JTextPane logText; private static final Queue logLineLengths = new LinkedList<>(); private static final int MAX_LOG_PANE_LINES = 1000; // History - private static JButton optionHistory; + private static JToggleButton optionHistory; private static final History HISTORY = new History(); private static JPanel historyPanel; private static JTable historyTable; @@ -85,12 +88,12 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JButton historyButtonRemove, historyButtonClear, historyButtonRerip; // Queue - public static JButton optionQueue; + public static JToggleButton optionQueue; private static JPanel queuePanel; private static DefaultListModel queueListModel; // Configuration - private static JButton optionConfiguration; + private static JToggleButton optionConfiguration; private static JPanel configurationPanel; private static JButton configUpdateButton; private static JLabel configUpdateLabel; @@ -409,10 +412,16 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib progressPanel.add(statusProgress, gbc); JPanel optionsPanel = new JPanel(new GridBagLayout()); - optionLog = new JButton(Utils.getLocalizedString("Log")); - optionHistory = new JButton(Utils.getLocalizedString("History")); - optionQueue = new JButton(Utils.getLocalizedString("queue")); - optionConfiguration = new JButton(Utils.getLocalizedString("Configuration")); + optionLog = new JToggleButton(Utils.getLocalizedString("Log")); + optionHistory = new JToggleButton(Utils.getLocalizedString("History")); + optionQueue = new JToggleButton(Utils.getLocalizedString("queue")); + optionConfiguration = new JToggleButton(Utils.getLocalizedString("Configuration")); + + panelButtonGroup.add(optionLog); + panelButtonGroup.add(optionHistory); + panelButtonGroup.add(optionQueue); + panelButtonGroup.add(optionConfiguration); + optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); @@ -661,6 +670,11 @@ public void setValueAt(Object value, int row, int col) { emptyPanel.setPreferredSize(new Dimension(0, 0)); emptyPanel.setSize(0, 0); + optionLog.putClientProperty(RIPME_PANEL, logPanel); + optionHistory.putClientProperty(RIPME_PANEL, historyPanel); + optionQueue.putClientProperty(RIPME_PANEL, queuePanel); + optionConfiguration.putClientProperty(RIPME_PANEL, configurationPanel); + gbc.anchor = GridBagConstraints.PAGE_START; gbc.gridy = 0; pane.add(ripPanel, gbc); @@ -827,73 +841,23 @@ private void update() { } }); - optionLog.addActionListener(event -> { - logPanel.setVisible(!logPanel.isVisible()); - emptyPanel.setVisible(!logPanel.isVisible()); - historyPanel.setVisible(false); - queuePanel.setVisible(false); - configurationPanel.setVisible(false); - if (logPanel.isVisible()) { - optionLog.setFont(optionLog.getFont().deriveFont(Font.BOLD)); - } else { - optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); + ActionListener panelSelectListener = event -> { + JToggleButton source = (JToggleButton) event.getSource(); + Enumeration buttons = panelButtonGroup.getElements(); + while (buttons.hasMoreElements()) { + AbstractButton button = buttons.nextElement(); + JPanel tabPanel = (JPanel) button.getClientProperty(RIPME_PANEL); + boolean visible = button == source && source.isSelected(); + tabPanel.setVisible(visible); } - optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); + emptyPanel.setVisible(!source.isSelected()); pack(); - }); - - optionHistory.addActionListener(event -> { - logPanel.setVisible(false); - historyPanel.setVisible(!historyPanel.isVisible()); - emptyPanel.setVisible(!historyPanel.isVisible()); - queuePanel.setVisible(false); - configurationPanel.setVisible(false); - optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - if (historyPanel.isVisible()) { - optionHistory.setFont(optionLog.getFont().deriveFont(Font.BOLD)); - } else { - optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - } - optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - pack(); - }); - - optionQueue.addActionListener(event -> { - logPanel.setVisible(false); - historyPanel.setVisible(false); - queuePanel.setVisible(!queuePanel.isVisible()); - emptyPanel.setVisible(!queuePanel.isVisible()); - configurationPanel.setVisible(false); - optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - if (queuePanel.isVisible()) { - optionQueue.setFont(optionLog.getFont().deriveFont(Font.BOLD)); - } else { - optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - } - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - pack(); - }); + }; - optionConfiguration.addActionListener(event -> { - logPanel.setVisible(false); - historyPanel.setVisible(false); - queuePanel.setVisible(false); - configurationPanel.setVisible(!configurationPanel.isVisible()); - emptyPanel.setVisible(!configurationPanel.isVisible()); - optionLog.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionHistory.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - optionQueue.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - if (configurationPanel.isVisible()) { - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.BOLD)); - } else { - optionConfiguration.setFont(optionLog.getFont().deriveFont(Font.PLAIN)); - } - pack(); - }); + optionLog.addActionListener(panelSelectListener); + optionHistory.addActionListener(panelSelectListener); + optionQueue.addActionListener(panelSelectListener); + optionConfiguration.addActionListener(panelSelectListener); historyButtonRemove.addActionListener(event -> { int[] indices = historyTable.getSelectedRows(); From 67780b8a017b657ae3750af2f85f9f29946a45a9 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 17:52:36 -0400 Subject: [PATCH 32/78] Prettier status --- .../ripme/ripper/AbstractHTMLRipper.java | 17 +- .../ripme/ripper/AbstractJSONRipper.java | 19 +- .../ripme/ripper/AbstractRipper.java | 60 ++++- .../ripper/AbstractSingleFileRipper.java | 17 -- .../rarchives/ripme/ripper/AlbumRipper.java | 14 -- .../ripme/ripper/DownloadFileThread.java | 6 +- .../rarchives/ripme/ripper/VideoRipper.java | 29 +-- .../com/rarchives/ripme/ui/MainWindow.java | 230 +++++++++++++----- .../rarchives/ripme/ui/ProgressTextField.java | 55 +++++ .../java/com/rarchives/ripme/utils/Utils.java | 2 +- 10 files changed, 292 insertions(+), 157 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ui/ProgressTextField.java diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 2ba66554e..5e4152695 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -385,18 +385,13 @@ public int getCompletionPercentage() { return (int) (100 * ( (total - itemsPending.size()) / total)); } - /** - * @return - * Human-readable information on the status of the current rip. - */ @Override - public String getStatusText() { - return getCompletionPercentage() + - "% " + - "- Pending: " + itemsPending.size() + - ", Completed: " + itemsCompleted.size() + - ", Errored: " + itemsErrored.size(); + public int getPendingCount() { + DownloadThreadPool threadPool = getRipperThreadPool(); + if (threadPool != null) { + return threadPool.getPendingThreadCount(); + } + return itemsPending.size(); } - } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 5ed3c637c..7d71ba758 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -186,20 +186,13 @@ public int getCompletionPercentage() { return (int) (100 * ( (total - itemsPending.size()) / total)); } - /** - * @return - * Human-readable information on the status of the current rip. - */ @Override - public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); + public int getPendingCount() { + DownloadThreadPool threadPool = getRipperThreadPool(); + if (threadPool != null) { + return threadPool.getPendingThreadCount(); + } + return itemsPending.size(); } - } diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 823c97eb4..44fc0e1f2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -44,6 +44,7 @@ public abstract class AbstractRipper private static final Logger logger = LogManager.getLogger(AbstractRipper.class); + // For albums protected final Set itemsPending = Collections.synchronizedSet(new HashSet<>()); protected final Map itemsCompleted = Collections.synchronizedMap(new HashMap<>()); protected final Map itemsErrored = Collections.synchronizedMap(new HashMap<>()); @@ -65,6 +66,11 @@ public abstract class AbstractRipper */ private final AtomicInteger itemsSeen = new AtomicInteger(0); + /** For individual files. See {@link #useByteProgessBar()} */ + protected long bytesCompleted = 0; + /** For individual files. See {@link #useByteProgessBar()} */ + protected long bytesTotal = 1; // avoid divide by 0 + private final String URLHistoryFile = Utils.getURLHistoryFile(); public static final String USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"; @@ -847,11 +853,6 @@ public void sendUpdate(STATUS status, Object message) { */ public abstract int getCompletionPercentage(); - /** - * @return Text for status - */ - public abstract String getStatusText(); - /** * Rips the album when the thread is invoked. */ @@ -950,12 +951,12 @@ protected boolean sleep(int milliseconds) { } } - public void setBytesTotal(int bytes) { - // Do nothing + protected void setBytesTotal(long bytes) { + this.bytesTotal = bytes; } - public void setBytesCompleted(int bytes) { - // Do nothing + protected void setBytesCompleted(long bytes) { + this.bytesCompleted = bytes; } /** Methods for detecting when we're running a test. */ @@ -969,7 +970,7 @@ protected static boolean isThisATest() { } // If true ripme uses a byte progress bar - protected boolean useByteProgessBar() { + public boolean useByteProgessBar() { return false; } @@ -994,9 +995,48 @@ protected static boolean shouldIgnoreExtension(URL url) { return false; } + public int getPendingCount() { + DownloadThreadPool threadPool = getRipperThreadPool(); + if (threadPool != null) { + return threadPool.getPendingThreadCount(); + } + return itemsPending.size(); + } + + public int getCompletedCount() { + return itemsCompleted.size(); + } + + public int getErroredCount() { + return itemsErrored.size(); + } + + public int getActiveCount() { + return ripperThreadPool.getActiveThreadCount(); + } + + public long getBytesTotal() { + return bytesTotal; + } + + public long getBytesCompleted() { + return bytesCompleted; + } + + /** + * Gets the best estimate of total items. + */ + public int getTotalCount() { + if (itemsTotal.get() == 0) { + return itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); + } + return itemsTotal.get(); + } + /** * Gets the asserted number of total items, or 0 if unknown. * Possibly useful in rippers. + * MainWindow probably wants {@link #getTotalCount()} */ protected int getItemsTotal() { return itemsTotal.get(); diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java index f1f8be41b..273d86367 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractSingleFileRipper.java @@ -11,33 +11,16 @@ * to help cut down on copy pasted code */ public abstract class AbstractSingleFileRipper extends AbstractHTMLRipper { - private int bytesTotal = 1; - private int bytesCompleted = 1; protected AbstractSingleFileRipper(URL url) throws IOException { super(url); } - @Override - public String getStatusText() { - return Utils.getByteStatusText(getCompletionPercentage(), bytesCompleted, bytesTotal); - } - @Override public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - @Override - public void setBytesTotal(int bytes) { - this.bytesTotal = bytes; - } - - @Override - public void setBytesCompleted(int bytes) { - this.bytesCompleted = bytes; - } - @Override public boolean useByteProgessBar() {return true;} } diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index 1570e11b0..bad1961d9 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -89,18 +89,4 @@ public int getCompletionPercentage() { return (int) (100 * ( (total - itemsPending.size()) / total)); } - /** - * @return - * Human-readable information on the status of the current rip. - */ - @Override - public String getStatusText() { - StringBuilder sb = new StringBuilder(); - sb.append(getCompletionPercentage()) - .append("% ") - .append("- Pending: " ).append(itemsPending.size()) - .append(", Completed: ").append(itemsCompleted.size()) - .append(", Errored: " ).append(itemsErrored.size()); - return sb.toString(); - } } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index b134412c5..85bcc6097 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -112,8 +112,8 @@ public void run() { String prettySaveAs = Utils.removeCWD(saveAs.toPath()); long fileSize = 0; - int bytesTotal; - int bytesDownloaded = 0; + long bytesTotal = 0; + long bytesDownloaded = 0; if (saveAs.exists() && observer.tryResumeDownload()) { fileSize = saveAs.length(); } @@ -220,7 +220,7 @@ public void run() { // If the ripper is using the bytes progress bar set bytesTotal to // huc.getContentLength() if (observer.useByteProgessBar()) { - bytesTotal = huc.getContentLength(); + bytesTotal = huc.getContentLengthLong(); observer.setBytesTotal(bytesTotal); observer.sendUpdate(STATUS.TOTAL_BYTES, bytesTotal); logger.debug("Size of file at " + url + " = " + bytesTotal + "b"); diff --git a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java index d2904feb2..bc57b9c4c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/VideoRipper.java @@ -18,9 +18,6 @@ public abstract class VideoRipper extends AbstractRipper { private static final Logger logger = LogManager.getLogger(VideoRipper.class); - private int bytesTotal = 1; - private int bytesCompleted = 1; - protected VideoRipper(URL url) throws IOException { super(url); } @@ -31,23 +28,13 @@ protected VideoRipper(URL url) throws IOException { public abstract String getGID(URL url) throws MalformedURLException; - @Override - public void setBytesTotal(int bytes) { - this.bytesTotal = bytes; - } - - @Override - public void setBytesCompleted(int bytes) { - this.bytesCompleted = bytes; - } - @Override public String getAlbumTitle(URL url) { return "videos"; } @Override - protected boolean useByteProgessBar() { + public boolean useByteProgessBar() { return true; } @@ -90,20 +77,6 @@ public int getCompletionPercentage() { return (int) (100 * (bytesCompleted / (float) bytesTotal)); } - /** - * Gets the status and changes it to a human-readable form. - * - * @return Status of current download. - */ - @Override - public String getStatusText() { - return String.valueOf(getCompletionPercentage()) + - "% - " + - Utils.bytesToHumanReadable(bytesCompleted) + - " / " + - Utils.bytesToHumanReadable(bytesTotal); - } - /** * Sanitizes URL. * Usually just returns itself. diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 2197e951b..d469821b7 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -37,9 +37,6 @@ import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.core.LoggerContext; -import org.apache.logging.log4j.core.config.Configuration; -import org.apache.logging.log4j.core.config.LoggerConfig; import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.uiUtils.ContextActionProtections; @@ -62,9 +59,14 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JButton panicButton; private static JLabel statusLabel; - private static final JLabel transferRateLabel = new JLabel(); - private static JButton openButton; - private static JProgressBar statusProgress; + private static final ProgressTextField currentlyRippingProgress = new ProgressTextField(); + private static final JLabel pendingValue = new JLabel("0"); + private static final JLabel activeValue = new JLabel("0"); + private static final JLabel completedValue = new JLabel("0"); + private static final JLabel erroredValue = new JLabel("0"); + private static final JLabel totalValue = new JLabel("0"); + private static final JLabel transferRateValue = new JLabel("0.00 B/s"); + private static final JButton openButton = new JButton(); // Put an empty JPanel on the bottom of the window to keep components // anchored to the top when there is no open lower panel @@ -152,10 +154,10 @@ public final class MainWindow implements Runnable, RipStatusHandler { rateRefresherFuture.cancel(true); rateRefresherFuture = null; } - transferRateLabel.setText(""); + transferRateValue.setText("0.00 B/s"); return; } - transferRateLabel.setText(transferRate.formatHumanTransferRate()); + transferRateValue.setText(transferRate.formatHumanTransferRate()); }; private void updateQueue(DefaultListModel model) { @@ -293,15 +295,6 @@ private void createUI(JPanel pane) { } pane.setBorder(new EmptyBorder(5, 5, 5, 5)); - GridBagConstraints gbc = new GridBagConstraints(); - gbc.fill = GridBagConstraints.HORIZONTAL; - gbc.weightx = 1; - gbc.ipadx = 2; - gbc.gridx = 0; - gbc.weighty = 0; - gbc.ipady = 2; - gbc.gridy = 0; - gbc.anchor = GridBagConstraints.PAGE_START; try { UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName()); @@ -365,8 +358,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib stopButton.setIcon(new ImageIcon(stopIcon)); } catch (Exception ignored) { } - JPanel ripPanel = new JPanel(new GridBagLayout()); + GridBagConstraints gbc; + JPanel ripPanel = new JPanel(new GridBagLayout()); + gbc = new GridBagConstraints(); gbc.fill = GridBagConstraints.BOTH; gbc.weightx = 0; gbc.gridx = 0; @@ -386,30 +381,131 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.weightx = 1; statusLabel = new JLabel(Utils.getLocalizedString("inactive")); + statusLabel.setFont(new Font(Font.DIALOG, Font.PLAIN, statusLabel.getFont().getSize())); statusLabel.setHorizontalAlignment(JLabel.CENTER); - transferRateLabel.setHorizontalAlignment(JLabel.RIGHT); - transferRateLabel.setFont(monospaced); - openButton = new JButton(); - openButton.setVisible(false); - JPanel statusPanel = new JPanel(new GridBagLayout()); + JPanel statusDetailPanel = new JPanel(new GridBagLayout()); + + pendingValue.setFont(monospaced); + gbc = new GridBagConstraints(); gbc.gridx = 0; - gbc.weightx = 1; - statusPanel.add(statusLabel, gbc); + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.EAST; + gbc.ipadx = 5; + statusDetailPanel.add(pendingValue, gbc); + JLabel pendingLabel = new JLabel("Pending"); + gbc = new GridBagConstraints(); gbc.gridx = 1; - gbc.weightx = 0; - statusPanel.add(transferRateLabel, gbc); + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.WEST; + gbc.ipadx = 5; + statusDetailPanel.add(pendingLabel, gbc); + activeValue.setFont(monospaced); + gbc = new GridBagConstraints(); gbc.gridx = 0; - gbc.weightx = 1; - gbc.gridwidth = 2; gbc.gridy = 1; - statusPanel.add(openButton, gbc); + gbc.anchor = GridBagConstraints.EAST; + gbc.ipadx = 5; + statusDetailPanel.add(activeValue, gbc); + JLabel activeLabel = new JLabel("Active"); + gbc = new GridBagConstraints(); + gbc.gridx = 1; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.WEST; + gbc.ipadx = 5; + statusDetailPanel.add(activeLabel, gbc); + completedValue.setFont(monospaced); + gbc = new GridBagConstraints(); + gbc.gridx = 3; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.EAST; + gbc.ipadx = 5; + statusDetailPanel.add(completedValue, gbc); + JLabel completedLabel = new JLabel("Completed"); + gbc = new GridBagConstraints(); + gbc.gridx = 4; gbc.gridy = 0; - gbc.gridwidth = 1; + gbc.anchor = GridBagConstraints.WEST; + gbc.ipadx = 5; + statusDetailPanel.add(completedLabel, gbc); + erroredValue.setFont(monospaced); + gbc = new GridBagConstraints(); + gbc.gridx = 3; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.EAST; + gbc.ipadx = 5; + statusDetailPanel.add(erroredValue, gbc); + JLabel erroredLabel = new JLabel("Errored"); + gbc = new GridBagConstraints(); + gbc.gridx = 4; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.WEST; + gbc.ipadx = 5; + statusDetailPanel.add(erroredLabel, gbc); + totalValue.setFont(monospaced); + gbc = new GridBagConstraints(); + gbc.gridx = 6; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.EAST; + gbc.ipadx = 5; + statusDetailPanel.add(totalValue, gbc); + JLabel totalLabel = new JLabel("Total"); + gbc = new GridBagConstraints(); + gbc.gridx = 7; + gbc.gridy = 0; + gbc.anchor = GridBagConstraints.WEST; + gbc.ipadx = 5; + statusDetailPanel.add(totalLabel, gbc); + transferRateValue.setFont(monospaced); + gbc = new GridBagConstraints(); + gbc.gridx = 6; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.EAST; + gbc.ipadx = 5; + statusDetailPanel.add(transferRateValue, gbc); + JLabel transferRateLabel2 = new JLabel("Speed"); + gbc = new GridBagConstraints(); + gbc.gridx = 7; + gbc.gridy = 1; + gbc.anchor = GridBagConstraints.WEST; + gbc.ipadx = 5; + statusDetailPanel.add(transferRateLabel2, gbc); - JPanel progressPanel = new JPanel(new GridBagLayout()); - statusProgress = new JProgressBar(0, 100); - progressPanel.add(statusProgress, gbc); + final JPanel spacer1 = new JPanel(); + gbc = new GridBagConstraints(); + gbc.gridx = 2; + gbc.gridy = 0; + gbc.weightx = 1.0; + gbc.fill = GridBagConstraints.HORIZONTAL; + statusDetailPanel.add(spacer1, gbc); + final JPanel spacer2 = new JPanel(); + gbc = new GridBagConstraints(); + gbc.gridx = 5; + gbc.gridy = 0; + gbc.weightx = 1.0; + gbc.fill = GridBagConstraints.HORIZONTAL; + statusDetailPanel.add(spacer2, gbc); + + statusDetailPanel.setPreferredSize(new Dimension(350, statusDetailPanel.getPreferredSize().height)); + + openButton.setVisible(false); + + gbc = new GridBagConstraints(); + JPanel statusPanel = new JPanel(new GridBagLayout()); + + gbc.weightx = 1.0; + gbc.gridy = 0; + gbc.fill = GridBagConstraints.HORIZONTAL; + statusPanel.add(statusLabel, gbc); + gbc.gridy = 1; + gbc.fill = GridBagConstraints.HORIZONTAL; + statusPanel.add(currentlyRippingProgress, gbc); + gbc.gridy = 2; + gbc.fill = GridBagConstraints.NONE; + statusPanel.add(statusDetailPanel, gbc); + gbc.gridy = 3; + gbc.fill = GridBagConstraints.HORIZONTAL; + statusPanel.add(openButton, gbc); JPanel optionsPanel = new JPanel(new GridBagLayout()); optionLog = new JToggleButton(Utils.getLocalizedString("Log")); @@ -675,29 +771,21 @@ public void setValueAt(Object value, int row, int col) { optionQueue.putClientProperty(RIPME_PANEL, queuePanel); optionConfiguration.putClientProperty(RIPME_PANEL, configurationPanel); - gbc.anchor = GridBagConstraints.PAGE_START; gbc.gridy = 0; + gbc.gridx = 0; pane.add(ripPanel, gbc); gbc.gridy = 1; pane.add(statusPanel, gbc); gbc.gridy = 2; - pane.add(progressPanel, gbc); - gbc.gridy = 3; pane.add(optionsPanel, gbc); gbc.weighty = 1; gbc.fill = GridBagConstraints.BOTH; - gbc.gridy = 4; + gbc.gridy = 3; pane.add(logPanel, gbc); - gbc.gridy = 5; pane.add(historyPanel, gbc); - gbc.gridy = 5; pane.add(queuePanel, gbc); - gbc.gridy = 5; pane.add(configurationPanel, gbc); - gbc.gridy = 5; pane.add(emptyPanel, gbc); - gbc.weighty = 0; - gbc.fill = GridBagConstraints.HORIZONTAL; } private JTextField configField(String key, int defaultValue) { @@ -814,10 +902,9 @@ private void update() { gracefulStop.set(true); queueListModel.add(0, ripper.getURL().toString()); stopButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + currentlyRippingProgress.setValue(0); + currentlyRippingProgress.setText(""); pack(); - statusProgress.setValue(0); //status(Utils.getLocalizedString("download.interrupted")); status("Rip gracefully stopping"); appendLog("Download interrupted", Color.RED); @@ -832,10 +919,9 @@ private void update() { queueListModel.add(0, ripper.getURL().toString()); stopButton.setEnabled(false); panicButton.setEnabled(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + currentlyRippingProgress.setValue(0); + currentlyRippingProgress.setText(""); pack(); - statusProgress.setValue(0); status("Rip interrupted"); // TODO localize appendLog("Download interrupted", Color.RED); } @@ -1348,8 +1434,8 @@ private void ripFinishCleanup() { stopButton.setEnabled(false); panicButton.setEnabled(false); isRipperActive.set(false); - statusProgress.setValue(0); - statusProgress.setVisible(false); + currentlyRippingProgress.setValue(0); + currentlyRippingProgress.setText(""); } private Thread ripAlbum(String urlString) { @@ -1373,10 +1459,17 @@ private Thread ripAlbum(String urlString) { } stopButton.setEnabled(true); panicButton.setEnabled(true); - statusProgress.setValue(100); + currentlyRippingProgress.setValue(0); + currentlyRippingProgress.setText(urlString); openButton.setVisible(false); statusLabel.setVisible(true); - transferRateLabel.setVisible(true); + + pendingValue.setText("0"); + activeValue.setText("0"); + completedValue.setText("0"); + totalValue.setText("0"); + erroredValue.setText("0"); + pack(); boolean failed = false; try { @@ -1412,7 +1505,7 @@ private Thread ripAlbum(String urlString) { } } stopButton.setEnabled(false); - statusProgress.setValue(0); + currentlyRippingProgress.setValue(0); pack(); return null; } @@ -1509,14 +1602,30 @@ private synchronized void handleEvent(StatusEvent evt) { // CHUNK_BYTES is noisy, so handle it before any other computation if (status == RipStatusMessage.STATUS.CHUNK_BYTES) { transferRate.addChunk((Long) msg.getObject()); - transferRateLabel.setText(transferRate.formatHumanTransferRate()); + transferRateValue.setText(transferRate.formatHumanTransferRate()); return; } - int completedPercent = evt.ripper.getCompletionPercentage(); - statusProgress.setValue(completedPercent); - statusProgress.setVisible(true); - status(evt.ripper.getStatusText()); + if (evt.ripper.useByteProgessBar()) { + long bytesTotal = evt.ripper.getBytesTotal(); + long bytesCompleted = evt.ripper.getBytesCompleted(); + pendingValue.setText(Utils.bytesToHumanReadable(bytesTotal - bytesCompleted)); + completedValue.setText(Utils.bytesToHumanReadable(bytesCompleted)); + totalValue.setText(Utils.bytesToHumanReadable(bytesTotal)); + currentlyRippingProgress.setValue(evt.ripper.getCompletionPercentage()); + } else { + int pendingCount = evt.ripper.getPendingCount(); + int activeCount = evt.ripper.getActiveCount(); // included in pendingCount + int completedCount = evt.ripper.getCompletedCount(); + int erroredCount = evt.ripper.getErroredCount(); + int totalCount = pendingCount + completedCount + erroredCount; + pendingValue.setText(String.valueOf(Math.max(0, pendingCount - activeCount))); // exclude active + activeValue.setText(String.valueOf(activeCount)); + completedValue.setText(String.valueOf(completedCount)); + erroredValue.setText(String.valueOf(erroredCount)); + totalValue.setText(String.valueOf(totalCount)); + currentlyRippingProgress.setValue(evt.ripper.getCompletionPercentage()); + } switch (status) { case LOADING_RESOURCE: @@ -1637,6 +1746,7 @@ private synchronized void handleEvent(StatusEvent evt) { } } appendLog("Rip complete, saved to " + f, Color.GREEN); + status(""); openButton.setActionCommand(f.toString()); openButton.addActionListener(event -> { try { diff --git a/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java b/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java new file mode 100644 index 000000000..eaf5a408a --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java @@ -0,0 +1,55 @@ +package com.rarchives.ripme.ui; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.swing.*; +import java.awt.BorderLayout; +import java.awt.Color; +import java.awt.Font; + +public class ProgressTextField extends JProgressBar { + private static final Logger logger = LogManager.getLogger(ProgressTextField.class); + private final JTextField textField = new JTextField(); + private final JLabel valueLabel = new JLabel(); + + public ProgressTextField() { + super(0, 100); + setLayout(new BorderLayout()); + + // Paint an empty string to reserve height for the text field + super.setStringPainted(true); + progressString = ""; // Directly set here so setString can be overridden + + textField.setOpaque(false); + textField.setEditable(false); + textField.setBorder(BorderFactory.createEmptyBorder(0, 5, 0, 0)); + add(textField, BorderLayout.CENTER); + + valueLabel.setFont(new Font(Font.MONOSPACED, Font.PLAIN, valueLabel.getFont().getSize())); + valueLabel.setBorder(BorderFactory.createCompoundBorder( + BorderFactory.createMatteBorder(0, 1, 0, 0, Color.GRAY), + BorderFactory.createEmptyBorder(0, 5, 0, 5) + )); + setValue(0); + + add(valueLabel, BorderLayout.EAST); + } + + @Override + public void setString(String s) { + logger.warn("bug: progress bar should use setText, not setString"); + } + + @Override + public void setValue(int n) { + super.setValue(n); + int minimum = getMinimum(); + // note: integer division + valueLabel.setText(100 * (n - minimum) / (getMaximum() - minimum) + "%"); + } + + public void setText(String t) { + textField.setText(t); + } +} diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index f7272914d..332eda27b 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -538,7 +538,7 @@ public static String getOriginalDirectory(String path) throws IOException { * @param bytes Non-human readable integer. * @return Human readable interpretation of a byte. */ - public static String bytesToHumanReadable(int bytes) { + public static String bytesToHumanReadable(long bytes) { float fbytes = (float) bytes; String[] mags = new String[]{"", "K", "M", "G", "T"}; int magIndex = 0; From ec46ca4c0fee968e42e30f2433d552f5cdd51474 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 24 Jul 2025 18:36:38 -0400 Subject: [PATCH 33/78] Add ripper support for estimated total item count --- .../com/rarchives/ripme/ripper/AbstractHTMLRipper.java | 10 +++++++--- .../com/rarchives/ripme/ripper/AbstractJSONRipper.java | 10 +++++++--- .../java/com/rarchives/ripme/ripper/AlbumRipper.java | 7 +++++-- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 2 +- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java index 5e4152695..8b4ca631a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java @@ -166,7 +166,8 @@ public void rip() throws IOException, URISyntaxException { for (String imageURL : imageURLs) { imageIndex += 1; - logger.debug("Found image url #" + imageIndex + ": '" + imageURL + "'"); + logger.debug("Found image url #{} of album {}: {}", imageIndex, this.url, imageURL); + setItemsTotal(Math.max(getItemsTotal(), imageIndex)); downloadURL(new URI(imageURL).toURL(), imageIndex); if (isStopped() || isThisATest()) { break; @@ -381,8 +382,11 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { */ @Override public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); + double total = getTotalCount(); + if (total == 0) { + return 0; + } + return (int) (100 * ( (itemsCompleted.size() + itemsErrored.size()) / total)); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java index 7d71ba758..d2d6931b5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java @@ -89,7 +89,8 @@ public void rip() throws IOException, URISyntaxException { } imageIndex += 1; - logger.debug("Found image url #" + imageIndex+ ": " + imageURL); + logger.debug("Found image url #{} of album {}: {}", imageIndex, this.url, imageURL); + setItemsTotal(Math.max(getItemsTotal(), imageIndex)); downloadURL(new URI(imageURL).toURL(), imageIndex); } @@ -182,8 +183,11 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { */ @Override public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); + double total = getTotalCount(); + if (total == 0) { + return 0; + } + return (int) (100 * ( (itemsCompleted.size() + itemsErrored.size()) / total)); } @Override diff --git a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java index bad1961d9..552f78709 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java @@ -85,8 +85,11 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException { */ @Override public int getCompletionPercentage() { - double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size(); - return (int) (100 * ( (total - itemsPending.size()) / total)); + double total = getTotalCount(); + if (total == 0) { + return 0; + } + return (int) (100 * ( (itemsCompleted.size() + itemsErrored.size()) / total)); } } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index d469821b7..7335ebf63 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1618,7 +1618,7 @@ private synchronized void handleEvent(StatusEvent evt) { int activeCount = evt.ripper.getActiveCount(); // included in pendingCount int completedCount = evt.ripper.getCompletedCount(); int erroredCount = evt.ripper.getErroredCount(); - int totalCount = pendingCount + completedCount + erroredCount; + int totalCount = evt.ripper.getTotalCount(); pendingValue.setText(String.valueOf(Math.max(0, pendingCount - activeCount))); // exclude active activeValue.setText(String.valueOf(activeCount)); completedValue.setText(String.valueOf(completedCount)); From 0d21ee9ce1d2984a2a1ebc46dc7b66f98b27bb40 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Fri, 25 Jul 2025 14:27:06 -0400 Subject: [PATCH 34/78] Enable changing thread count live --- .../rarchives/ripme/ripper/AbstractRipper.java | 4 ++++ .../ripme/ripper/DownloadThreadPool.java | 11 +++++++++++ .../java/com/rarchives/ripme/ui/MainWindow.java | 15 +++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 44fc0e1f2..9f4868461 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -995,6 +995,10 @@ protected static boolean shouldIgnoreExtension(URL url) { return false; } + public void setThreadPoolSize(int size) { + ripperThreadPool.setThreadPoolSize(size); + } + public int getPendingCount() { DownloadThreadPool threadPool = getRipperThreadPool(); if (threadPool != null) { diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 2de0f1e80..b751e3339 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -40,6 +40,17 @@ public void addThread(Runnable t) { threadPool.execute(t); } + public void setThreadPoolSize(int threads) { + logger.debug("Setting thread pool size to {}", threads); + if (threads > threadPool.getMaximumPoolSize()) { + threadPool.setMaximumPoolSize(threads); + threadPool.setCorePoolSize(threads); + } else { + threadPool.setCorePoolSize(threads); + threadPool.setMaximumPoolSize(threads); + } + } + /** * Tries to shutdown threadpool. */ diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 7335ebf63..144e76e51 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -696,6 +696,21 @@ public void setValueAt(Object value, int row, int col) { configRetriesLabel = new JLabel(Utils.getLocalizedString("retry.download.count"), JLabel.RIGHT); configRetrySleepLabel = new JLabel(Utils.getLocalizedString("retry.sleep.mill"), JLabel.RIGHT); configThreadsText = configField("threads.size", 3); + configThreadsText.addActionListener(e -> { + Document document = configThreadsText.getDocument(); + LOGGER.info("Updating thread pool size"); + if (ripper != null && document != null) { + String text = configThreadsText.getText().trim(); + try { + int threads = Integer.parseInt(text); + if (threads >= 0) { + ripper.setThreadPoolSize(threads); + } + } catch (NumberFormatException ex) { + // ignore invalid input + } + } + }); configTimeoutText = configField("download.timeout", 60000); configRetriesText = configField("download.retries", 3); configRetrySleepText = configField("download.retry.sleep", 5000); From c54a3bbb91b782bf2cb53bf90ec59c03485c25e6 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 28 Jul 2025 21:53:26 -0400 Subject: [PATCH 35/78] Enable reordering queue --- .../com/rarchives/ripme/ui/MainWindow.java | 1 + .../ripme/ui/QueueMenuMouseListener.java | 49 +++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 144e76e51..63720a7db 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -666,6 +666,7 @@ public void setValueAt(Object value, int row, int col) { queueList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); QueueMenuMouseListener queueMenuMouseListener = new QueueMenuMouseListener(d -> updateQueue(queueListModel)); queueList.addMouseListener(queueMenuMouseListener); + queueList.addMouseMotionListener(queueMenuMouseListener); JScrollPane queueListScroll = new JScrollPane(queueList, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, JScrollPane.HORIZONTAL_SCROLLBAR_NEVER); diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index 0be4b46f8..0030a6df0 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -1,7 +1,6 @@ package com.rarchives.ripme.ui; import java.awt.event.ActionEvent; -import java.awt.event.InputEvent; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; import java.util.function.Consumer; @@ -12,6 +11,7 @@ import javax.swing.JList; import javax.swing.JOptionPane; import javax.swing.JPopupMenu; +import javax.swing.SwingUtilities; import com.rarchives.ripme.utils.Utils; @@ -20,13 +20,14 @@ class QueueMenuMouseListener extends MouseAdapter { private JList queueList; private DefaultListModel queueListModel; private Consumer> updateQueue; + private boolean mouseDragging = false; + private int dragSourceIndex; public QueueMenuMouseListener(Consumer> updateQueue) { this.updateQueue = updateQueue; updateUI(); } - @SuppressWarnings("serial") public void updateUI() { popup.removeAll(); @@ -61,16 +62,58 @@ public void actionPerformed(ActionEvent ae) { @Override public void mousePressed(MouseEvent e) { checkPopupTrigger(e); + handleDragStart(e); } @Override public void mouseReleased(MouseEvent e) { checkPopupTrigger(e); + handleDragEnd(e); + } + + @SuppressWarnings("unchecked") + public void handleDragStart(MouseEvent e) { + if (!(e.getSource() instanceof JList)) { + return; + } + if (SwingUtilities.isLeftMouseButton(e)) { + queueList = (JList) e.getSource(); + queueListModel = (DefaultListModel) queueList.getModel(); + + dragSourceIndex = queueList.getSelectedIndex(); + mouseDragging = true; + } + } + + public void handleDragEnd(MouseEvent e) { + mouseDragging = false; + } + + @Override + @SuppressWarnings("unchecked") + public void mouseDragged(MouseEvent e) { + if (!(e.getSource() instanceof JList)) { + return; + } + if (mouseDragging) { + queueList = (JList) e.getSource(); + queueListModel = (DefaultListModel) queueList.getModel(); + int currentIndex = queueList.locationToIndex(e.getPoint()); + if (currentIndex != dragSourceIndex) { + int dragTargetIndex = queueList.getSelectedIndex(); + dragTargetIndex = Math.max(0, dragTargetIndex); + dragTargetIndex = Math.min(queueListModel.size() - 1, dragTargetIndex); + Object dragElement = queueListModel.get(dragSourceIndex); + queueListModel.remove(dragSourceIndex); + queueListModel.add(dragTargetIndex, dragElement); + dragSourceIndex = currentIndex; + } + } } @SuppressWarnings("unchecked") private void checkPopupTrigger(MouseEvent e) { - if (e.getModifiersEx() == InputEvent.BUTTON3_DOWN_MASK) { + if (SwingUtilities.isRightMouseButton(e)) { if (!(e.getSource() instanceof JList)) { return; } From 4d618dfcf3f7839d63917abd9ece858c931fe602 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 18:18:11 -0400 Subject: [PATCH 36/78] Target current Java LTS version --- build.gradle.kts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index b8b4a613e..c3fc08546 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,7 +5,7 @@ // gradle clean build -PjavacRelease=21 // gradle clean build -PcustomVersion=1.0.0-10-asdf val customVersion = (project.findProperty("customVersion") ?: "") as String -val javacRelease = (project.findProperty("javacRelease") ?: "17") as String +val javacRelease = (project.findProperty("javacRelease") ?: "21") as String plugins { id("fr.brouillard.oss.gradle.jgitver") version "0.9.1" @@ -44,6 +44,11 @@ group = "com.rarchives.ripme" version = "1.7.94" description = "ripme" +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + jacoco { toolVersion = "0.8.12" } From b0218348c8b4af3a99bc6c7d482b23b744134cb0 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 8 Sep 2025 21:35:01 -0400 Subject: [PATCH 37/78] Use virtual threads --- .../java/com/rarchives/ripme/ripper/DownloadThreadPool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index b751e3339..77551cb9b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -26,7 +26,7 @@ public DownloadThreadPool(String threadPoolName) { int threads = Utils.getConfigInteger("threads.size", 10); logger.debug("Initializing " + threadPoolName + " thread pool with " + threads + " threads"); this.name = threadPoolName; - this.threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads); + this.threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads, Thread.ofVirtual().factory()); } /** From b188cf78f6398751aa37511b874a73a2e45912ac Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 29 Jul 2025 11:48:04 -0400 Subject: [PATCH 38/78] Prevent data loss: save queue on change Now that any time the queue list model changes, the config is saved, because the queue list model can change many times per second when modifying it, the call to save the config is debounced. --- .../com/rarchives/ripme/ui/MainWindow.java | 25 +++--- .../ripme/ui/QueueMenuMouseListener.java | 6 +- .../ripme/utils/DebouncedRunnable.java | 79 +++++++++++++++++++ .../java/com/rarchives/ripme/utils/Utils.java | 7 +- 4 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/utils/DebouncedRunnable.java diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 63720a7db..cd7483f5d 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -40,6 +40,7 @@ import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.uiUtils.ContextActionProtections; +import com.rarchives.ripme.utils.DebouncedRunnable; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.TransferRate; import com.rarchives.ripme.utils.Utils; @@ -143,6 +144,8 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static final AtomicBoolean panicStop = new AtomicBoolean(false); // Immediately stop active transfers, then stop ripping. private static final AtomicBoolean isRipperActive = new AtomicBoolean(false); + private final DebouncedRunnable debouncedSaveConfig = new DebouncedRunnable(Utils::saveConfig, 500); + public static final int TRANSFER_RATE_REFRESH_RATE = 200; private static final TransferRate transferRate = new TransferRate(); @@ -160,21 +163,11 @@ public final class MainWindow implements Runnable, RipStatusHandler { transferRateValue.setText(transferRate.formatHumanTransferRate()); }; - private void updateQueue(DefaultListModel model) { - if (model == null) - model = queueListModel; - - if (model.size() > 0) { - Utils.setConfigList("queue", model.elements()); - Utils.saveConfig(); - } - - MainWindow.optionQueue.setText(String.format("%s%s", Utils.getLocalizedString("queue"), - model.size() == 0 ? "" : "(" + model.size() + ")")); - } - private void updateQueue() { - updateQueue(null); + Utils.setConfigList("queue", queueListModel.elements()); + debouncedSaveConfig.run(); + MainWindow.optionQueue.setText(String.format("%s%s", Utils.getLocalizedString("queue"), + queueListModel.isEmpty() ? "" : "(" + queueListModel.size() + ")")); } private static void addCheckboxListener(JCheckBox checkBox, String configString) { @@ -664,7 +657,7 @@ public void setValueAt(Object value, int row, int col) { queueListModel = new DefaultListModel<>(); JList queueList = new JList<>(queueListModel); queueList.setSelectionMode(ListSelectionModel.MULTIPLE_INTERVAL_SELECTION); - QueueMenuMouseListener queueMenuMouseListener = new QueueMenuMouseListener(d -> updateQueue(queueListModel)); + QueueMenuMouseListener queueMenuMouseListener = new QueueMenuMouseListener(); queueList.addMouseListener(queueMenuMouseListener); queueList.addMouseMotionListener(queueMenuMouseListener); JScrollPane queueListScroll = new JScrollPane(queueList, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, @@ -1146,10 +1139,12 @@ public void intervalAdded(ListDataEvent arg0) { @Override public void contentsChanged(ListDataEvent arg0) { + updateQueue(); } @Override public void intervalRemoved(ListDataEvent arg0) { + updateQueue(); } }); } diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index 0030a6df0..cbad95315 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -3,7 +3,6 @@ import java.awt.event.ActionEvent; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; -import java.util.function.Consumer; import javax.swing.AbstractAction; import javax.swing.Action; @@ -19,12 +18,10 @@ class QueueMenuMouseListener extends MouseAdapter { private JPopupMenu popup = new JPopupMenu(); private JList queueList; private DefaultListModel queueListModel; - private Consumer> updateQueue; private boolean mouseDragging = false; private int dragSourceIndex; - public QueueMenuMouseListener(Consumer> updateQueue) { - this.updateQueue = updateQueue; + public QueueMenuMouseListener() { updateUI(); } @@ -56,7 +53,6 @@ public void actionPerformed(ActionEvent ae) { }; popup.add(clearQueue); - updateQueue.accept(queueListModel); } @Override diff --git a/src/main/java/com/rarchives/ripme/utils/DebouncedRunnable.java b/src/main/java/com/rarchives/ripme/utils/DebouncedRunnable.java new file mode 100644 index 000000000..135bffdce --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/DebouncedRunnable.java @@ -0,0 +1,79 @@ +package com.rarchives.ripme.utils; + +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +public class DebouncedRunnable implements Runnable, AutoCloseable { + private final ScheduledExecutorService scheduler; + private final Runnable task; + private final long maxDelayMs; + private final AtomicLong lastInvoke = new AtomicLong(); + private final AtomicLong lastRun = new AtomicLong(); + private volatile ScheduledFuture future; + private final AtomicLong counter = new AtomicLong(0); + private final AtomicBoolean active = new AtomicBoolean(false); + private final Thread shutdownHook; + private volatile boolean closed = false; + + /** + * Debounce a task. + * @param task The task to run + * @param maxDelayMs Maximum delay to wait between calls in milliseconds + */ + public DebouncedRunnable(Runnable task, long maxDelayMs) { + this.task = task; + this.maxDelayMs = maxDelayMs; + this.scheduler = Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()); + this.shutdownHook = new Thread(this::shutdown); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + + public void run() { + long now = System.currentTimeMillis(); + long timeSinceLastRun = now - lastRun.getAndSet(now); + boolean overMaxDelaySinceLastRun = timeSinceLastRun >= maxDelayMs; + if (overMaxDelaySinceLastRun) { + invoke(); + return; + } + long timeSinceLastInvoke = now - lastInvoke.get(); + boolean underMaxDelaySinceLastInvoke = timeSinceLastInvoke < maxDelayMs; + if (underMaxDelaySinceLastInvoke && future != null) { + future.cancel(false); + } + future = scheduler.schedule(this::invoke, maxDelayMs - timeSinceLastInvoke, TimeUnit.MILLISECONDS); + + } + + private void invoke() { + try { + active.set(true); + task.run(); + } finally { + lastInvoke.set(System.currentTimeMillis()); + active.set(false); + } + } + + public void shutdown() { + closed = true; + scheduler.shutdown(); + if (future != null && !future.isDone() && !active.get()) { + // Shutting down, but we still have a scheduled thread + future.cancel(false); + invoke(); + } + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException e) { + // The shutdown has already begun + } + } + + public void close() throws Exception { + if (!closed) { + shutdown(); + } + } +} diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 332eda27b..43f958239 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -197,7 +197,12 @@ public static void setConfigList(String key, Enumeration enumeration) { public static void saveConfig() { try { - config.save(getConfigFilePath()); + // Simple hack: saveConfig is called from debouncedSaveConfig. + // Clone the config so we don't get a ConcurrentModificationException + // on config.save() if the config is updated between the time + // debounceSavedConfig.run() is called and the time saveConfig() is called. + PropertiesConfiguration clone = (PropertiesConfiguration) config.clone(); + clone.save(getConfigFilePath()); LOGGER.info("Saved configuration to " + getConfigFilePath()); } catch (ConfigurationException e) { LOGGER.error("Error while saving configuration: ", e); From 92494704eafff459d93f85687cee913e209fd56f Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 29 Jul 2025 12:13:23 -0400 Subject: [PATCH 39/78] Add rip number column to History table Rip number is the default order, but if you sorted by one of the other columns, there was no way to restore the original rip number sort order --- .../java/com/rarchives/ripme/ui/History.java | 24 +++---- .../ripme/ui/HistoryMenuMouseListener.java | 9 +-- .../com/rarchives/ripme/ui/MainWindow.java | 66 ++++++++++++++----- 3 files changed, 64 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/History.java b/src/main/java/com/rarchives/ripme/ui/History.java index 190eeeb8e..735a133ac 100644 --- a/src/main/java/com/rarchives/ripme/ui/History.java +++ b/src/main/java/com/rarchives/ripme/ui/History.java @@ -19,6 +19,7 @@ public class History { private final List list; private static final String[] COLUMNS = new String[] { "URL", + "N", "created", "modified", "#", @@ -52,20 +53,15 @@ public int getColumnCount() { } public Object getValueAt(int row, int col) { HistoryEntry entry = this.list.get(row); - switch (col) { - case 0: - return entry.url; - case 1: - return dateToHumanReadable(entry.startDate); - case 2: - return dateToHumanReadable(entry.modifiedDate); - case 3: - return entry.count; - case 4: - return entry.selected; - default: - return null; - } + return switch (col) { + case 0 -> entry.url; + case 1 -> row; + case 2 -> dateToHumanReadable(entry.startDate); + case 3 -> dateToHumanReadable(entry.modifiedDate); + case 4 -> entry.count; + case 5 -> entry.selected; + default -> null; + }; } private String dateToHumanReadable(Date date) { SimpleDateFormat sdf = new SimpleDateFormat("yyyy/MM/dd"); diff --git a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java index 8a69477cc..0f6045e59 100644 --- a/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/HistoryMenuMouseListener.java @@ -15,6 +15,7 @@ class HistoryMenuMouseListener extends MouseAdapter { private JPopupMenu popup = new JPopupMenu(); private JTable tableComponent; + private final int checkboxColumn = 5; @SuppressWarnings("serial") public HistoryMenuMouseListener() { @@ -22,7 +23,7 @@ public HistoryMenuMouseListener() { @Override public void actionPerformed(ActionEvent ae) { for (int row = 0; row < tableComponent.getRowCount(); row++) { - tableComponent.setValueAt(true, row, 4); + tableComponent.setValueAt(true, row, checkboxColumn); } } }; @@ -32,7 +33,7 @@ public void actionPerformed(ActionEvent ae) { @Override public void actionPerformed(ActionEvent ae) { for (int row = 0; row < tableComponent.getRowCount(); row++) { - tableComponent.setValueAt(false, row, 4); + tableComponent.setValueAt(false, row, checkboxColumn); } } }; @@ -44,7 +45,7 @@ public void actionPerformed(ActionEvent ae) { @Override public void actionPerformed(ActionEvent ae) { for (int row : tableComponent.getSelectedRows()) { - tableComponent.setValueAt(true, row, 4); + tableComponent.setValueAt(true, row, checkboxColumn); } } }; @@ -54,7 +55,7 @@ public void actionPerformed(ActionEvent ae) { @Override public void actionPerformed(ActionEvent ae) { for (int row : tableComponent.getSelectedRows()) { - tableComponent.setValueAt(false, row, 4); + tableComponent.setValueAt(false, row, checkboxColumn); } } }; diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index cd7483f5d..8326aec5d 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -32,6 +32,7 @@ import javax.swing.event.ListDataEvent; import javax.swing.event.ListDataListener; import javax.swing.table.AbstractTableModel; +import javax.swing.table.TableCellRenderer; import javax.swing.text.*; import org.apache.logging.log4j.Level; @@ -591,12 +592,12 @@ public int getColumnCount() { @Override public boolean isCellEditable(int row, int col) { - return (col == 0 || col == 4); + return (col == 0 || col == 5); } @Override public void setValueAt(Object value, int row, int col) { - if (col == 4) { + if (col == 5) { HISTORY.get(row).selected = (Boolean) value; historyTableModel.fireTableDataChanged(); } @@ -607,21 +608,12 @@ public void setValueAt(Object value, int row, int col) { historyTable.addMouseListener(new HistoryMenuMouseListener()); historyTable.setAutoCreateRowSorter(true); - for (int i = 0; i < historyTable.getColumnModel().getColumnCount(); i++) { - int width = 130; // Default - switch (i) { - case 0: // URL - width = 270; - break; - case 3: - width = 40; - break; - case 4: - width = 15; - break; - } - historyTable.getColumnModel().getColumn(i).setPreferredWidth(width); - } + historyTable.getColumnModel().getColumn(0).setPreferredWidth(270); // URL + //historyTable.getColumnModel().getColumn(1).setPreferredWidth(270); // Number + //historyTable.getColumnModel().getColumn(2).setPreferredWidth(130); // Date + //historyTable.getColumnModel().getColumn(3).setPreferredWidth(130); // Date + historyTable.getColumnModel().getColumn(4).setPreferredWidth(40); // Count + historyTable.getColumnModel().getColumn(5).setPreferredWidth(15); // Selected JScrollPane historyTableScrollPane = new JScrollPane(historyTable); historyButtonRemove = new JButton(Utils.getLocalizedString("remove")); @@ -1372,6 +1364,46 @@ private void loadHistory() throws IOException { // Fix "WARNING: row index is bigger than sorter's row count. Most likely this is a wrong sorter usage" historyTableModel.fireTableDataChanged(); } + + // Calculate preferred column widths + int autoResizeMode = historyTable.getAutoResizeMode(); + historyTable.setAutoResizeMode(JTable.AUTO_RESIZE_OFF); + int lastRow = historyTable.getRowCount() - 1; + int nColumn = 1; + TableCellRenderer renderer; + renderer = historyTable.getCellRenderer(lastRow, nColumn); + Component comp = historyTable.prepareRenderer(renderer, lastRow, nColumn); + int width = Math.min(Math.max(15, comp.getMinimumSize().width + 15), 130); + historyTable.getColumnModel().getColumn(nColumn).setPreferredWidth(width); + historyTable.getColumnModel().getColumn(nColumn).setMaxWidth(130); + + renderer = historyTable.getDefaultRenderer(String.class); + int cWidth; + int column; + column = 2; // date + cWidth = renderer.getTableCellRendererComponent( + historyTable, "8888/88/88", false, false, 0, column) + .getPreferredSize().width + 15; + historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); + column = 3; // date + cWidth = renderer.getTableCellRendererComponent( + historyTable, "8888/88/88", false, false, 0, column) + .getPreferredSize().width + 15; + historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); + column = 4; // count + cWidth = renderer.getTableCellRendererComponent( + historyTable, "88888", false, false, 0, column) + .getPreferredSize().width + 15; + historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); + + renderer = historyTable.getDefaultRenderer(Boolean.class); + column = 5; // selected + cWidth = renderer.getTableCellRendererComponent( + historyTable, true, false, false, 0, column) + .getPreferredSize().width; + historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); + + historyTable.setAutoResizeMode(autoResizeMode); } private void saveHistory() { From d96a1936c10e2d6518fd855a3d0023d81237a12f Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 26 Aug 2025 23:42:32 -0400 Subject: [PATCH 40/78] Update localization key --- .../java/com/rarchives/ripme/ui/QueueMenuMouseListener.java | 2 +- src/main/resources/LabelsBundle.properties | 2 +- src/main/resources/LabelsBundle_el_GR.properties | 4 ++-- src/main/resources/LabelsBundle_es_ES.properties | 4 ++-- src/main/resources/LabelsBundle_pt_PT.properties | 4 ++-- src/main/resources/LabelsBundle_zh_CN.properties | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index cbad95315..6da380d88 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -44,7 +44,7 @@ public void actionPerformed(ActionEvent ae) { Action clearQueue = new AbstractAction(Utils.getLocalizedString("queue.remove.all")) { @Override public void actionPerformed(ActionEvent ae) { - if (JOptionPane.showConfirmDialog(null, Utils.getLocalizedString("queue.validation"), "RipMe", + if (JOptionPane.showConfirmDialog(null, Utils.getLocalizedString("queue.remove.all.validation"), "RipMe", JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) { queueListModel.removeAllElements(); updateUI(); diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 6a48b245e..b73a9f51e 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -31,7 +31,7 @@ loading.history.from = Loading history from # Queue keys queue.remove.all = Remove All -queue.validation = Are you sure you want to remove all elements from the queue? +queue.remove.all.validation = Are you sure you want to remove all elements from the queue? queue.remove.selected = Remove Selected # History diff --git a/src/main/resources/LabelsBundle_el_GR.properties b/src/main/resources/LabelsBundle_el_GR.properties index 14656e877..07c61900f 100644 --- a/src/main/resources/LabelsBundle_el_GR.properties +++ b/src/main/resources/LabelsBundle_el_GR.properties @@ -29,7 +29,7 @@ loading.history.from = Φόρτωση ιστορικού από # Queue keys queue.remove.all = Διαγραφή όλων -queue.validation = Είσαι σίγουρος οτι θέλεις να διαγράφουν όλα τα στοιχεια της ουράς? +queue.remove.all.validation = Είσαι σίγουρος οτι θέλεις να διαγράφουν όλα τα στοιχεια της ουράς? queue.remove.selected = Διαγραφή επιλεγμένου # History @@ -72,4 +72,4 @@ http.status.exception = HTTP status λάθος exception.while.downloading.file = Λάθος ενω μεταφορτώνοταν ενα αρχειο failed.to.download = Αποτυχία μεταφόρτωσης skipping = Παράκαμψη -file.already.exists = το αρχείο υπάρχει ήδη \ No newline at end of file +file.already.exists = το αρχείο υπάρχει ήδη diff --git a/src/main/resources/LabelsBundle_es_ES.properties b/src/main/resources/LabelsBundle_es_ES.properties index fea84e5d5..96f170dbf 100644 --- a/src/main/resources/LabelsBundle_es_ES.properties +++ b/src/main/resources/LabelsBundle_es_ES.properties @@ -30,7 +30,7 @@ loading.history.from = Cargando historia desde # Queue keys queue.remove.all = Eliminar todos los elementos -queue.validation = ¿Está seguro que desea eliminar todos los elementos de la lista? +queue.remove.all.validation = ¿Está seguro que desea eliminar todos los elementos de la lista? queue.remove.selected = Eliminar elementos seleccionados # History @@ -73,4 +73,4 @@ http.status.exception = Error de estado HTTP exception.while.downloading.file = Error al descargar archivo failed.to.download = Descarga fallida skipping = Saltando -file.already.exists = el fichero ya existe \ No newline at end of file +file.already.exists = el fichero ya existe diff --git a/src/main/resources/LabelsBundle_pt_PT.properties b/src/main/resources/LabelsBundle_pt_PT.properties index 500049ce9..dba5adca3 100644 --- a/src/main/resources/LabelsBundle_pt_PT.properties +++ b/src/main/resources/LabelsBundle_pt_PT.properties @@ -29,7 +29,7 @@ loading.history.from = Carregar histórico de # Queue keys queue.remove.all = Remover todos -queue.validation = Tem a certeza de que quer remover todos os elementos da fila? +queue.remove.all.validation = Tem a certeza de que quer remover todos os elementos da fila? queue.remove.selected = Remover seleccionados # History @@ -72,4 +72,4 @@ http.status.exception = Exceção de status HTTP exception.while.downloading.file = Exceção enquanto o ficheiro era baixado failed.to.download = Falha no download skipping = Pulando -file.already.exists = Ficheiro já existe \ No newline at end of file +file.already.exists = Ficheiro já existe diff --git a/src/main/resources/LabelsBundle_zh_CN.properties b/src/main/resources/LabelsBundle_zh_CN.properties index 7cf6d7810..c7865cc44 100644 --- a/src/main/resources/LabelsBundle_zh_CN.properties +++ b/src/main/resources/LabelsBundle_zh_CN.properties @@ -29,7 +29,7 @@ loading.history.from = 加载历史从 # Queue keys queue.remove.all = 移除全部 -queue.validation = 您确定要移除队列内的全部项目？ +queue.remove.all.validation = 您确定要移除队列内的全部项目？ queue.remove.selected = 移除所选项目 # History @@ -72,4 +72,4 @@ http.status.exception = HTTP 状态意外 exception.while.downloading.file = 下载文件时发生意外 failed.to.download = 下载失败 skipping = 跳过 -file.already.exists = 文件已存在 \ No newline at end of file +file.already.exists = 文件已存在 From 67c039ada86aab82cebc86b3a65c4892b64bccf4 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 26 Aug 2025 23:45:45 -0400 Subject: [PATCH 41/78] Add confirmation for single selection queue remove If it's annoying, then just select all the elements you want to remove at once. --- .../rarchives/ripme/ui/QueueMenuMouseListener.java | 13 ++++++++----- src/main/resources/LabelsBundle.properties | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index 6da380d88..28c9c57af 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -31,12 +31,15 @@ public void updateUI() { Action removeSelected = new AbstractAction(Utils.getLocalizedString("queue.remove.selected")) { @Override public void actionPerformed(ActionEvent ae) { - Object o = queueList.getSelectedValue(); - while (o != null) { - queueListModel.removeElement(o); - o = queueList.getSelectedValue(); + if (JOptionPane.showConfirmDialog(null, Utils.getLocalizedString("queue.remove.selected.validation"), "RipMe", + JOptionPane.YES_NO_OPTION) == JOptionPane.YES_OPTION) { + Object o = queueList.getSelectedValue(); + while (o != null) { + queueListModel.removeElement(o); + o = queueList.getSelectedValue(); + } + updateUI(); } - updateUI(); } }; popup.add(removeSelected); diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index b73a9f51e..92d15b2d2 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -33,6 +33,7 @@ loading.history.from = Loading history from queue.remove.all = Remove All queue.remove.all.validation = Are you sure you want to remove all elements from the queue? queue.remove.selected = Remove Selected +queue.remove.selected.validation = Are you sure you want to remove the selected elements from the queue? # History re-rip.checked = Re-rip Checked From 37134278dd6f1a84487a22997c2e7cadade7e111 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 27 Aug 2025 00:00:04 -0400 Subject: [PATCH 42/78] Enable moving queue selection to top or bottom --- .../ripme/ui/QueueMenuMouseListener.java | 28 +++++++++++++++++++ src/main/resources/LabelsBundle.properties | 2 ++ 2 files changed, 30 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java index 28c9c57af..de794730c 100644 --- a/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java +++ b/src/main/java/com/rarchives/ripme/ui/QueueMenuMouseListener.java @@ -3,6 +3,7 @@ import java.awt.event.ActionEvent; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; +import java.util.List; import javax.swing.AbstractAction; import javax.swing.Action; @@ -56,6 +57,33 @@ public void actionPerformed(ActionEvent ae) { }; popup.add(clearQueue); + Action moveSelectedToTop = new AbstractAction(Utils.getLocalizedString("queue.move.selected.to.top")) { + @Override + public void actionPerformed(ActionEvent ae) { + List selectedElements = queueList.getSelectedValuesList(); + for (Object selectedElement : selectedElements) { + queueListModel.removeElement(selectedElement); + } + queueListModel.addAll(0, selectedElements); + queueList.setSelectionInterval(0, selectedElements.size() - 1); + updateUI(); + } + }; + popup.add(moveSelectedToTop); + + Action moveSelectedToBottom = new AbstractAction(Utils.getLocalizedString("queue.move.selected.to.bottom")) { + @Override + public void actionPerformed(ActionEvent ae) { + List selectedElements = queueList.getSelectedValuesList(); + for (Object selectedElement : selectedElements) { + queueListModel.removeElement(selectedElement); + } + queueListModel.addAll(selectedElements); + queueList.setSelectionInterval(queueListModel.size() - selectedElements.size(), queueListModel.size() - 1); + updateUI(); + } + }; + popup.add(moveSelectedToBottom); } @Override diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 92d15b2d2..90ecb31ec 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -34,6 +34,8 @@ queue.remove.all = Remove All queue.remove.all.validation = Are you sure you want to remove all elements from the queue? queue.remove.selected = Remove Selected queue.remove.selected.validation = Are you sure you want to remove the selected elements from the queue? +queue.move.selected.to.top = Move Selected to Top +queue.move.selected.to.bottom = Move Selected to Bottom # History re-rip.checked = Re-rip Checked From db4cf2465c971962e898000ab9f9fc4ae000c8a3 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 27 Aug 2025 00:07:52 -0400 Subject: [PATCH 43/78] Simplify queue initialization --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 8326aec5d..bee98b552 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -655,9 +655,7 @@ public void setValueAt(Object value, int row, int col) { JScrollPane queueListScroll = new JScrollPane(queueList, JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, JScrollPane.HORIZONTAL_SCROLLBAR_NEVER); - for (String item : Utils.getConfigList("queue")) { - queueListModel.addElement(item); - } + queueListModel.addAll(Utils.getConfigList("queue")); updateQueue(); gbc.gridx = 0; From 648d1911882a917cc40cdb4a1a4c2be54ef1d113 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 27 Aug 2025 00:08:43 -0400 Subject: [PATCH 44/78] Do not clobber rip text field on auto rip --- .../com/rarchives/ripme/ui/MainWindow.java | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index bee98b552..88b656187 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -22,6 +22,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.stream.Stream; import javax.imageio.ImageIO; @@ -139,6 +140,9 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static Image mainIcon; + private static Function addUserInputUrlToQueueStatic; + private static Runnable ripNextAlbumStatic; + private static AbstractRipper ripper; private static final AtomicBoolean gracefulStop = new AtomicBoolean(false); // Allow active transfers to finish, then stop ripping. @@ -859,6 +863,8 @@ private void changeLocale() { } private void setupHandlers() { + addUserInputUrlToQueueStatic = this::addUserInputUrlToQueue; + ripNextAlbumStatic = this::ripNextAlbum; ripButton.addActionListener(new RipButtonHandler(this)); ripTextfield.addActionListener(new RipButtonHandler(this)); ripTextfield.getDocument().addDocumentListener(new DocumentListener() { @@ -1551,7 +1557,7 @@ private Thread ripAlbum(String urlString) { return null; } - private boolean canRip(String urlString) { + private static boolean canRip(String urlString) { try { String urlText = urlString.trim(); if (urlText.equals("")) { @@ -1581,6 +1587,43 @@ public static DefaultListModel getQueueListModel() { return queueListModel; } + /** + * @param url User input that might be a URL + * @return true if the URL is in the queue + */ + private boolean addUserInputUrlToQueue(String url) { + boolean urlInQueue = false; + boolean url_not_empty = !url.equals(""); + if (!queueListModel.contains(url) && url_not_empty) { + // Check if we're ripping a range of urls + if (url.contains("{")) { + // Make sure the user hasn't forgotten the closing } + if (url.contains("}")) { + String rangeToParse = url.substring(url.indexOf("{") + 1, url.indexOf("}")); + int rangeStart = Integer.parseInt(rangeToParse.split("-")[0]); + int rangeEnd = Integer.parseInt(rangeToParse.split("-")[1]); + for (int i = rangeStart; i < rangeEnd + 1; i++) { + String realURL = url.replaceAll("\\{\\S*\\}", Integer.toString(i)); + if (canRip(realURL)) { + queueListModel.addElement(realURL); + urlInQueue = true; + } else { + displayAndLogError("Can't find ripper for " + realURL, Color.RED); + } + } + } + } else { + queueListModel.addElement(url); + urlInQueue = true; + } + } else if (url_not_empty) { + displayAndLogError("This URL is already in queue: " + url, Color.RED); + statusWithColor("This URL is already in queue: " + url, Color.ORANGE); + urlInQueue = true; + } + return urlInQueue; + } + static class RipButtonHandler implements ActionListener { private MainWindow mainWindow; @@ -1590,32 +1633,8 @@ public RipButtonHandler(MainWindow mainWindow) { public void actionPerformed(ActionEvent event) { String url = ripTextfield.getText(); - boolean url_not_empty = !url.equals(""); - if (!queueListModel.contains(url) && url_not_empty) { - // Check if we're ripping a range of urls - if (url.contains("{")) { - // Make sure the user hasn't forgotten the closing } - if (url.contains("}")) { - String rangeToParse = url.substring(url.indexOf("{") + 1, url.indexOf("}")); - int rangeStart = Integer.parseInt(rangeToParse.split("-")[0]); - int rangeEnd = Integer.parseInt(rangeToParse.split("-")[1]); - for (int i = rangeStart; i < rangeEnd + 1; i++) { - String realURL = url.replaceAll("\\{\\S*\\}", Integer.toString(i)); - if (mainWindow.canRip(realURL)) { - queueListModel.addElement(realURL); - ripTextfield.setText(""); - } else { - mainWindow.displayAndLogError("Can't find ripper for " + realURL, Color.RED); - } - } - } - } else { - queueListModel.addElement(url); - ripTextfield.setText(""); - } - } else if (url_not_empty) { - mainWindow.displayAndLogError("This URL is already in queue: " + url, Color.RED); - mainWindow.statusWithColor("This URL is already in queue: " + url, Color.ORANGE); + boolean urlInQueue = mainWindow.addUserInputUrlToQueue(url); + if (urlInQueue) { ripTextfield.setText(""); } mainWindow.ripNextAlbum(); @@ -1824,8 +1843,10 @@ public void update(AbstractRipper ripper, RipStatusMessage message) { } public static void ripAlbumStatic(String url) { - ripTextfield.setText(url.trim()); - ripButton.doClick(); + boolean urlInQueue = addUserInputUrlToQueueStatic.apply(url.trim()); + if (urlInQueue) { + ripNextAlbumStatic.run(); + } } private static boolean hasWindowPositionBug() { From 8152b51d192a58fa5ee50de550a5c08c359fc0ab Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 29 Jul 2025 13:11:52 -0400 Subject: [PATCH 45/78] Fix open button Duplicate ActionListeners would be added each time a rip completed, causing multiple file manager windows to open, but now only one ActionListener is added. --- .../java/com/rarchives/ripme/ui/MainWindow.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 88b656187..0266a3ebb 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -932,6 +932,14 @@ private void update() { } }); + openButton.addActionListener(event -> { + try { + Desktop.getDesktop().open(new File(event.getActionCommand())); + } catch (Exception e) { + LOGGER.error(e); + } + }); + ActionListener panelSelectListener = event -> { JToggleButton source = (JToggleButton) event.getSource(); Enumeration buttons = panelButtonGroup.getElements(); @@ -1808,13 +1816,6 @@ private synchronized void handleEvent(StatusEvent evt) { appendLog("Rip complete, saved to " + f, Color.GREEN); status(""); openButton.setActionCommand(f.toString()); - openButton.addActionListener(event -> { - try { - Desktop.getDesktop().open(new File(event.getActionCommand())); - } catch (Exception e) { - LOGGER.error(e); - } - }); ripFinishCleanup(); pack(); ripNextAlbum(); From 98a7ecab8444df4f9bfa514d344d2034b87da463 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 29 Jul 2025 13:44:58 -0400 Subject: [PATCH 46/78] FlickrRipper: set total items --- .../java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java index 37acde4ac..a958b0972 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FlickrRipper.java @@ -253,6 +253,8 @@ public List getURLsFromPage(Document doc) { } int totalPages = rootData.getInt("pages"); + int totalFiles = rootData.getInt("total"); + setItemsTotal(totalFiles); logger.info(jsonData); JSONArray pictures = rootData.getJSONArray("photo"); for (int i = 0; i < pictures.length(); i++) { From faad8f458859360d18bf147753607c9e105c2f4f Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 30 Jul 2025 00:33:33 -0400 Subject: [PATCH 47/78] Flatten nesting --- .../ripme/ripper/AbstractRipper.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 9f4868461..20e9ecc11 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -361,17 +361,16 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di } return true; } - else { - itemsPending.add(ripUrlId); - DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); - if (referrer != null) { - dft.setReferrer(referrer); - } - if (cookies != null) { - dft.setCookies(cookies); - } - getRipperThreadPool().addThread(dft); + + itemsPending.add(ripUrlId); + DownloadFileThread dft = new DownloadFileThread(tug, ripUrlId, directory, filename, this, getFileExtFromMIME); + if (referrer != null) { + dft.setReferrer(referrer); + } + if (cookies != null) { + dft.setCookies(cookies); } + getRipperThreadPool().addThread(dft); return true; } From 856569ea3af45697bbcdc1a0a857f4af8ff17dda Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 18:19:07 -0400 Subject: [PATCH 48/78] Add simple addUrlToDownload TokenedUrlGetter helper --- .../java/com/rarchives/ripme/ripper/AbstractRipper.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 20e9ecc11..38740f5cb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -310,6 +310,12 @@ public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map cookies, Boolean getFileExtFromMIME) { return addURLToDownload(tug, ripUrlId, directory, null, referrer, cookies, getFileExtFromMIME); } From c2ac3eb7dacb7edd4fe37e50a09d807f3ddb2efb Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 18:24:44 -0400 Subject: [PATCH 49/78] Fix log message --- .../java/com/rarchives/ripme/ripper/DownloadFileThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 85bcc6097..e3669fa25 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -122,7 +122,7 @@ public void run() { || Utils.fuzzyExists(Paths.get(saveAs.getParent()), saveAs.getName()) && getFileExtFromMIME && !observer.tryResumeDownload()) { if (Utils.getConfigBoolean("file.overwrite", false)) { - logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + prettySaveAs); + logger.info("[!] " + Utils.getLocalizedString("deleting.existing.file") + " " + prettySaveAs); if (!saveAs.delete()) logger.error("could not delete existing file: " + saveAs.getAbsolutePath()); } else { logger.info("[!] " + Utils.getLocalizedString("skipping") + " " + url + " -- " From 8ba77295a5410e2cb16a620d1c9a3100ba8db1b5 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 18:24:50 -0400 Subject: [PATCH 50/78] Add comment --- src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index e3669fa25..5ef353796 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -187,6 +187,7 @@ public void run() { return; } if (statusCode / 100 == 3) { // 3xx Redirect + // FIXME Should not happen because of above line: huc.setInstanceFollowRedirects(true); ??? if (!redirected) { // Don't increment retries on the first redirect tries--; From 861275a9e55eab616cfac40c67c6c50538c8e4d6 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 18:26:06 -0400 Subject: [PATCH 51/78] Fix opening folders and links on Linux --- .../ripme/ripper/AbstractRipper.java | 3 +- .../com/rarchives/ripme/ui/MainWindow.java | 7 ++- .../java/com/rarchives/ripme/utils/Utils.java | 50 +++++++++++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 38740f5cb..f3870259e 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -1,6 +1,5 @@ package com.rarchives.ripme.ripper; -import java.awt.Desktop; import java.io.BufferedWriter; import java.io.File; import java.io.FileNotFoundException; @@ -751,7 +750,7 @@ protected void notifyComplete() { if (Utils.getConfigBoolean("urls_only.save", false)) { String urlFile = this.workingDir + File.separator + "urls.txt"; try { - Desktop.getDesktop().open(new File(urlFile)); + Utils.open(new File(urlFile)); } catch (IOException e) { logger.warn("Error while opening " + urlFile, e); } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 0266a3ebb..d43ee2372 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -934,7 +934,7 @@ private void update() { openButton.addActionListener(event -> { try { - Desktop.getDesktop().open(new File(event.getActionCommand())); + Utils.open(new File(event.getActionCommand())); } catch (Exception e) { LOGGER.error(e); } @@ -1057,8 +1057,7 @@ public void mouseClicked(MouseEvent e) { Path file; try { file = Utils.getWorkingDirectory(); - Desktop desktop = Desktop.getDesktop(); - desktop.open(file.toFile()); + Utils.open(file.toFile()); } catch (IOException ex) { LOGGER.warn(ex.getMessage()); } @@ -1241,7 +1240,7 @@ public void windowIconified(WindowEvent e) { JOptionPane.YES_NO_OPTION, JOptionPane.PLAIN_MESSAGE, new ImageIcon(mainIcon)); if (response == JOptionPane.YES_OPTION) { try { - Desktop.getDesktop().browse(URI.create("http://github.com/ripmeapp/ripme")); + Utils.browse(URI.create("http://github.com/ripmeapp/ripme")); } catch (IOException e) { LOGGER.error("Exception while opening project home page", e); } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 43f958239..38684bdcf 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -1,5 +1,6 @@ package com.rarchives.ripme.utils; +import java.awt.Desktop; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -896,4 +897,53 @@ public static void sleep(long time) { e1.printStackTrace(); } } + + /** + * Open a File using the default OS handler, but not in a child process, allowing the program to exit without closing all opened windows. + *
+ * In comparison, {@link Desktop#open(File)} opens the file manager or browser in a child process, blocking the JVM from exiting. + * @param file The File to open. + */ + public static void open(File file) throws IOException { + if (isUnix() && which("nohup") && which("xdg-open")) { + linuxOpen(file.toURI()); + return; + } + Desktop.getDesktop().open(file); + } + + /** + * Open a URI using the default OS handler, but not in a child process, allowing the program to exit without closing all opened windows. + *
+ * In comparison, {@link Desktop#open(File)} opens the file manager or browser in a child process, blocking the JVM from exiting. + * @param uri The URI to open. + */ + public static void browse(URI uri) throws IOException { + if (isUnix() && which("nohup") && which("xdg-open")) { + linuxOpen(uri); + return; + } + Desktop.getDesktop().browse(uri); + } + + private static void linuxOpen(URI uri) throws IOException { + new ProcessBuilder("nohup", "xdg-open", uri.toString()) + .redirectOutput(ProcessBuilder.Redirect.DISCARD) + .redirectError(ProcessBuilder.Redirect.DISCARD) + .start(); + } + + public static boolean which(String command) { + String pathEnv = System.getenv("PATH"); + if (isWindows()) { + command = command + ".exe"; + } + for (String dir : pathEnv.split(File.pathSeparator)) { + File file = new File(dir, command); + if (file.isFile() && file.canExecute()) { + return true; + } + } + return false; + } } From 8f589e26a51ebb1c572acd46b5361b93dadbd3b5 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 18:31:46 -0400 Subject: [PATCH 52/78] Improve logging --- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 4 +++- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index f3870259e..7b3410d9b 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -732,7 +732,7 @@ protected void notifyComplete() { } if (!completed.getAndSet(true)) { - logger.info(" Rip completed!"); + logger.info(" Rip of {} completed!", getURL()); RipStatusComplete rsc = new RipStatusComplete(workingDir.toPath(), getCount()); RipStatusMessage msg = new RipStatusMessage(STATUS.RIP_COMPLETE, rsc); @@ -862,6 +862,7 @@ public void sendUpdate(STATUS status, Object message) { */ public void run() { try { + logger.info("Rip started: {}", getURL()); rip(); } catch (HttpStatusException e) { logger.error("Got exception while running ripper:", e); @@ -872,6 +873,7 @@ public void run() { waitForRipperThreads(false); sendUpdate(STATUS.RIP_ERRORED, e.getMessage()); } finally { + logger.info("Rip ended: {}", getURL()); cleanup(); } } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index d43ee2372..8434d18e7 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1497,6 +1497,8 @@ private Thread ripAlbum(String urlString) { optionLog.doClick(); } urlString = urlString.trim(); + LOGGER.info("Attempting to start rip for album {}", urlString); + appendLog("Attempting to start rip for album " + urlString, Color.GREEN); if (urlString.toLowerCase().startsWith("gonewild:")) { urlString = "http://gonewild.com/user/" + urlString.substring(urlString.indexOf(':') + 1); } @@ -1813,6 +1815,7 @@ private synchronized void handleEvent(StatusEvent evt) { } } appendLog("Rip complete, saved to " + f, Color.GREEN); + LOGGER.info("Rip complete: {}", url); status(""); openButton.setActionCommand(f.toString()); ripFinishCleanup(); From 91c2d0c45c43e4db6e994fa9413422db4d262df5 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 21:15:48 -0400 Subject: [PATCH 53/78] Fix initial pack --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 8434d18e7..d1dd459b3 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -228,6 +228,7 @@ public void run() { pack(); restoreWindowPosition(mainFrame); mainFrame.setVisible(true); + mainFrame.pack(); } private void shutdownCleanup() { From 59f86d675e9c0772c25c2f598374df90b6fa53ca Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 21:17:03 -0400 Subject: [PATCH 54/78] Reduce button padding --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index d1dd459b3..d45a6bcde 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -286,6 +286,10 @@ private boolean isCollapsed() { } private void createUI(JPanel pane) { + + Insets buttonPadding = new Insets(2,2,2,2); + UIManager.getDefaults().put("Button.margin", buttonPadding); + // If creating the tray icon fails, ignore it. try { setupTrayIcon(); From 71b7c326f22bed7bed9c33adb6fbe3769d0d5e21 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 21:19:59 -0400 Subject: [PATCH 55/78] No HTML buttons --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index d45a6bcde..b5313b133 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -351,10 +351,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib */ ImageIcon ripIcon = new ImageIcon(mainIcon); - ripButton = new JButton("Rip", ripIcon); - stopButton = new JButton("Stop"); + ripButton = new JButton("Rip", ripIcon); + stopButton = new JButton("Stop"); stopButton.setEnabled(false); - panicButton = new JButton("Panic!"); + panicButton = new JButton("Panic!"); panicButton.setEnabled(false); try { Image stopIcon = ImageIO.read(getClass().getClassLoader().getResource("stop.png")); From 05e19da834dd88b6548eb66a7d65b7b34f54c417 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 22:40:04 -0400 Subject: [PATCH 56/78] Prevent status collapse; no empty string --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index b5313b133..cb1eb2a48 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -265,6 +265,9 @@ private void error(String text) { } private void statusWithColor(String text, Color color) { + if (text == null || text.trim().isEmpty()) { + return; + } statusLabel.setForeground(color); statusLabel.setText(text); pack(); @@ -1821,7 +1824,7 @@ private synchronized void handleEvent(StatusEvent evt) { } appendLog("Rip complete, saved to " + f, Color.GREEN); LOGGER.info("Rip complete: {}", url); - status(""); + status(Utils.getLocalizedString("inactive")); openButton.setActionCommand(f.toString()); ripFinishCleanup(); pack(); From 6ebaf459e338999ef2836e13c073bf1691d69e7a Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Mon, 4 Aug 2025 22:40:17 -0400 Subject: [PATCH 57/78] Set date column preferred width --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index cb1eb2a48..f855c5bcb 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1403,11 +1403,13 @@ private void loadHistory() throws IOException { cWidth = renderer.getTableCellRendererComponent( historyTable, "8888/88/88", false, false, 0, column) .getPreferredSize().width + 15; + historyTable.getColumnModel().getColumn(column).setPreferredWidth(cWidth); historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); column = 3; // date cWidth = renderer.getTableCellRendererComponent( historyTable, "8888/88/88", false, false, 0, column) .getPreferredSize().width + 15; + historyTable.getColumnModel().getColumn(column).setPreferredWidth(cWidth); historyTable.getColumnModel().getColumn(column).setMaxWidth(cWidth); column = 4; // count cWidth = renderer.getTableCellRendererComponent( From da5c8cb281ba3ee247b55c936d46a061a1fb5a44 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 01:35:12 -0400 Subject: [PATCH 58/78] Extract Pattern field --- src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java index 8d3fc1af7..e8af2a488 100644 --- a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java @@ -47,6 +47,7 @@ public static String getClipboardString() { } class AutoripThread extends Thread { + private static final Pattern rippableUrlPattern = Pattern.compile("^(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); volatile boolean isRunning = false; private final Set rippedURLs = new HashSet<>(); @@ -57,8 +58,7 @@ public void run() { // Check clipboard String clipboard = ClipboardUtils.getClipboardString(); if (clipboard != null) { - Pattern p = Pattern.compile("^(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); - Matcher m = p.matcher(clipboard); + Matcher m = rippableUrlPattern.matcher(clipboard); while (m.find()) { String url = m.group(); if (!rippedURLs.contains(url)) { From a1408e9fdcb5e37431009d3e2f64130b47f26935 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 01:51:17 -0400 Subject: [PATCH 59/78] Reduce clipboard rip interval It's stupid, but Java has no cross-platform API to watch clipboard changes. DataFlavor listeners only watch changes in DataFlavor; changes of underlying data with the same DataFlavor are ignored. Using ClipboardOwner causes a feedback loop with any other program that grabs clipboard ownership on a loop. 250ms is long enough that it shouldn't suck too much battery, but short enough that links won't be missed when copied rapidly. --- src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java index e8af2a488..9705de6f3 100644 --- a/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java +++ b/src/main/java/com/rarchives/ripme/ui/ClipboardUtils.java @@ -68,7 +68,7 @@ public void run() { } } } - Thread.sleep(1000); + Thread.sleep(250); } } catch (InterruptedException e) { e.printStackTrace(); From 235ae48d708657870e9cad83e2b3cf85466fd379 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 02:10:37 -0400 Subject: [PATCH 60/78] Add padding to queue button for queue size --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index f855c5bcb..50e080e66 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -545,7 +545,14 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib // Prevent button sizes/positions from shifting when text bolds/unbolds optionLog.setPreferredSize(optionLog.getPreferredSize()); optionHistory.setPreferredSize(optionHistory.getPreferredSize()); + + // Set preferred size with space for queue size string + optionQueue.setText(Utils.getLocalizedString("queue") + " (8888)"); optionQueue.setPreferredSize(optionQueue.getPreferredSize()); + // Restore original text + optionQueue.setText(Utils.getLocalizedString("queue")); + // updateQueue() is called below, which initializes the real queue button text + optionConfiguration.setPreferredSize(optionConfiguration.getPreferredSize()); gbc.gridx = 0; From 29f61729a5b5a9d932deaa37e6f0dbaf6f0c7ef6 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 02:14:34 -0400 Subject: [PATCH 61/78] Add log message --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 50e080e66..c03f0c559 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1359,6 +1359,7 @@ private void loadHistory() throws IOException { try { LOGGER.info(Utils.getLocalizedString("loading.history.from") + " " + historyFile.getCanonicalPath()); HISTORY.fromFile(historyFile.getCanonicalPath()); + LOGGER.info("Finished loading history"); } catch (IOException e) { LOGGER.error("Failed to load history from file " + historyFile, e); JOptionPane.showMessageDialog(null, From 590ad10d0d752533a64d1781a4e2e4737083aa01 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 03:33:44 -0400 Subject: [PATCH 62/78] Silence unimportant warning --- src/main/java/com/rarchives/ripme/ui/ProgressTextField.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java b/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java index eaf5a408a..74b23bc8e 100644 --- a/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java +++ b/src/main/java/com/rarchives/ripme/ui/ProgressTextField.java @@ -38,7 +38,10 @@ public ProgressTextField() { @Override public void setString(String s) { - logger.warn("bug: progress bar should use setText, not setString"); + if (s != null) { + // super() calls setString(null) during construction; not a problem + logger.warn("bug: progress bar should use setText, not setString"); + } } @Override From ad4ac80b72de57e96fe9edfd5657b651218bb187 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 04:46:27 -0400 Subject: [PATCH 63/78] Set transfer rate value width hint --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index c03f0c559..cf51b86e3 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -463,6 +463,11 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.ipadx = 5; statusDetailPanel.add(totalLabel, gbc); transferRateValue.setFont(monospaced); + transferRateValue.setText("1000.00 KiB/s"); // Maximum width value + // Set preferred size to maximum width value + transferRateValue.setPreferredSize(transferRateValue.getPreferredSize()); + transferRateValue.setMinimumSize(transferRateValue.getPreferredSize()); + transferRateValue.setText("0 B/s"); // Restore default value gbc = new GridBagConstraints(); gbc.gridx = 6; gbc.gridy = 1; From 5a762a1b953ee5affbf940fc12c51859722570e9 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 11:15:42 -0400 Subject: [PATCH 64/78] Fix label alignment when setting preferred size --- .../com/rarchives/ripme/ui/MainWindow.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index cf51b86e3..2920e433d 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -27,6 +27,7 @@ import javax.imageio.ImageIO; import javax.swing.*; +import javax.swing.border.Border; import javax.swing.border.EmptyBorder; import javax.swing.event.DocumentEvent; import javax.swing.event.DocumentListener; @@ -392,77 +393,81 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib JPanel statusDetailPanel = new JPanel(new GridBagLayout()); + Border valueLabelBorder = BorderFactory.createEmptyBorder(0, 5, 0, 5); + pendingValue.setFont(monospaced); + pendingValue.setHorizontalAlignment(JLabel.TRAILING); + pendingValue.setBorder(valueLabelBorder); gbc = new GridBagConstraints(); gbc.gridx = 0; gbc.gridy = 0; gbc.anchor = GridBagConstraints.EAST; - gbc.ipadx = 5; statusDetailPanel.add(pendingValue, gbc); JLabel pendingLabel = new JLabel("Pending"); gbc = new GridBagConstraints(); gbc.gridx = 1; gbc.gridy = 0; gbc.anchor = GridBagConstraints.WEST; - gbc.ipadx = 5; statusDetailPanel.add(pendingLabel, gbc); activeValue.setFont(monospaced); + activeValue.setHorizontalAlignment(JLabel.TRAILING); + activeValue.setBorder(valueLabelBorder); gbc = new GridBagConstraints(); gbc.gridx = 0; gbc.gridy = 1; gbc.anchor = GridBagConstraints.EAST; - gbc.ipadx = 5; statusDetailPanel.add(activeValue, gbc); JLabel activeLabel = new JLabel("Active"); gbc = new GridBagConstraints(); gbc.gridx = 1; gbc.gridy = 1; gbc.anchor = GridBagConstraints.WEST; - gbc.ipadx = 5; statusDetailPanel.add(activeLabel, gbc); completedValue.setFont(monospaced); + completedValue.setHorizontalAlignment(JLabel.TRAILING); + completedValue.setBorder(valueLabelBorder); gbc = new GridBagConstraints(); gbc.gridx = 3; gbc.gridy = 0; gbc.anchor = GridBagConstraints.EAST; - gbc.ipadx = 5; statusDetailPanel.add(completedValue, gbc); JLabel completedLabel = new JLabel("Completed"); gbc = new GridBagConstraints(); gbc.gridx = 4; gbc.gridy = 0; gbc.anchor = GridBagConstraints.WEST; - gbc.ipadx = 5; statusDetailPanel.add(completedLabel, gbc); erroredValue.setFont(monospaced); + erroredValue.setHorizontalAlignment(JLabel.TRAILING); + erroredValue.setBorder(valueLabelBorder); gbc = new GridBagConstraints(); gbc.gridx = 3; gbc.gridy = 1; gbc.anchor = GridBagConstraints.EAST; - gbc.ipadx = 5; statusDetailPanel.add(erroredValue, gbc); JLabel erroredLabel = new JLabel("Errored"); gbc = new GridBagConstraints(); gbc.gridx = 4; gbc.gridy = 1; gbc.anchor = GridBagConstraints.WEST; - gbc.ipadx = 5; statusDetailPanel.add(erroredLabel, gbc); totalValue.setFont(monospaced); + totalValue.setHorizontalAlignment(JLabel.TRAILING); + totalValue.setBorder(valueLabelBorder); gbc = new GridBagConstraints(); gbc.gridx = 6; gbc.gridy = 0; gbc.anchor = GridBagConstraints.EAST; - gbc.ipadx = 5; statusDetailPanel.add(totalValue, gbc); JLabel totalLabel = new JLabel("Total"); gbc = new GridBagConstraints(); gbc.gridx = 7; gbc.gridy = 0; gbc.anchor = GridBagConstraints.WEST; - gbc.ipadx = 5; statusDetailPanel.add(totalLabel, gbc); transferRateValue.setFont(monospaced); + transferRateValue.setHorizontalAlignment(JLabel.TRAILING); + transferRateValue.setBorder(valueLabelBorder); transferRateValue.setText("1000.00 KiB/s"); // Maximum width value // Set preferred size to maximum width value transferRateValue.setPreferredSize(transferRateValue.getPreferredSize()); @@ -472,14 +477,12 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridx = 6; gbc.gridy = 1; gbc.anchor = GridBagConstraints.EAST; - gbc.ipadx = 5; statusDetailPanel.add(transferRateValue, gbc); JLabel transferRateLabel2 = new JLabel("Speed"); gbc = new GridBagConstraints(); gbc.gridx = 7; gbc.gridy = 1; gbc.anchor = GridBagConstraints.WEST; - gbc.ipadx = 5; statusDetailPanel.add(transferRateLabel2, gbc); final JPanel spacer1 = new JPanel(); From 5948fdae480305c56beaaa0bbe98568d3f2d7576 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 11:33:28 -0400 Subject: [PATCH 65/78] Set preferred size of other values To reduce width jumping when counters go up an order of magnitude --- .../com/rarchives/ripme/ui/MainWindow.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 2920e433d..de1d38140 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -398,6 +398,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib pendingValue.setFont(monospaced); pendingValue.setHorizontalAlignment(JLabel.TRAILING); pendingValue.setBorder(valueLabelBorder); + pendingValue.setText("1000"); + pendingValue.setPreferredSize(pendingValue.getPreferredSize()); + pendingValue.setMinimumSize(pendingValue.getPreferredSize()); + pendingValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 0; gbc.gridy = 0; @@ -412,6 +416,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib activeValue.setFont(monospaced); activeValue.setHorizontalAlignment(JLabel.TRAILING); activeValue.setBorder(valueLabelBorder); + activeValue.setText("1000"); + activeValue.setPreferredSize(activeValue.getPreferredSize()); + activeValue.setMinimumSize(activeValue.getPreferredSize()); + activeValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 0; gbc.gridy = 1; @@ -426,6 +434,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib completedValue.setFont(monospaced); completedValue.setHorizontalAlignment(JLabel.TRAILING); completedValue.setBorder(valueLabelBorder); + completedValue.setText("1000"); + completedValue.setPreferredSize(completedValue.getPreferredSize()); + completedValue.setMinimumSize(completedValue.getPreferredSize()); + completedValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 3; gbc.gridy = 0; @@ -440,6 +452,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib erroredValue.setFont(monospaced); erroredValue.setHorizontalAlignment(JLabel.TRAILING); erroredValue.setBorder(valueLabelBorder); + erroredValue.setText("1000"); + erroredValue.setPreferredSize(erroredValue.getPreferredSize()); + erroredValue.setMinimumSize(erroredValue.getPreferredSize()); + erroredValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 3; gbc.gridy = 1; @@ -454,6 +470,10 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib totalValue.setFont(monospaced); totalValue.setHorizontalAlignment(JLabel.TRAILING); totalValue.setBorder(valueLabelBorder); + totalValue.setText("1000"); + totalValue.setPreferredSize(totalValue.getPreferredSize()); + totalValue.setMinimumSize(totalValue.getPreferredSize()); + totalValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 6; gbc.gridy = 0; From eeb91dd3f29fa37180e823e5a22494930d5f8056 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Tue, 5 Aug 2025 20:44:22 -0400 Subject: [PATCH 66/78] Preserve status when gracefully stopping --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index de1d38140..07a861243 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -950,8 +950,6 @@ private void update() { gracefulStop.set(true); queueListModel.add(0, ripper.getURL().toString()); stopButton.setEnabled(false); - currentlyRippingProgress.setValue(0); - currentlyRippingProgress.setText(""); pack(); //status(Utils.getLocalizedString("download.interrupted")); status("Rip gracefully stopping"); From 60d8e78efd9d6427ac71ddfa88c54458fafd7528 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 6 Aug 2025 03:45:06 -0400 Subject: [PATCH 67/78] Add Mockito --- build.gradle.kts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index c3fc08546..1922b1277 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { implementation("org.graalvm.js:js:22.3.2") testImplementation(enforcedPlatform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation("org.mockito:mockito-core:5.+") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } @@ -115,6 +116,8 @@ tasks.test { includeEngines("junit-vintage") } finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run + + jvmArgs("-javaagent:${classpath.find { it.name.contains("mockito") }?.absolutePath}") } tasks.register("testAll") { From ae5d694b542208c0bdab2ded2bc494168a81a3c5 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 6 Aug 2025 03:41:29 -0400 Subject: [PATCH 68/78] Minimize transfer rate width --- .../com/rarchives/ripme/ui/MainWindow.java | 2 +- .../rarchives/ripme/utils/TransferRate.java | 14 ++++---- .../ripme/utils/TransferRateTest.java | 35 +++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 src/test/java/com/rarchives/ripme/utils/TransferRateTest.java diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 07a861243..1c899cf3a 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -488,7 +488,7 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib transferRateValue.setFont(monospaced); transferRateValue.setHorizontalAlignment(JLabel.TRAILING); transferRateValue.setBorder(valueLabelBorder); - transferRateValue.setText("1000.00 KiB/s"); // Maximum width value + transferRateValue.setText("999.00 KiB/s"); // Maximum width value // Set preferred size to maximum width value transferRateValue.setPreferredSize(transferRateValue.getPreferredSize()); transferRateValue.setMinimumSize(transferRateValue.getPreferredSize()); diff --git a/src/main/java/com/rarchives/ripme/utils/TransferRate.java b/src/main/java/com/rarchives/ripme/utils/TransferRate.java index 9227160f3..c41d4296f 100644 --- a/src/main/java/com/rarchives/ripme/utils/TransferRate.java +++ b/src/main/java/com/rarchives/ripme/utils/TransferRate.java @@ -40,14 +40,16 @@ public String formatHumanTransferRate() { int giB = 1024 * 1024 * 1024; int miB = 1024 * 1024; int kiB = 1024; - if (bps >= giB) { - return String.format("%.2f GiB/s", bps / giB); - } else if (bps >= miB) { - return String.format("%.2f MiB/s", bps / miB); - } else if (bps >= kiB) { + + // Format the string for less than 4 integer part digits if possible + if (bps < 1000) { + return String.format("%.2f B/s", bps); + } else if (bps / kiB < 1000) { return String.format("%.2f KiB/s", bps / kiB); + } else if (bps / miB < 1000) { + return String.format("%.2f MiB/s", bps / miB); } else { - return String.format("%.2f B/s", bps); + return String.format("%.2f GiB/s", bps / giB); } } diff --git a/src/test/java/com/rarchives/ripme/utils/TransferRateTest.java b/src/test/java/com/rarchives/ripme/utils/TransferRateTest.java new file mode 100644 index 000000000..66d1463cc --- /dev/null +++ b/src/test/java/com/rarchives/ripme/utils/TransferRateTest.java @@ -0,0 +1,35 @@ +package com.rarchives.ripme.utils; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class TransferRateTest { + + @Test + void formatHumanTransferRate() { + //TransferRate transferRate = new TransferRate(); + TransferRate transferRate = Mockito.spy(); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(999.); + Assertions.assertEquals("999.00 B/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1013.); + Assertions.assertEquals("0.99 KiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1024); + Assertions.assertEquals("1.00 MiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1013); + Assertions.assertEquals("0.99 MiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1024 * 1024); + Assertions.assertEquals("1.00 GiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1024 * 1013); + Assertions.assertEquals("0.99 GiB/s", transferRate.formatHumanTransferRate()); + + Mockito.when(transferRate.calculateBytesPerSecond()).thenReturn(1024.0 * 1024 * 1024 * 1013); + Assertions.assertEquals("1013.00 GiB/s", transferRate.formatHumanTransferRate()); + } +} From 778f1b929e47e86768f8c01f8850f3fdb073d6a5 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 6 Aug 2025 03:55:07 -0400 Subject: [PATCH 69/78] Stop retry when not ripping (ugly) --- src/main/java/com/rarchives/ripme/utils/Http.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index a1705f5a9..559f60e58 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -201,6 +201,10 @@ public Response response() throws IOException { IOException lastException = null; int retries = this.retries; while (--retries >= 0) { + // TODO uncomment and fix tests + //if (!MainWindow.isRipping.get()) { + // throw new IOException("Rip stopped, not making http request"); + //} try { response = connection.execute(); return response; From 5696177d5ed2ea0d3dee7993bff62b68972a9d2d Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 6 Aug 2025 19:33:50 -0400 Subject: [PATCH 70/78] Only open log if no panel is already open --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 1c899cf3a..289ad51cc 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1537,7 +1537,8 @@ private void ripFinishCleanup() { } private Thread ripAlbum(String urlString) { - if (!logPanel.isVisible()) { + boolean isAnyPanelOpen = panelButtonGroup.getSelection() != null; + if (!isAnyPanelOpen) { optionLog.doClick(); } urlString = urlString.trim(); From 63be4ea6054460da001969b16cd0c1aff0b911e7 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Wed, 13 Aug 2025 15:31:30 -0400 Subject: [PATCH 71/78] Check if rip is possible --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 289ad51cc..8db042e99 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1667,8 +1667,12 @@ private boolean addUserInputUrlToQueue(String url) { } } } else { - queueListModel.addElement(url); - urlInQueue = true; + if (canRip(url)) { + queueListModel.addElement(url); + urlInQueue = true; + } else { + displayAndLogError("Can't find ripper for " + url, Color.RED); + } } } else if (url_not_empty) { displayAndLogError("This URL is already in queue: " + url, Color.RED); From 9420258ff634f324cf4a0a550abf6a5296ece57d Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:12:32 -0400 Subject: [PATCH 72/78] Check ENOSPC locale independently --- .../ripme/ripper/DownloadFileThread.java | 24 +++++++++++++++++-- src/main/resources/LabelsBundle.properties | 3 ++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 5ef353796..5e39b3ff2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -2,6 +2,7 @@ import java.io.*; import java.net.*; +import java.nio.file.FileStore; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -334,9 +335,9 @@ public void run() { return; } } catch (IOException e) { - if (e.getMessage().matches("No space left on device")) { + if (guessIsENOSPC(e, saveAs)) { logger.debug("IOException", e); - observer.downloadErrored(ripUrlId, "No space left on device"); // TODO localize; TODO cancel all rips + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("no.space.left.on.device")); // TODO cancel all rips return; } logger.debug("IOException", e); @@ -388,4 +389,23 @@ public void run() { logger.info("[+] Saved " + url + " as " + prettySaveAs); } + @SuppressWarnings("UnnecessaryLocalVariable") + private boolean guessIsENOSPC(IOException e, File saveAs) { + // The ENOSPC IOException message is localized in Java, so this only works on English locale systems. + if (e.getMessage().matches("No space left on device")) { + return true; + } + // Fallback: check usable space on the filesystem + try { + FileStore fs = Files.getFileStore(saveAs.toPath()); + // could check for 0 bytes, but 256 kilobytes is small enough + int downloadBufferSizeBytes = 1024 * 256; + boolean notEnoughUsableBytes = fs.getUsableSpace() < downloadBufferSizeBytes; + return notEnoughUsableBytes; + } catch (IOException ex) { + // unable to determine if no space left on device; fall through + } + return false; + } + } diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 90ecb31ec..936448693 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -77,4 +77,5 @@ http.status.exception = HTTP status exception exception.while.downloading.file = Exception while downloading file failed.to.download = Failed to download skipping = Skipping -file.already.exists = file already exists \ No newline at end of file +file.already.exists = file already exists +no.space.left.on.device = No space left on device From 1a208fdb990c0832c59950a29b8964806f3b22d8 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:00:03 -0400 Subject: [PATCH 73/78] Localize some strings --- .../ripme/ripper/AbstractRipper.java | 4 +-- .../ripme/ripper/DownloadFileThread.java | 25 ++++++++++--------- .../com/rarchives/ripme/ui/MainWindow.java | 14 ++++++----- .../java/com/rarchives/ripme/utils/Utils.java | 14 ++++++++--- src/main/resources/LabelsBundle.properties | 13 ++++++++++ 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 7b3410d9b..06fe8cab2 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -353,7 +353,7 @@ public boolean addURLToDownload(TokenedUrlGetter tug, RipUrlId ripUrlId, Path di return false; } if (AbstractRipper.shouldIgnoreExtension(url)) { - sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + sendUpdate(STATUS.DOWNLOAD_SKIP, Utils.getLocalizedString("skipping.ignored.extension") + ": " + url.toExternalForm()); return false; } String text = url.toExternalForm() + System.lineSeparator(); @@ -710,7 +710,7 @@ protected void downloadExists(RipUrlId ripUrlId, Path file) { itemsPending.remove(ripUrlId); itemsCompleted.put(ripUrlId, file); - observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, ripUrlId + " already saved as " + file)); + observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, Utils.getLocalizedString("0.already.saved.as.1", ripUrlId, file))); //checkIfComplete(); } diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java index 5e39b3ff2..41c12e7cf 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java @@ -79,12 +79,12 @@ public void run() { try { url = tokenedUrlGetter.getTokenedUrl(); } catch (HttpStatusException e) { - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.get.url.for.0", ripUrlId)); logger.error("[!] Failed to get URL for " + ripUrlId); return; // do not retry } catch (IOException | URISyntaxException e) { logger.error("[!] Failed to get URL for " + ripUrlId, e); - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.get.url.for.0", ripUrlId)); return; // do not retry } if (filename == null) { @@ -94,7 +94,7 @@ public void run() { // First thing we make sure the file name doesn't have any illegal chars in it filename = Utils.sanitizeSaveAs(filename); if (AbstractRipper.shouldIgnoreExtension(url)) { - observer.sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension"); + observer.sendUpdate(STATUS.DOWNLOAD_SKIP, Utils.getLocalizedString("skipping.ignored.extension") + ": " + url.toExternalForm()); return; } @@ -104,7 +104,7 @@ public void run() { Files.createDirectories(directory); } catch (IOException e) { logger.error("Error creating directory", e); - observer.downloadErrored(ripUrlId, "Error creating directory: " + directory + " ; " + e.getMessage()); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("error.creating.directory") + ": " + directory + " ; " + e.getMessage()); return; } } @@ -182,8 +182,9 @@ public void run() { if (statusCode != 206 && observer.tryResumeDownload() && saveAs.exists()) { // TODO find a better way to handle servers that don't support resuming // downloads then just erroring out - observer.downloadErrored(ripUrlId, "Local file exists, resume attempted, but server does not support resuming downloads: " - + statusCode + " while downloading " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, + Utils.getLocalizedString("server.doesnt.support.resuming.downloads") + ": " + + Utils.getLocalizedString("0.while.downloading.1", statusCode, url.toExternalForm())); //throw new IOException(Utils.getLocalizedString("server.doesnt.support.resuming.downloads")); return; } @@ -203,12 +204,12 @@ public void run() { logger.error("[!] " + Utils.getLocalizedString("nonretriable.status.code") + " " + statusCode + " while downloading from " + url); observer.downloadErrored(ripUrlId, Utils.getLocalizedString("nonretriable.status.code") + " " - + statusCode + " while downloading " + url.toExternalForm()); + + Utils.getLocalizedString("0.while.downloading.1", statusCode, url.toExternalForm())); return; // Not retriable, drop out. } if (statusCode / 100 == 5) { // 5xx errors - observer.downloadErrored(ripUrlId, Utils.getLocalizedString("retriable.status.code") + " " + statusCode - + " while downloading " + url.toExternalForm()); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("retriable.status.code") + " " + + Utils.getLocalizedString("0.while.downloading.1", statusCode, url.toExternalForm())); // Throw exception so download can be retried throw new IOException(Utils.getLocalizedString("retriable.status.code") + " " + statusCode); } @@ -331,7 +332,7 @@ public void run() { logger.error("[!] HTTP status " + hse.getStatusCode() + " while downloading from " + hse.getUrl()); if (hse.getStatusCode() == 404 && Utils.getConfigBoolean("errors.skip404", false)) { observer.downloadErrored(ripUrlId, - "HTTP status code " + hse.getStatusCode() + " while downloading " + url.toExternalForm()); + Utils.getLocalizedString("0.while.downloading.1", "HTTP " + hse.getStatusCode(), url.toExternalForm())); return; } } catch (IOException e) { @@ -375,12 +376,12 @@ public void run() { try { url = tokenedUrlGetter.getTokenedUrl(); } catch (HttpStatusException e) { - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.get.url.for.0", ripUrlId)); logger.error("[!] Failed to get URL for " + ripUrlId); return; // do not retry } catch (IOException | URISyntaxException e) { logger.error("[!] Failed to get URL for " + ripUrlId, e); - observer.downloadErrored(ripUrlId, "Failed to get URL for " + ripUrlId); + observer.downloadErrored(ripUrlId, Utils.getLocalizedString("failed.to.get.url.for.0", ripUrlId)); return; // do not retry } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 8db042e99..29e383726 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -903,6 +903,9 @@ private void changeLocale() { optionHistory.setText(Utils.getLocalizedString("History")); optionQueue.setText(Utils.getLocalizedString("queue")); optionConfiguration.setText(Utils.getLocalizedString("Configuration")); + ripButton.setText(Utils.getLocalizedString("Rip")); + stopButton.setText(Utils.getLocalizedString("Stop")); + panicButton.setText(Utils.getLocalizedString("Panic")); } private void setupHandlers() { @@ -951,9 +954,8 @@ private void update() { queueListModel.add(0, ripper.getURL().toString()); stopButton.setEnabled(false); pack(); - //status(Utils.getLocalizedString("download.interrupted")); - status("Rip gracefully stopping"); - appendLog("Download interrupted", Color.RED); + status(Utils.getLocalizedString("rip.gracefully.stopping")); + appendLog(Utils.getLocalizedString("download.interrupted"), Color.RED); } }); @@ -968,8 +970,8 @@ private void update() { currentlyRippingProgress.setValue(0); currentlyRippingProgress.setText(""); pack(); - status("Rip interrupted"); // TODO localize - appendLog("Download interrupted", Color.RED); + status(Utils.getLocalizedString("rip.interrupted")); + appendLog(Utils.getLocalizedString("download.interrupted"), Color.RED); } }); @@ -1543,7 +1545,7 @@ private Thread ripAlbum(String urlString) { } urlString = urlString.trim(); LOGGER.info("Attempting to start rip for album {}", urlString); - appendLog("Attempting to start rip for album " + urlString, Color.GREEN); + appendLog(Utils.getLocalizedString("attempting.to.start.rip.for.album.0", urlString), Color.GREEN); if (urlString.toLowerCase().startsWith("gonewild:")) { urlString = "http://gonewild.com/user/" + urlString.substring(urlString.indexOf(':') + 1); } diff --git a/src/main/java/com/rarchives/ripme/utils/Utils.java b/src/main/java/com/rarchives/ripme/utils/Utils.java index 38684bdcf..802e28577 100644 --- a/src/main/java/com/rarchives/ripme/utils/Utils.java +++ b/src/main/java/com/rarchives/ripme/utils/Utils.java @@ -16,6 +16,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -819,9 +820,16 @@ public static String[] getSupportedLanguages() { } public static String getLocalizedString(String key) { - LOGGER.trace(String.format("Key %s in %s is: %s", key, getSelectedLanguage(), - resourceBundle.getString(key))); - return resourceBundle.getString(key); + String message = resourceBundle.getString(key); + LOGGER.trace("Key {} in {} is: {}", key, getSelectedLanguage(), message); + return message; + } + + @SuppressWarnings({"JavaExistingMethodCanBeUsed", "LoggingSimilarMessage"}) + public static String getLocalizedString(String key, Object... args) { + String pattern = resourceBundle.getString(key); + LOGGER.trace("Key {} in {} is: {}", key, getSelectedLanguage(), pattern); + return MessageFormat.format(pattern, args); } /** diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index 936448693..c04a4401c 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -1,3 +1,6 @@ +Rip = Rip +Stop = Stop +Panic = Panic! Log = Log History = History created = created @@ -62,6 +65,8 @@ inactive = Inactive download.url.list = Download url list select.save.dir = Select Save Directory +attempting.to.start.rip.for.album.0 = Attempting to start rip for album {0} + # Keys for the logs generated by DownloadFileThread nonretriable.status.code = Non-retriable status code retriable.status.code = Retriable status code @@ -72,10 +77,18 @@ magic.number.was = Magic number was deleting.existing.file = Deleting existing file request.properties = Request properties download.interrupted = Download interrupted +rip.interrupted = Rip interrupted +rip.gracefully.stopping = Rip gracefully stopping exceeded.maximum.retries = Exceeded maximum retries http.status.exception = HTTP status exception exception.while.downloading.file = Exception while downloading file failed.to.download = Failed to download skipping = Skipping file.already.exists = file already exists +skipping.ignored.extension = Skipping ignored extension + +failed.to.get.url.for.0 = Failed to get URL for {0} +0.already.saved.as.1 = {0} already saved as {1} +error.creating.directory = Error creating directory +0.while.downloading.1 = {0} while downloading {1} no.space.left.on.device = No space left on device From c26801056c21c31ace078faf3a2a8e2ec11f2bf1 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:49:09 -0400 Subject: [PATCH 74/78] Localize status labels --- .../com/rarchives/ripme/ui/MainWindow.java | 22 +++++++++++++------ src/main/resources/LabelsBundle.properties | 9 ++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index 29e383726..fe5eb09a4 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -65,11 +65,17 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JLabel statusLabel; private static final ProgressTextField currentlyRippingProgress = new ProgressTextField(); private static final JLabel pendingValue = new JLabel("0"); + private static final JLabel pendingLabel = new JLabel("Pending"); private static final JLabel activeValue = new JLabel("0"); + private static final JLabel activeLabel = new JLabel("Active"); private static final JLabel completedValue = new JLabel("0"); + private static final JLabel completedLabel = new JLabel("Completed"); private static final JLabel erroredValue = new JLabel("0"); + private static final JLabel erroredLabel = new JLabel("Errored"); private static final JLabel totalValue = new JLabel("0"); + private static final JLabel totalLabel = new JLabel("Total"); private static final JLabel transferRateValue = new JLabel("0.00 B/s"); + private static final JLabel transferRateLabel = new JLabel("Speed"); private static final JButton openButton = new JButton(); // Put an empty JPanel on the bottom of the window to keep components @@ -407,7 +413,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridy = 0; gbc.anchor = GridBagConstraints.EAST; statusDetailPanel.add(pendingValue, gbc); - JLabel pendingLabel = new JLabel("Pending"); gbc = new GridBagConstraints(); gbc.gridx = 1; gbc.gridy = 0; @@ -425,7 +430,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridy = 1; gbc.anchor = GridBagConstraints.EAST; statusDetailPanel.add(activeValue, gbc); - JLabel activeLabel = new JLabel("Active"); gbc = new GridBagConstraints(); gbc.gridx = 1; gbc.gridy = 1; @@ -443,7 +447,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridy = 0; gbc.anchor = GridBagConstraints.EAST; statusDetailPanel.add(completedValue, gbc); - JLabel completedLabel = new JLabel("Completed"); gbc = new GridBagConstraints(); gbc.gridx = 4; gbc.gridy = 0; @@ -461,7 +464,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridy = 1; gbc.anchor = GridBagConstraints.EAST; statusDetailPanel.add(erroredValue, gbc); - JLabel erroredLabel = new JLabel("Errored"); gbc = new GridBagConstraints(); gbc.gridx = 4; gbc.gridy = 1; @@ -479,7 +481,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridy = 0; gbc.anchor = GridBagConstraints.EAST; statusDetailPanel.add(totalValue, gbc); - JLabel totalLabel = new JLabel("Total"); gbc = new GridBagConstraints(); gbc.gridx = 7; gbc.gridy = 0; @@ -498,12 +499,11 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.gridy = 1; gbc.anchor = GridBagConstraints.EAST; statusDetailPanel.add(transferRateValue, gbc); - JLabel transferRateLabel2 = new JLabel("Speed"); gbc = new GridBagConstraints(); gbc.gridx = 7; gbc.gridy = 1; gbc.anchor = GridBagConstraints.WEST; - statusDetailPanel.add(transferRateLabel2, gbc); + statusDetailPanel.add(transferRateLabel, gbc); final JPanel spacer1 = new JPanel(); gbc = new GridBagConstraints(); @@ -906,6 +906,14 @@ private void changeLocale() { ripButton.setText(Utils.getLocalizedString("Rip")); stopButton.setText(Utils.getLocalizedString("Stop")); panicButton.setText(Utils.getLocalizedString("Panic")); + + pendingLabel.setText(Utils.getLocalizedString("Pending")); + activeLabel.setText(Utils.getLocalizedString("Active")); + completedLabel.setText(Utils.getLocalizedString("Completed")); + erroredLabel.setText(Utils.getLocalizedString("Errored")); + totalLabel.setText(Utils.getLocalizedString("Total")); + transferRateLabel.setText(Utils.getLocalizedString("Speed")); + } private void setupHandlers() { diff --git a/src/main/resources/LabelsBundle.properties b/src/main/resources/LabelsBundle.properties index c04a4401c..8648ca91b 100644 --- a/src/main/resources/LabelsBundle.properties +++ b/src/main/resources/LabelsBundle.properties @@ -1,3 +1,4 @@ +# Buttons Rip = Rip Stop = Stop Panic = Panic! @@ -9,6 +10,14 @@ queue = Queue Configuration = Configuration open = Open +# Status labels +Pending = Pending +Active = Active +Completed = Completed +Errored = Errored +Total = Total +Speed = Speed + # Keys for the Configuration menu current.version = Current version check.for.updates = Check for updates From fc879383e4fbe612657ff03bb207bf39b0384f56 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:56:56 -0400 Subject: [PATCH 75/78] Update tab button preferred size on locale change --- .../com/rarchives/ripme/ui/MainWindow.java | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index fe5eb09a4..faf3c24cf 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -570,18 +570,7 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib LOGGER.warn(e.getMessage()); } - // Prevent button sizes/positions from shifting when text bolds/unbolds - optionLog.setPreferredSize(optionLog.getPreferredSize()); - optionHistory.setPreferredSize(optionHistory.getPreferredSize()); - - // Set preferred size with space for queue size string - optionQueue.setText(Utils.getLocalizedString("queue") + " (8888)"); - optionQueue.setPreferredSize(optionQueue.getPreferredSize()); - // Restore original text - optionQueue.setText(Utils.getLocalizedString("queue")); - // updateQueue() is called below, which initializes the real queue button text - - optionConfiguration.setPreferredSize(optionConfiguration.getPreferredSize()); + setTabButtonPreferredSizes(); gbc.gridx = 0; optionsPanel.add(optionLog, gbc); @@ -834,6 +823,23 @@ public void setValueAt(Object value, int row, int col) { pane.add(emptyPanel, gbc); } + private static void setTabButtonPreferredSizes() { + // Recalculate preferred size each time by using getUI() + + // Prevent button sizes/positions from shifting when text bolds/unbolds + optionLog.setPreferredSize(optionLog.getUI().getPreferredSize(optionLog)); + optionHistory.setPreferredSize(optionHistory.getUI().getPreferredSize(optionHistory)); + + // Set preferred size with space for queue size string + optionQueue.setText(Utils.getLocalizedString("queue") + " (8888)"); + optionQueue.setPreferredSize(optionQueue.getUI().getPreferredSize(optionQueue)); + // Restore original text + optionQueue.setText(Utils.getLocalizedString("queue")); + // updateQueue() is called below, which initializes the real queue button text + + optionConfiguration.setPreferredSize(optionConfiguration.getUI().getPreferredSize(optionConfiguration)); + } + private JTextField configField(String key, int defaultValue) { final var field = new JTextField(Integer.toString(Utils.getConfigInteger(key, defaultValue))); field.getDocument().addDocumentListener(new DocumentListener() { @@ -914,6 +920,7 @@ private void changeLocale() { totalLabel.setText(Utils.getLocalizedString("Total")); transferRateLabel.setText(Utils.getLocalizedString("Speed")); + setTabButtonPreferredSizes(); // Preferred size may change with different width labels } private void setupHandlers() { From 42afdccd89de91c435c1bfda3fcc57de2aee0cb2 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:57:26 -0400 Subject: [PATCH 76/78] Open https URL --- src/main/java/com/rarchives/ripme/ui/MainWindow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index faf3c24cf..a768d0c60 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1298,7 +1298,7 @@ public void windowIconified(WindowEvent e) { JOptionPane.YES_NO_OPTION, JOptionPane.PLAIN_MESSAGE, new ImageIcon(mainIcon)); if (response == JOptionPane.YES_OPTION) { try { - Utils.browse(URI.create("http://github.com/ripmeapp/ripme")); + Utils.browse(URI.create("https://github.com/ripmeapp/ripme")); } catch (IOException e) { LOGGER.error("Exception while opening project home page", e); } From f566bb6dcb9948208c85b0caece09104848148d6 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Fri, 15 Aug 2025 14:35:19 -0400 Subject: [PATCH 77/78] Fix minimum label width GridBagLayout does not respect minimum size, only preferred size. GridBagLayout also treats preferred size as the maximum size. In order to set a minimum width, while allowing a label to grow if text overflows, the only solution is to override getPreferredSize dynamically. --- .../com/rarchives/ripme/ui/MainWindow.java | 39 +++---------------- .../rarchives/ripme/ui/MinimumWidthLabel.java | 33 ++++++++++++++++ 2 files changed, 39 insertions(+), 33 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index a768d0c60..f421d75c6 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -64,17 +64,17 @@ public final class MainWindow implements Runnable, RipStatusHandler { private static JLabel statusLabel; private static final ProgressTextField currentlyRippingProgress = new ProgressTextField(); - private static final JLabel pendingValue = new JLabel("0"); + private static final JLabel pendingValue = new MinimumWidthLabel("1000", "0"); private static final JLabel pendingLabel = new JLabel("Pending"); - private static final JLabel activeValue = new JLabel("0"); + private static final JLabel activeValue = new MinimumWidthLabel("1000", "0"); private static final JLabel activeLabel = new JLabel("Active"); - private static final JLabel completedValue = new JLabel("0"); + private static final JLabel completedValue = new MinimumWidthLabel("1000", "0"); private static final JLabel completedLabel = new JLabel("Completed"); - private static final JLabel erroredValue = new JLabel("0"); + private static final JLabel erroredValue = new MinimumWidthLabel("1000", "0"); private static final JLabel erroredLabel = new JLabel("Errored"); - private static final JLabel totalValue = new JLabel("0"); + private static final JLabel totalValue = new MinimumWidthLabel("1000", "0"); private static final JLabel totalLabel = new JLabel("Total"); - private static final JLabel transferRateValue = new JLabel("0.00 B/s"); + private static final JLabel transferRateValue = new MinimumWidthLabel("999.00 KiB/s", "0.00 B/s"); private static final JLabel transferRateLabel = new JLabel("Speed"); private static final JButton openButton = new JButton(); @@ -404,10 +404,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib pendingValue.setFont(monospaced); pendingValue.setHorizontalAlignment(JLabel.TRAILING); pendingValue.setBorder(valueLabelBorder); - pendingValue.setText("1000"); - pendingValue.setPreferredSize(pendingValue.getPreferredSize()); - pendingValue.setMinimumSize(pendingValue.getPreferredSize()); - pendingValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 0; gbc.gridy = 0; @@ -421,10 +417,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib activeValue.setFont(monospaced); activeValue.setHorizontalAlignment(JLabel.TRAILING); activeValue.setBorder(valueLabelBorder); - activeValue.setText("1000"); - activeValue.setPreferredSize(activeValue.getPreferredSize()); - activeValue.setMinimumSize(activeValue.getPreferredSize()); - activeValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 0; gbc.gridy = 1; @@ -438,10 +430,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib completedValue.setFont(monospaced); completedValue.setHorizontalAlignment(JLabel.TRAILING); completedValue.setBorder(valueLabelBorder); - completedValue.setText("1000"); - completedValue.setPreferredSize(completedValue.getPreferredSize()); - completedValue.setMinimumSize(completedValue.getPreferredSize()); - completedValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 3; gbc.gridy = 0; @@ -455,10 +443,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib erroredValue.setFont(monospaced); erroredValue.setHorizontalAlignment(JLabel.TRAILING); erroredValue.setBorder(valueLabelBorder); - erroredValue.setText("1000"); - erroredValue.setPreferredSize(erroredValue.getPreferredSize()); - erroredValue.setMinimumSize(erroredValue.getPreferredSize()); - erroredValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 3; gbc.gridy = 1; @@ -472,10 +456,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib totalValue.setFont(monospaced); totalValue.setHorizontalAlignment(JLabel.TRAILING); totalValue.setBorder(valueLabelBorder); - totalValue.setText("1000"); - totalValue.setPreferredSize(totalValue.getPreferredSize()); - totalValue.setMinimumSize(totalValue.getPreferredSize()); - totalValue.setText("0"); gbc = new GridBagConstraints(); gbc.gridx = 6; gbc.gridy = 0; @@ -489,11 +469,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib transferRateValue.setFont(monospaced); transferRateValue.setHorizontalAlignment(JLabel.TRAILING); transferRateValue.setBorder(valueLabelBorder); - transferRateValue.setText("999.00 KiB/s"); // Maximum width value - // Set preferred size to maximum width value - transferRateValue.setPreferredSize(transferRateValue.getPreferredSize()); - transferRateValue.setMinimumSize(transferRateValue.getPreferredSize()); - transferRateValue.setText("0 B/s"); // Restore default value gbc = new GridBagConstraints(); gbc.gridx = 6; gbc.gridy = 1; @@ -520,8 +495,6 @@ public void replace(FilterBypass fb, int offset, int length, String text, Attrib gbc.fill = GridBagConstraints.HORIZONTAL; statusDetailPanel.add(spacer2, gbc); - statusDetailPanel.setPreferredSize(new Dimension(350, statusDetailPanel.getPreferredSize().height)); - openButton.setVisible(false); gbc = new GridBagConstraints(); diff --git a/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java b/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java new file mode 100644 index 000000000..b2370167b --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java @@ -0,0 +1,33 @@ +package com.rarchives.ripme.ui; + +import javax.swing.JLabel; +import java.awt.Dimension; + +/** + * GridBagLayout does not respect minimum size, only preferred size. + * In order to set a minimum width, we need to override getPreferredSize. + */ +public class MinimumWidthLabel extends JLabel { + private String minimumWidthText; + + public MinimumWidthLabel(String minimumWidthText, String defaultText) { + this.minimumWidthText = minimumWidthText; + setText(defaultText); + } + + @Override + public Dimension getPreferredSize() { + String text = getText(); + Dimension preferredSize = super.getPreferredSize(); + setText(minimumWidthText); + int minimumWidth = super.getPreferredSize().width; + // Hopefully GridBagLayout correctly handles maximum size and we don't need to clamp + preferredSize.width = Math.max(minimumWidth, preferredSize.width); + setText(text); + return preferredSize; + } + + public void setMinimumWidthText(String minimumWidthText) { + this.minimumWidthText = minimumWidthText; + } +} From 852027edbd881be935f1f8bcb73a15ce752744c5 Mon Sep 17 00:00:00 2001 From: iqqu <33103874+iqqu@users.noreply.github.com> Date: Thu, 11 Sep 2025 18:54:10 -0400 Subject: [PATCH 78/78] Work around approximate active status --- .../ripme/ripper/DownloadThreadPool.java | 3 +++ .../java/com/rarchives/ripme/ui/MainWindow.java | 17 ++++++++++++++++- .../rarchives/ripme/ui/MinimumWidthLabel.java | 12 ++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java index 77551cb9b..08aa615c1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java +++ b/src/main/java/com/rarchives/ripme/ripper/DownloadThreadPool.java @@ -108,6 +108,9 @@ public int getPendingThreadCount() { return threadPool.getQueue().size(); } + /** + * @return The approximate active thread count + */ public int getActiveThreadCount() { return threadPool.getActiveCount(); } diff --git a/src/main/java/com/rarchives/ripme/ui/MainWindow.java b/src/main/java/com/rarchives/ripme/ui/MainWindow.java index f421d75c6..5566079d9 100644 --- a/src/main/java/com/rarchives/ripme/ui/MainWindow.java +++ b/src/main/java/com/rarchives/ripme/ui/MainWindow.java @@ -1703,6 +1703,8 @@ public void run() { } } + private long lastStatusUpdate = 0; + private synchronized void handleEvent(StatusEvent evt) { RipStatusMessage msg = evt.msg; RipStatusMessage.STATUS status = msg.getStatus(); @@ -1711,7 +1713,15 @@ private synchronized void handleEvent(StatusEvent evt) { if (status == RipStatusMessage.STATUS.CHUNK_BYTES) { transferRate.addChunk((Long) msg.getObject()); transferRateValue.setText(transferRate.formatHumanTransferRate()); - return; + + // Quick hack: ripper.getActiveCount() and dependent values are approximate and the value can be outdated (too large) when it is called when the ripper notifies COMPLETE, + // so allow the status info to update too, but throttle it a little. + long now = System.currentTimeMillis(); + boolean allowStatusUpdate = now > lastStatusUpdate + 200; + if (!allowStatusUpdate) { + return; + } + lastStatusUpdate = now; } if (evt.ripper.useByteProgessBar()) { @@ -1735,6 +1745,11 @@ private synchronized void handleEvent(StatusEvent evt) { currentlyRippingProgress.setValue(evt.ripper.getCompletionPercentage()); } + // Quick hack finish: + if (status == RipStatusMessage.STATUS.CHUNK_BYTES) { + return; + } + switch (status) { case LOADING_RESOURCE: case DOWNLOAD_STARTED: diff --git a/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java b/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java index b2370167b..0cae84c07 100644 --- a/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java +++ b/src/main/java/com/rarchives/ripme/ui/MinimumWidthLabel.java @@ -2,6 +2,7 @@ import javax.swing.JLabel; import java.awt.Dimension; +import java.util.Objects; /** * GridBagLayout does not respect minimum size, only preferred size. @@ -9,6 +10,7 @@ */ public class MinimumWidthLabel extends JLabel { private String minimumWidthText; + private String currentText; public MinimumWidthLabel(String minimumWidthText, String defaultText) { this.minimumWidthText = minimumWidthText; @@ -30,4 +32,14 @@ public Dimension getPreferredSize() { public void setMinimumWidthText(String minimumWidthText) { this.minimumWidthText = minimumWidthText; } + + @Override + public void setText(String text) { + // Cache the last text to save cycles, + // because this may be called with the same value very often + if (!Objects.equals(currentText, text)) { + currentText = text; + super.setText(text); + } + } }