From 7045c6867a7bbbc7c5e72a7d5b21a23db0b5b8ec Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 11 Nov 2025 04:04:57 +0000 Subject: [PATCH 01/10] Add logging functionality to extractor and add some logging Set ConsoleLogger globally for all tests --- .../schabi/newpipe/extractor/Extractor.java | 5 ++ .../org/schabi/newpipe/extractor/Info.java | 3 + .../org/schabi/newpipe/extractor/NewPipe.java | 6 ++ .../newpipe/extractor/stream/StreamInfo.java | 21 ++++- .../extractor/utils/ExtractorLogger.java | 87 +++++++++++++++++++ .../newpipe/extractor/utils/Logger.java | 10 +++ .../newpipe/extractor/LoggerExtension.java | 16 ++++ .../org.junit.jupiter.api.extension.Extension | 1 + .../test/resources/junit-platform.properties | 1 + 9 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/Logger.java create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/LoggerExtension.java create mode 100644 extractor/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension create mode 100644 extractor/src/test/resources/junit-platform.properties diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index e973b44167..cc75c8e7ec 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -7,6 +7,7 @@ import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; import org.schabi.newpipe.extractor.localization.TimeAgoParser; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -15,6 +16,8 @@ import java.util.Objects; public abstract class Extractor { + private final String TAG = getClass().getSimpleName() + "@" + hashCode(); + /** * {@link StreamingService} currently related to this extractor.
* Useful for getting other things from a service (like the url handlers for @@ -54,7 +57,9 @@ public LinkHandler getLinkHandler() { * @throws ExtractionException if the pages content is not understood */ public void fetchPage() throws IOException, ExtractionException { + ExtractorLogger.d(TAG, "base fetchPage called"); if (pageFetched) { + ExtractorLogger.d(TAG, "Page already fetched"); return; } onFetchPage(downloader); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java index 78a15553b1..0bcfeb559a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java @@ -2,6 +2,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.linkhandler.LinkHandler; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; import java.io.Serializable; import java.util.ArrayList; @@ -10,6 +11,7 @@ public abstract class Info implements Serializable { + private static final String TAG = "Info"; private final int serviceId; /** * Id of this Info object
@@ -52,6 +54,7 @@ public Info(final int serviceId, this.url = url; this.originalUrl = originalUrl; this.name = name; + ExtractorLogger.d(TAG, "Base Created " + this); } public Info(final int serviceId, final LinkHandler linkHandler, final String name) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index 7dfa4c4cde..0fd06872b4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -24,6 +24,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; import java.util.List; @@ -34,6 +35,7 @@ * Provides access to streaming services supported by NewPipe. */ public final class NewPipe { + private static final String TAG = NewPipe.class.getSimpleName(); private static Downloader downloader; private static Localization preferredLocalization; private static ContentCountry preferredContentCountry; @@ -42,15 +44,19 @@ private NewPipe() { } public static void init(final Downloader d) { + ExtractorLogger.d(TAG, "Default init called"); init(d, Localization.DEFAULT); } public static void init(final Downloader d, final Localization l) { + ExtractorLogger.d(TAG, "Default init called with localization"); init(d, l, l.getCountryCode().isEmpty() ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode())); } public static void init(final Downloader d, final Localization l, final ContentCountry c) { + ExtractorLogger.d(TAG, "Initializing with downloader: " + + d.getClass().getSimpleName() + ", " + l + ", " + c); downloader = d; preferredLocalization = l; preferredContentCountry = c; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 62fb6bbf74..f29b5b73a0 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -31,6 +31,7 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.localization.DateWrapper; import org.schabi.newpipe.extractor.utils.ExtractorHelper; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; import java.io.IOException; import java.util.List; @@ -44,7 +45,7 @@ * Info object for opened contents, i.e. the content ready to play. */ public class StreamInfo extends Info { - + private static final String TAG = StreamInfo.class.getSimpleName(); public static class StreamExtractException extends ExtractionException { StreamExtractException(final String message) { super(message); @@ -61,19 +62,37 @@ public StreamInfo(final int serviceId, super(serviceId, id, url, originalUrl, name); this.streamType = streamType; this.ageLimit = ageLimit; + ExtractorLogger.d(TAG, "Created " + this); + + } + + @Override + public String toString() { + return TAG + "[" + + "serviceId=" + getServiceId() + + ", url='" + getUrl() + '\'' + + ", originalUrl='" + getOriginalUrl() + '\'' + + ", id='" + getId() + '\'' + + ", name='" + getName() + '\'' + + ", streamType=" + streamType + + ", ageLimit=" + ageLimit + + ']'; } public static StreamInfo getInfo(final String url) throws IOException, ExtractionException { + ExtractorLogger.d(TAG, "getInfo(" + url + ")"); return getInfo(NewPipe.getServiceByUrl(url), url); } public static StreamInfo getInfo(@Nonnull final StreamingService service, final String url) throws IOException, ExtractionException { + ExtractorLogger.d(TAG, "getInfo(" + service.getClass().getSimpleName() + ", " + url + ")"); return getInfo(service.getStreamExtractor(url)); } public static StreamInfo getInfo(@Nonnull final StreamExtractor extractor) throws ExtractionException, IOException { + ExtractorLogger.d(TAG, "getInfo(" + extractor.getClass().getSimpleName() + ")"); extractor.fetchPage(); final StreamInfo streamInfo; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java new file mode 100644 index 0000000000..6894b6f5c2 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java @@ -0,0 +1,87 @@ +package org.schabi.newpipe.extractor.utils; + +public final class ExtractorLogger { + + private ExtractorLogger() { } + + private static Logger logger = new EmptyLogger(); + + public static void setLogger(final Logger customLogger) { + logger = customLogger; + } + + public static void d(final String tag, final String msg) { + logger.debug(tag, msg); + } + + public static void d(final String tag, final String msg, final Throwable t) { + logger.debug(tag, msg, t); + } + + public static void w(final String tag, final String msg) { + logger.warn(tag, msg); + } + + public static void w(final String tag, final String msg, final Throwable t) { + logger.warn(tag, msg, t); + } + + public static void e(final String tag, final String msg) { + logger.error(tag, msg); + } + + public static void e(final String tag, final String msg, final Throwable t) { + logger.error(tag, msg, t); + } + + + private static final class EmptyLogger implements Logger { + public void debug(final String tag, final String msg) { } + + @Override + public void debug(final String tag, final String msg, final Throwable throwable) { } + + public void warn(final String tag, final String msg) { } + + @Override + public void warn(final String tag, final String msg, final Throwable t) { } + + public void error(final String tag, final String msg) { } + + public void error(final String tag, final String msg, final Throwable t) { } + } + + /** + * Logger that prints to stdout + */ + public static final class ConsoleLogger implements Logger { + public void debug(final String tag, final String msg) { + System.out.println("[DEBUG][" + tag + "] " + msg); + } + + @Override + public void debug(final String tag, final String msg, final Throwable throwable) { + debug(tag, msg); + throwable.printStackTrace(System.err); + } + + public void warn(final String tag, final String msg) { + System.out.println("[WARN ][" + tag + "] " + msg); + } + + @Override + public void warn(final String tag, final String msg, final Throwable t) { + warn(tag, msg); + t.printStackTrace(System.err); + } + + public void error(final String tag, final String msg) { + System.err.println("[ERROR][" + tag + "] " + msg); + } + + public void error(final String tag, final String msg, final Throwable t) { + System.err.println("[ERROR][" + tag + "] " + msg); + t.printStackTrace(System.err); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Logger.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Logger.java new file mode 100644 index 0000000000..c7cd9db49c --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Logger.java @@ -0,0 +1,10 @@ +package org.schabi.newpipe.extractor.utils; + +public interface Logger { + void debug(String tag, String message); + void debug(String tag, String message, Throwable throwable); + void warn(String tag, String message); + void warn(String tag, String message, Throwable throwable); + void error(String tag, String message); + void error(String tag, String message, Throwable t); +} diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/LoggerExtension.java b/extractor/src/test/java/org/schabi/newpipe/extractor/LoggerExtension.java new file mode 100644 index 0000000000..7ef509cebc --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/LoggerExtension.java @@ -0,0 +1,16 @@ +package org.schabi.newpipe.extractor; + +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; + +public class LoggerExtension implements BeforeAllCallback { + private static boolean set = false; + + @Override + public void beforeAll(ExtensionContext context) { + if (set) return; + set = true; + ExtractorLogger.setLogger(new ExtractorLogger.ConsoleLogger()); + } +} \ No newline at end of file diff --git a/extractor/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/extractor/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension new file mode 100644 index 0000000000..ad03392dbd --- /dev/null +++ b/extractor/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension @@ -0,0 +1 @@ +org.schabi.newpipe.extractor.LoggerExtension \ No newline at end of file diff --git a/extractor/src/test/resources/junit-platform.properties b/extractor/src/test/resources/junit-platform.properties new file mode 100644 index 0000000000..1cebb76d5a --- /dev/null +++ b/extractor/src/test/resources/junit-platform.properties @@ -0,0 +1 @@ +junit.jupiter.extensions.autodetection.enabled = true \ No newline at end of file From d879c70d0fd93bd59401a5b61d32f354f6289cb3 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 18 Nov 2025 05:50:07 +0000 Subject: [PATCH 02/10] Fix null exception in tests Add string formatting to logger Add tests for logger formatting --- .../schabi/newpipe/extractor/Extractor.java | 5 + .../org/schabi/newpipe/extractor/NewPipe.java | 4 +- .../extractor/downloader/Downloader.java | 5 + .../newpipe/extractor/stream/StreamInfo.java | 8 +- .../extractor/utils/ExtractorLogger.java | 148 +++++++++++++++--- .../extractor/utils/ExtractorLoggerTest.java | 144 +++++++++++++++++ 6 files changed, 286 insertions(+), 28 deletions(-) create mode 100644 extractor/src/test/java/org/schabi/newpipe/extractor/utils/ExtractorLoggerTest.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index cc75c8e7ec..1daf0b86c7 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -156,4 +156,9 @@ public ContentCountry getExtractorContentCountry() { public TimeAgoParser getTimeAgoParser() { return getService().getTimeAgoParser(getExtractorLocalization()); } + + @Override + public String toString() { + return getClass().getSimpleName(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index 0fd06872b4..d7fea41bd1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -55,8 +55,8 @@ public static void init(final Downloader d, final Localization l) { } public static void init(final Downloader d, final Localization l, final ContentCountry c) { - ExtractorLogger.d(TAG, "Initializing with downloader: " - + d.getClass().getSimpleName() + ", " + l + ", " + c); + ExtractorLogger.d(TAG, "Initializing with downloader={}, localization={}, country={}", + d, l, c); downloader = d; preferredLocalization = l; preferredContentCountry = c; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java index aa7987156d..218848d0ac 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java @@ -243,4 +243,9 @@ public Response postWithContentTypeJson(final String url, */ public abstract Response execute(@Nonnull Request request) throws IOException, ReCaptchaException; + + @Override + public String toString() { + return getClass().getSimpleName(); + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index f29b5b73a0..aa0263bf7d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -62,7 +62,7 @@ public StreamInfo(final int serviceId, super(serviceId, id, url, originalUrl, name); this.streamType = streamType; this.ageLimit = ageLimit; - ExtractorLogger.d(TAG, "Created " + this); + ExtractorLogger.d(TAG, "Created {}", this); } @@ -80,19 +80,19 @@ public String toString() { } public static StreamInfo getInfo(final String url) throws IOException, ExtractionException { - ExtractorLogger.d(TAG, "getInfo(" + url + ")"); + ExtractorLogger.d(TAG, "getInfo({url})", url); return getInfo(NewPipe.getServiceByUrl(url), url); } public static StreamInfo getInfo(@Nonnull final StreamingService service, final String url) throws IOException, ExtractionException { - ExtractorLogger.d(TAG, "getInfo(" + service.getClass().getSimpleName() + ", " + url + ")"); + ExtractorLogger.d(TAG, "getInfo({service},{url})", service, url); return getInfo(service.getStreamExtractor(url)); } public static StreamInfo getInfo(@Nonnull final StreamExtractor extractor) throws ExtractionException, IOException { - ExtractorLogger.d(TAG, "getInfo(" + extractor.getClass().getSimpleName() + ")"); + ExtractorLogger.d(TAG, "getInfo({extractor)", extractor); extractor.fetchPage(); final StreamInfo streamInfo; try { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java index 6894b6f5c2..5cf56144e9 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java @@ -4,72 +4,176 @@ public final class ExtractorLogger { private ExtractorLogger() { } - private static Logger logger = new EmptyLogger(); + private static final Logger EMPTY_LOGGER = new EmptyLogger(); + private static volatile Logger logger = EMPTY_LOGGER; public static void setLogger(final Logger customLogger) { - logger = customLogger; + logger = customLogger != null ? customLogger : EMPTY_LOGGER; } + public enum Level { DEBUG, WARN, ERROR } + + @SuppressWarnings("checkstyle:NeedBraces") + private static void log(final Level level, + final String tag, + final String message, + final Throwable t) { + if (logger == EMPTY_LOGGER) return; + switch (level) { + case DEBUG: + if (t == null) { + logger.debug(tag, message); + } else { + logger.debug(tag, message, t); + } + break; + case WARN: + if (t == null) { + logger.warn(tag, message); + } else { + logger.warn(tag, message, t); + } + break; + case ERROR: + if (t == null) { + logger.error(tag, message); + } else { + logger.error(tag, message, t); + } + break; + } + } + + @SuppressWarnings("checkstyle:NeedBraces") + private static void logFormat(final Level level, + final String tag, + final Throwable t, + final String template, + final Object... args) { + if (logger == EMPTY_LOGGER) return; + log(level, tag, format(template, args), t); + } + + // DEBUG public static void d(final String tag, final String msg) { - logger.debug(tag, msg); + log(Level.DEBUG, tag, msg, null); } public static void d(final String tag, final String msg, final Throwable t) { - logger.debug(tag, msg, t); + log(Level.DEBUG, tag, msg, t); + } + + public static void d(final String tag, final String template, final Object... args) { + logFormat(Level.DEBUG, tag, null, template, args); } + public static void d(final String tag, + final Throwable t, + final String template, + final Object... args) { + logFormat(Level.DEBUG, tag, t, template, args); + } + + // WARN public static void w(final String tag, final String msg) { - logger.warn(tag, msg); + log(Level.WARN, tag, msg, null); } public static void w(final String tag, final String msg, final Throwable t) { - logger.warn(tag, msg, t); + log(Level.WARN, tag, msg, t); + } + + public static void w(final String tag, final String template, final Object... args) { + logFormat(Level.WARN, tag, null, template, args); } + public static void w(final String tag, + final Throwable t, + final String template, + final Object... args) { + logFormat(Level.WARN, tag, t, template, args); + } + + // ERROR public static void e(final String tag, final String msg) { - logger.error(tag, msg); + log(Level.ERROR, tag, msg, null); } public static void e(final String tag, final String msg, final Throwable t) { - logger.error(tag, msg, t); + log(Level.ERROR, tag, msg, t); + } + + public static void e(final String tag, final String template, final Object... args) { + logFormat(Level.ERROR, tag, null, template, args); } + public static void e(final String tag, + final Throwable t, + final String template, + final Object... args) { + logFormat(Level.ERROR, tag, t, template, args); + } + + /** + * Simple string format method for easier logger in the form of + * {@code ExtractorLogger.d("Hello my name {Name} {}", name, surname)} + * @param template The template string to format + * @param args Arguments to replace identifiers with in {@code template} + * @return Formatted string with arguments replaced + */ + private static String format(final String template, final Object... args) { + if (template == null || args == null || args.length == 0) { + return template; + } + final var out = new StringBuilder(template.length() + Math.min(32, 16 * args.length)); + int cursorIndex = 0; + int argIndex = 0; + final int n = template.length(); + while (cursorIndex < n) { + // Find first/next open brace + final int openBraceIndex = template.indexOf('{', cursorIndex); + if (openBraceIndex < 0) { + // If none found then there's no more arguments to replace + out.append(template, cursorIndex, n); break; + } + + // Find matching closing brace + final int close = template.indexOf('}', openBraceIndex + 1); + if (close < 0) { + // If none found then there's no more arguments to replace + out.append(template, cursorIndex, n); break; + } + out.append(template, cursorIndex, openBraceIndex); + out.append(argIndex < args.length + ? String.valueOf(args[argIndex++]) + : template.substring(openBraceIndex, close + 1)); + cursorIndex = close + 1; + } + return out.toString(); + } private static final class EmptyLogger implements Logger { public void debug(final String tag, final String msg) { } - - @Override public void debug(final String tag, final String msg, final Throwable throwable) { } - public void warn(final String tag, final String msg) { } - - @Override public void warn(final String tag, final String msg, final Throwable t) { } - public void error(final String tag, final String msg) { } - public void error(final String tag, final String msg, final Throwable t) { } } - /** - * Logger that prints to stdout - */ public static final class ConsoleLogger implements Logger { public void debug(final String tag, final String msg) { - System.out.println("[DEBUG][" + tag + "] " + msg); + System.out.println("[DEBUG][" + tag + "] " + msg); } - @Override public void debug(final String tag, final String msg, final Throwable throwable) { debug(tag, msg); throwable.printStackTrace(System.err); } - public void warn(final String tag, final String msg) { System.out.println("[WARN ][" + tag + "] " + msg); } - @Override public void warn(final String tag, final String msg, final Throwable t) { warn(tag, msg); t.printStackTrace(System.err); diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ExtractorLoggerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ExtractorLoggerTest.java new file mode 100644 index 0000000000..3530e696ab --- /dev/null +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ExtractorLoggerTest.java @@ -0,0 +1,144 @@ +package org.schabi.newpipe.extractor.utils; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class ExtractorLoggerTest { + + private CapturingLogger logger; + + @BeforeEach + void setup() { + logger = new CapturingLogger(); + ExtractorLogger.setLogger(logger); + } + + @Test + void replacesSinglePlaceholder() { + ExtractorLogger.d("T", "Hello {Name}", "Alice"); + assertEquals("Hello Alice", logger.lastDebug); + } + + @Test + void replacesMultiplePlaceholdersSequentially() { + ExtractorLogger.d("T", "A={A} B={B} C={C}", 1, 2, 3); + assertEquals("A=1 B=2 C=3", logger.lastDebug); + } + + @Test + void leavesExtraPlaceholdersWhenNotEnoughArgs() { + ExtractorLogger.d("T", "First={F} Second={S} Third={T}", "X", "Y"); + assertEquals("First=X Second=Y Third={T}", logger.lastDebug); + } + + @Test + void ignoresExtraArgs() { + ExtractorLogger.d("T", "Only {One}", "X", "Y", "Z"); + assertEquals("Only X", logger.lastDebug); + } + + @Test + void noArgsReturnsTemplateUnchanged() { + ExtractorLogger.d("T", "No placeholders {} here"); + assertEquals("No placeholders {} here", logger.lastDebug); + } + + @Test + void nullTemplatePrintsNull() { + ExtractorLogger.d("T", (String) null, "X"); + assertNull(logger.lastDebug); + } + + @Test + void unmatchedBraceLeavesRemainder() { + ExtractorLogger.d("T", "Value {Unclosed", "X"); + assertEquals("Value {Unclosed", logger.lastDebug); + } + + @Test + void debugFormatWithThrowable() { + RuntimeException ex = new RuntimeException("boom"); + ExtractorLogger.d("T", ex, "Failure {Code} at {Step}", 500, "init"); + assertEquals("Failure 500 at init", logger.lastDebug); + assertSame(ex, logger.lastDebugThrowable); + } + + @Test + void warnFormatWithThrowable() { + IllegalStateException ex = new IllegalStateException("warned"); + ExtractorLogger.w("T", ex, "Warn {What}", "disk"); + assertEquals("Warn disk", logger.lastWarn); + assertSame(ex, logger.lastWarnThrowable); + } + + @Test + void errorFormatWithThrowable() { + Exception ex = new Exception("fatal"); + ExtractorLogger.e("T", ex, "Error {Type} code={Code}", "IO", 42); + assertEquals("Error IO code=42", logger.lastError); + assertSame(ex, logger.lastErrorThrowable); + } + + @Test + void debugFormatWithThrowableNotEnoughArgsLeavesPlaceholder() { + RuntimeException ex = new RuntimeException("x"); + ExtractorLogger.d("T", ex, "Only one {A} and leftover {B}", "arg1"); + assertEquals("Only one arg1 and leftover {B}", logger.lastDebug); + assertSame(ex, logger.lastDebugThrowable); + } + + @Test + void errorFormatWithThrowableExtraArgsIgnored() { + Exception ex = new Exception("x"); + ExtractorLogger.e("T", ex, "Val {V}", 10, 20, 30); + assertEquals("Val 10", logger.lastError); + assertSame(ex, logger.lastErrorThrowable); + } + + private static final class CapturingLogger implements Logger { + String lastDebug; + Throwable lastDebugThrowable; + String lastWarn; + Throwable lastWarnThrowable; + String lastError; + Throwable lastErrorThrowable; + + @Override + public void debug(String tag, String message) { + lastDebug = message; + lastDebugThrowable = null; + } + + @Override + public void debug(String tag, String message, Throwable throwable) { + lastDebug = message; + lastDebugThrowable = throwable; + } + + @Override + public void warn(String tag, String message) { + lastWarn = message; + lastWarnThrowable = null; + } + + @Override + public void warn(String tag, String message, Throwable throwable) { + lastWarn = message; + lastWarnThrowable = throwable; + } + + @Override + public void error(String tag, String message) { + lastError = message; + lastErrorThrowable = null; + } + + @Override + public void error(String tag, String message, Throwable t) { + lastError = message; + lastErrorThrowable = t; + } + } +} \ No newline at end of file From 953675bd6a98eda611fe7e591743ff3d30649c9a Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 18 Nov 2025 06:49:08 +0000 Subject: [PATCH 03/10] Final edits --- .../src/main/java/org/schabi/newpipe/extractor/Extractor.java | 2 +- .../src/main/java/org/schabi/newpipe/extractor/Info.java | 2 +- .../src/main/java/org/schabi/newpipe/extractor/NewPipe.java | 2 +- .../java/org/schabi/newpipe/extractor/stream/StreamInfo.java | 1 - .../org/schabi/newpipe/extractor/utils/ExtractorLogger.java | 4 +++- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java index 1daf0b86c7..7d984e8338 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java @@ -59,7 +59,7 @@ public LinkHandler getLinkHandler() { public void fetchPage() throws IOException, ExtractionException { ExtractorLogger.d(TAG, "base fetchPage called"); if (pageFetched) { - ExtractorLogger.d(TAG, "Page already fetched"); + ExtractorLogger.d(TAG, "Page already fetched; returning"); return; } onFetchPage(downloader); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java b/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java index 0bcfeb559a..7ad5e8dd77 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/Info.java @@ -54,7 +54,7 @@ public Info(final int serviceId, this.url = url; this.originalUrl = originalUrl; this.name = name; - ExtractorLogger.d(TAG, "Base Created " + this); + ExtractorLogger.d(TAG, "Base Created {}", this); } public Info(final int serviceId, final LinkHandler linkHandler, final String name) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index d7fea41bd1..77fe1667a8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -49,7 +49,7 @@ public static void init(final Downloader d) { } public static void init(final Downloader d, final Localization l) { - ExtractorLogger.d(TAG, "Default init called with localization"); + ExtractorLogger.d(TAG, "Default init called with localization={}"); init(d, l, l.getCountryCode().isEmpty() ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode())); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index aa0263bf7d..412c27ea57 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -63,7 +63,6 @@ public StreamInfo(final int serviceId, this.streamType = streamType; this.ageLimit = ageLimit; ExtractorLogger.d(TAG, "Created {}", this); - } @Override diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java index 5cf56144e9..eb1557946c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java @@ -115,7 +115,7 @@ public static void e(final String tag, } /** - * Simple string format method for easier logger in the form of + * Simple string format method for easier logging in the form of * {@code ExtractorLogger.d("Hello my name {Name} {}", name, surname)} * @param template The template string to format * @param args Arguments to replace identifiers with in {@code template} @@ -143,7 +143,9 @@ private static String format(final String template, final Object... args) { // If none found then there's no more arguments to replace out.append(template, cursorIndex, n); break; } + // Append everything from cursor up to before the open brace out.append(template, cursorIndex, openBraceIndex); + // Append arguments in the brace out.append(argIndex < args.length ? String.valueOf(args[argIndex++]) : template.substring(openBraceIndex, close + 1)); From 348d8fb109cd3ab6046fec09795c40f76d7ff921 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 11 Nov 2025 04:04:25 +0000 Subject: [PATCH 04/10] [SoundCloud] Validate http response code in SoundcloudParsingHelper --- .../extractor/downloader/Response.java | 20 +++++++++++ .../exceptions/HttpResponseException.java | 15 ++++++++ .../soundcloud/SoundcloudParsingHelper.java | 34 +++++++++++------- .../newpipe/extractor/utils/HttpUtils.java | 36 +++++++++++++++++++ 4 files changed, 92 insertions(+), 13 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index ac792dc756..87c3577ef4 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -6,6 +6,9 @@ import java.util.List; import java.util.Map; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; +import org.schabi.newpipe.extractor.utils.HttpUtils; + /** * A Data class used to hold the results from requests made by the Downloader implementation. */ @@ -80,4 +83,21 @@ public String getHeader(final String name) { return null; } + // CHECKSTYLE:OFF + /** + * Helper function simply to make it easier to validate response code inline + * before getting the code/body/latestUrl/etc. + * Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid + * @see HttpUtils#validateResponseCode(Response, int...) + * @param validResponseCodes Expected valid response codes + * @return {@link this} response + * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, + * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. + */ + // CHECKSTYLE:ON + public Response validateResponseCode(final int... validResponseCodes) + throws HttpResponseException { + HttpUtils.validateResponseCode(this, validResponseCodes); + return this; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java new file mode 100644 index 0000000000..c07850a9d3 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java @@ -0,0 +1,15 @@ +package org.schabi.newpipe.extractor.exceptions; + +import java.io.IOException; +import org.schabi.newpipe.extractor.downloader.Response; + +public class HttpResponseException extends IOException { + public HttpResponseException(final Response response) { + this("Error in HTTP Response for " + response.latestUrl() + "\n\t" + + response.responseCode() + " - " + response.responseMessage()); + } + + public HttpResponseException(final String message) { + super(message); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index aeff6bd363..26856fc8c8 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -5,6 +5,7 @@ import static org.schabi.newpipe.extractor.ServiceList.SoundCloud; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps; +import static org.schabi.newpipe.extractor.utils.HttpUtils.validateResponseCode; import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; @@ -12,7 +13,6 @@ import com.grack.nanojson.JsonParserException; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; import org.schabi.newpipe.extractor.Image; @@ -105,8 +105,8 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final Downloader dl = NewPipe.getDownloader(); - final Response download = dl.get("https://soundcloud.com"); - final String responseBody = download.responseBody(); + final Response downloadResponse = dl.get("https://soundcloud.com").validateResponseCode(); + final String responseBody = downloadResponse.responseBody(); final String clientIdPattern = ",client_id:\"(.*?)\""; final Document doc = Jsoup.parse(responseBody); @@ -117,11 +117,12 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final var headers = Map.of("Range", List.of("bytes=0-50000")); - for (final Element element : possibleScripts) { + for (final var element : possibleScripts) { final String srcUrl = element.attr("src"); if (!isNullOrEmpty(srcUrl)) { try { clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers) + .validateResponseCode() .responseBody()); return clientId; } catch (final RegexException ignored) { @@ -149,11 +150,13 @@ public static DateWrapper parseDate(final String uploadDate) throws ParsingExcep } } + // CHECKSTYLE:OFF /** - * Call the endpoint "/resolve" of the API.

+ * Call the endpoint "/resolve" of the API. *

- * See https://developers.soundcloud.com/docs/api/reference#resolve + * See https://web.archive.org/web/20170804051146/https://developers.soundcloud.com/docs/api/reference#resolve */ + // CHECKSTYLE:ON public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) throws IOException, ExtractionException { final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve" @@ -178,10 +181,11 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException, ReCaptchaException { - final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" - + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()).responseBody(); - - return Jsoup.parse(response).select("link[rel=\"canonical\"]").first() + final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url=" + + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()); + validateResponseCode(response); + final var responseBody = response.responseBody(); + return Jsoup.parse(responseBody).select("link[rel=\"canonical\"]").first() .attr("abs:href"); } @@ -190,6 +194,7 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc * * @return the resolved id */ + // TODO: what makes this method different from the others? Don' they all return the same? public static String resolveIdWithWidgetApi(final String urlString) throws IOException, ParsingException { String fixedUrl = urlString; @@ -225,9 +230,12 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url=" + Utils.encodeUrlUtf8(url.toString()) + "&format=json&client_id=" + SoundcloudParsingHelper.clientId(); - final String response = NewPipe.getDownloader().get(widgetUrl, - SoundCloud.getLocalization()).responseBody(); - final JsonObject o = JsonParser.object().from(response); + + final var response = NewPipe.getDownloader().get(widgetUrl, + SoundCloud.getLocalization()); + + final var responseBody = response.validateResponseCode().responseBody(); + final JsonObject o = JsonParser.object().from(responseBody); return String.valueOf(JsonUtils.getValue(o, "id")); } catch (final JsonParserException e) { throw new ParsingException("Could not parse JSON response", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java new file mode 100644 index 0000000000..31c937ea09 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java @@ -0,0 +1,36 @@ +package org.schabi.newpipe.extractor.utils; + +import java.util.Arrays; + +import org.schabi.newpipe.extractor.downloader.Response; +import org.schabi.newpipe.extractor.exceptions.HttpResponseException; + +public final class HttpUtils { + + private HttpUtils() { + // Utility class, no instances allowed + } + + // CHECKSTYLE:OFF + /** + * Validates the response codes for the given {@link Response}, and throws + * a {@link HttpResponseException} if the code is invalid + * @param response The response to validate + * @param validResponseCodes Expected valid response codes + * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, + * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. + */ + // CHECKSTYLE:ON + public static void validateResponseCode(final Response response, + final int... validResponseCodes) + throws HttpResponseException { + final int code = response.responseCode(); + final var throwError = (validResponseCodes == null || validResponseCodes.length == 0) + ? code >= 400 && code <= 599 + : Arrays.stream(validResponseCodes).noneMatch(c -> c == code); + + if (throwError) { + throw new HttpResponseException(response); + } + } +} From 0d851d02dec2f165179c32cee7be780a92a1c742 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 11 Nov 2025 04:04:57 +0000 Subject: [PATCH 05/10] [SoundCloud] Add logging to SoundcloudParsingHelper, SoundcloudStreamExtractor, StreamInfo --- .../org/schabi/newpipe/extractor/NewPipe.java | 2 +- .../newpipe/extractor/StreamingServiceId.java | 23 +++++++++++++++++++ .../soundcloud/SoundcloudParsingHelper.java | 6 +++++ .../extractors/SoundcloudStreamExtractor.java | 9 +++++++- .../newpipe/extractor/stream/StreamInfo.java | 12 ++++++++++ 5 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/StreamingServiceId.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index 77fe1667a8..d7fea41bd1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -49,7 +49,7 @@ public static void init(final Downloader d) { } public static void init(final Downloader d, final Localization l) { - ExtractorLogger.d(TAG, "Default init called with localization={}"); + ExtractorLogger.d(TAG, "Default init called with localization"); init(d, l, l.getCountryCode().isEmpty() ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode())); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingServiceId.java b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingServiceId.java new file mode 100644 index 0000000000..5c40b5d5c0 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/StreamingServiceId.java @@ -0,0 +1,23 @@ +package org.schabi.newpipe.extractor; + +import java.util.Objects; + +public enum StreamingServiceId { + NO_SERVICE_ID, + YOUTUBE, + SOUNDCLOUD, + MEDIACCC, + PEERTUBE, + BANDCAMP; + + + private static final StreamingServiceId[] VALUES = values(); + + public static String nameFromId(final int serviceId) { + try { + return VALUES[Objects.checkIndex(serviceId + 1, VALUES.length)].name(); + } catch (final IndexOutOfBoundsException e) { + throw new IllegalArgumentException("Invalid serviceId: " + serviceId, e); + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 26856fc8c8..61a80e6648 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -29,6 +29,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudLikesInfoItemExtractor; import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; import org.schabi.newpipe.extractor.utils.ImageSuffix; import org.schabi.newpipe.extractor.utils.JsonUtils; import org.schabi.newpipe.extractor.utils.Parser; @@ -87,6 +88,7 @@ public final class SoundcloudParsingHelper { private static final List VISUALS_IMAGE_SUFFIXES = List.of(new ImageSuffix("t1240x260", 1240, 260, MEDIUM), new ImageSuffix("t2480x520", 2480, 520, MEDIUM)); + public static final String TAG = SoundcloudParsingHelper.class.getSimpleName(); private static String clientId; public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/"; @@ -100,6 +102,7 @@ private SoundcloudParsingHelper() { public static synchronized String clientId() throws ExtractionException, IOException { if (!isNullOrEmpty(clientId)) { + ExtractorLogger.d(TAG, "Returning clientId=" + clientId); return clientId; } @@ -121,9 +124,11 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final String srcUrl = element.attr("src"); if (!isNullOrEmpty(srcUrl)) { try { + ExtractorLogger.d(TAG, "Searching for clientId in " + srcUrl); clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers) .validateResponseCode() .responseBody()); + ExtractorLogger.d(TAG, "Found clientId=" + clientId); return clientId; } catch (final RegexException ignored) { // Ignore it and proceed to try searching other script @@ -159,6 +164,7 @@ public static DateWrapper parseDate(final String uploadDate) throws ParsingExcep // CHECKSTYLE:ON public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) throws IOException, ExtractionException { + ExtractorLogger.d(TAG, "resolveFor(" + url + ")"); final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve" + "?url=" + Utils.encodeUrlUtf8(url) + "&client_id=" + clientId(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 4de7114de6..78e8ea5c6c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -35,6 +35,7 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; import org.schabi.newpipe.extractor.stream.StreamType; import org.schabi.newpipe.extractor.stream.VideoStream; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; @@ -46,6 +47,7 @@ import javax.annotation.Nullable; public class SoundcloudStreamExtractor extends StreamExtractor { + public static final String TAG = SoundcloudStreamExtractor.class.getSimpleName(); private JsonObject track; private boolean isAvailable = true; @@ -57,9 +59,12 @@ public SoundcloudStreamExtractor(final StreamingService service, @Override public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { - track = SoundcloudParsingHelper.resolveFor(downloader, getUrl()); + final var url = getUrl(); + ExtractorLogger.d(TAG, "onFetchPage(" + url + ")"); + track = SoundcloudParsingHelper.resolveFor(downloader, url); final String policy = track.getString("policy", ""); + ExtractorLogger.d(TAG, "policy is: " + policy); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { isAvailable = false; @@ -164,6 +169,7 @@ public List getAudioStreams() throws ExtractionException { // For playing the track, it is only necessary to have a streamable track. // If this is not the case, this track might not be published yet. if (!track.getBoolean("streamable") || !isAvailable) { + ExtractorLogger.d(TAG, "Not streamable track: " + getUrl()); return audioStreams; } @@ -172,6 +178,7 @@ public List getAudioStreams() throws ExtractionException { .getArray("transcodings"); if (!isNullOrEmpty(transcodings)) { // Get information about what stream formats are available + ExtractorLogger.d(TAG, "Extracting audio streams for " + getName()); extractAudioStreams(transcodings, audioStreams); } } catch (final NullPointerException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 412c27ea57..7ecd6019d6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -186,6 +186,18 @@ private static void extractStreams(final StreamInfo streamInfo, // Either audio or video has to be available, otherwise we didn't get a stream (since // videoOnly are optional, they don't count). if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) { + final var errors = streamInfo.getErrors(); + final var url = streamInfo.getOriginalUrl(); + final var name = streamInfo.getName(); + if (errors.isEmpty()) { + ExtractorLogger.e(TAG, "Error extracting " + name + " " + url + + "\nCould not get any stream and didn't catch any errors"); + } else { + errors.forEach(m -> ExtractorLogger.e(TAG, + "Error for " + streamInfo.getOriginalUrl(), + m)); + } + throw new StreamExtractException( "Could not get any stream. See error variable to get further details."); } From 61534c8c7242a27cec341012e8aca1af51474e37 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Tue, 11 Nov 2025 04:05:13 +0000 Subject: [PATCH 06/10] [SoundCloud] Refactor Soundcloud audio stream extraction code to separate building Hls and Progressive streams Add HlsAudioStream to facilitate refreshing expired Hls playlists --- .../extractors/SoundcloudStreamExtractor.java | 166 ++++++++++++------ .../newpipe/extractor/stream/AudioStream.java | 4 +- .../extractor/stream/HlsAudioStream.java | 60 +++++++ .../extractor/stream/RefreshableStream.java | 15 ++ .../extractor/stream/SoundcloudHlsUtils.java | 90 ++++++++++ 5 files changed, 284 insertions(+), 51 deletions(-) create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/HlsAudioStream.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/RefreshableStream.java create mode 100644 extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index 78e8ea5c6c..eef01bc772 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -11,12 +11,9 @@ import com.grack.nanojson.JsonArray; import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; import org.schabi.newpipe.extractor.Image; import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException; @@ -30,6 +27,8 @@ import org.schabi.newpipe.extractor.stream.AudioStream; import org.schabi.newpipe.extractor.stream.DeliveryMethod; import org.schabi.newpipe.extractor.stream.Description; +import org.schabi.newpipe.extractor.stream.HlsAudioStream; +import org.schabi.newpipe.extractor.stream.SoundcloudHlsUtils; import org.schabi.newpipe.extractor.stream.Stream; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector; @@ -188,25 +187,106 @@ public List getAudioStreams() throws ExtractionException { return audioStreams; } - @Nonnull - private String getTranscodingUrl(final String endpointUrl) - throws IOException, ExtractionException { - String apiStreamUrl = endpointUrl + "?client_id=" + clientId(); + // TODO: put this somewhere better + /** + * Constructs the API endpoint url for this track that will be called to get the url + * for retrieving the actual byte data for playback + * (e.g. the actual url could be an m3u8 playlist) + * @param baseUrl The baseUrl needed to construct the full url + * @return The full API endpoint url to call to get the actual playback url + * @throws IOException If there is a problem getting clientId + * @throws ExtractionException For the same reason + */ + private String getApiStreamUrl(final String baseUrl) throws ExtractionException, IOException { + String apiStreamUrl = baseUrl + "?client_id=" + clientId(); final String trackAuthorization = track.getString("track_authorization"); if (!isNullOrEmpty(trackAuthorization)) { apiStreamUrl += "&track_authorization=" + trackAuthorization; } + return apiStreamUrl; + } - final String response = NewPipe.getDownloader().get(apiStreamUrl).responseBody(); - final JsonObject urlObject; - try { - urlObject = JsonParser.object().from(response); - } catch (final JsonParserException e) { - throw new ParsingException("Could not parse streamable URL", e); + public static final class StreamBuildResult { + public final String contentUrl; + public final MediaFormat mediaFormat; + + public StreamBuildResult(final String contentUrl, final MediaFormat mediaFormat) { + this.contentUrl = contentUrl; + this.mediaFormat = mediaFormat; + } + + @Override + public String toString() { + return "StreamBuildResult{" + + "contentUrl='" + contentUrl + '\'' + + ", mediaFormat=" + mediaFormat + + '}'; + } + } + + /** + * Builds the common audio stream components for all SoundCloud audio streams

+ * Returns the stream content url if we support this type of transcoding, {@code null} otherwise + * @param transcoding The SoundCloud JSON transcoding object for this stream + * @param builder AudioStream builder to set the common values + * @return the stream content url if this transcoding is supported and common values were built + * {@code null} otherwise + */ + @Nullable + private StreamBuildResult buildBaseAudioStream(final JsonObject transcoding, + final AudioStream.Builder builder) + throws ExtractionException, IOException { + ExtractorLogger.d(TAG, getName() + " Building base audio stream info"); + final var preset = transcoding.getString("preset", ID_UNKNOWN); + final MediaFormat mediaFormat; + if (preset.contains("mp3")) { + mediaFormat = MediaFormat.MP3; + builder.setAverageBitrate(128); + } else if (preset.contains("opus")) { + mediaFormat = MediaFormat.OPUS; + builder.setAverageBitrate(64); + builder.setDeliveryMethod(DeliveryMethod.HLS); + } else if (preset.contains("aac_160k")) { + mediaFormat = MediaFormat.M4A; + builder.setAverageBitrate(160); + } else { + // Unknown format, return null to skip to the next audio stream + return null; } - return urlObject.getString("url"); + builder.setMediaFormat(mediaFormat); + + builder.setId(preset); + final var url = transcoding.getString("url"); + final var hlsPlaylistUrl = SoundcloudHlsUtils.getStreamContentUrl(getApiStreamUrl(url)); + builder.setContent(hlsPlaylistUrl, true); + return new StreamBuildResult(hlsPlaylistUrl, mediaFormat); + } + + private HlsAudioStream buildHlsAudioStream(final JsonObject transcoding) + throws ExtractionException, IOException { + ExtractorLogger.d(TAG, getName() + "Extracting hls audio stream"); + final var builder = new HlsAudioStream.Builder(); + final StreamBuildResult buildResult = buildBaseAudioStream(transcoding, builder); + if (buildResult == null) { + return null; + } + + builder.setApiStreamUrl(getApiStreamUrl(transcoding.getString("url"))); + builder.setPlaylistId(SoundcloudHlsUtils.extractHlsPlaylistId(buildResult.contentUrl, + buildResult.mediaFormat)); + + return builder.build(); + } + + private AudioStream buildProgressiveAudioStream(final JsonObject transcoding) + throws ExtractionException, IOException { + ExtractorLogger.d(TAG, getName() + "Extracting progressive audio stream"); + final var builder = new AudioStream.Builder(); + final StreamBuildResult buildResult = buildBaseAudioStream(transcoding, builder); + return buildResult == null ? null : builder.build(); + // TODO: anything else? } private void extractAudioStreams(@Nonnull final JsonArray transcodings, @@ -220,47 +300,35 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings, return; } - try { - final String preset = transcoding.getString("preset", ID_UNKNOWN); - final String protocol = transcoding.getObject("format") - .getString("protocol"); - - if (protocol.contains("encrypted")) { - // Skip DRM-protected streams, which have encrypted in their protocol - // name - return; - } - - final AudioStream.Builder builder = new AudioStream.Builder() - .setId(preset); + final String protocol = transcoding.getObject("format") + .getString("protocol"); - if (protocol.equals("hls")) { - builder.setDeliveryMethod(DeliveryMethod.HLS); - } - - builder.setContent(getTranscodingUrl(url), true); - - if (preset.contains("mp3")) { - builder.setMediaFormat(MediaFormat.MP3); - builder.setAverageBitrate(128); - } else if (preset.contains("opus")) { - builder.setMediaFormat(MediaFormat.OPUS); - builder.setAverageBitrate(64); - } else if (preset.contains("aac_160k")) { - builder.setMediaFormat(MediaFormat.M4A); - builder.setAverageBitrate(160); - } else { - // Unknown format, skip to the next audio stream - return; - } + if (protocol.contains("encrypted")) { + // Skip DRM-protected streams, which have encrypted in their protocol + // name + return; + } - final AudioStream audioStream = builder.build(); - if (!Stream.containSimilarStream(audioStream, audioStreams)) { + final AudioStream audioStream; + try { + // SoundCloud only has one progressive stream, the rest are HLS + audioStream = protocol.equals("hls") + ? buildHlsAudioStream(transcoding) + : buildProgressiveAudioStream(transcoding); + if (audioStream != null + && !Stream.containSimilarStream(audioStream, audioStreams)) { + ExtractorLogger.d(TAG, audioStream.getFormat().getName() + " " + + getName() + " " + audioStream.getContent()); audioStreams.add(audioStream); } - } catch (final ExtractionException | IOException ignored) { + } catch (final ExtractionException | IOException e) { // Something went wrong when trying to get and add this audio stream, // skip to the next one + final var preset = transcoding.getString("preset", "unknown"); + ExtractorLogger.e(TAG, + getName() + " Failed to extract audio stream for transcoding " + + '[' + protocol + '/' + preset + "] " + url, + e); } }); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java index 410a20592f..935ff1ff73 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java @@ -28,7 +28,7 @@ import java.util.Locale; import java.util.Objects; -public final class AudioStream extends Stream { +public class AudioStream extends Stream { public static final int UNKNOWN_BITRATE = -1; private final int averageBitrate; @@ -60,7 +60,7 @@ public final class AudioStream extends Stream { * Class to build {@link AudioStream} objects. */ @SuppressWarnings("checkstyle:hiddenField") - public static final class Builder { + public static class Builder { private String id; private String content; private boolean isUrl; diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/HlsAudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/HlsAudioStream.java new file mode 100644 index 0000000000..d935abb419 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/HlsAudioStream.java @@ -0,0 +1,60 @@ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + +import java.io.IOException; + +import javax.annotation.Nonnull; + +public class HlsAudioStream extends AudioStream implements RefreshableStream { + private final String apiStreamUrl; + private final String playlistId; + + HlsAudioStream(final Builder builder) { + super(builder); + apiStreamUrl = builder.apiStreamUrl; + playlistId = builder.playlistId; + } + + @Nonnull + public String fetchLatestUrl() throws IOException, ExtractionException { + return SoundcloudHlsUtils.getStreamContentUrl(apiStreamUrl); + } + + @Nonnull + public String initialUrl() { + return getContent(); + } + + @Override + public String playlistId() { + return playlistId; + } + + @SuppressWarnings({"checkstyle:HiddenField", "UnusedReturnValue"}) + public static class Builder extends AudioStream.Builder { + private String apiStreamUrl; + private String playlistId; + + public Builder() { + setDeliveryMethod(DeliveryMethod.HLS); + } + + @Override + @Nonnull + public HlsAudioStream build() { + validateBuild(); + return new HlsAudioStream(this); + } + + public Builder setApiStreamUrl(@Nonnull final String apiStreamUrl) { + this.apiStreamUrl = apiStreamUrl; + return this; + } + + public Builder setPlaylistId(@Nonnull final String playlistId) { + this.playlistId = playlistId; + return this; + } + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/RefreshableStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/RefreshableStream.java new file mode 100644 index 0000000000..5d5a456adb --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/RefreshableStream.java @@ -0,0 +1,15 @@ +package org.schabi.newpipe.extractor.stream; + +import org.schabi.newpipe.extractor.exceptions.ExtractionException; + +import javax.annotation.Nonnull; +import java.io.IOException; + +@SuppressWarnings("checkstyle:LeftCurly") +public interface RefreshableStream { + @Nonnull + String fetchLatestUrl() throws IOException, ExtractionException; + String initialUrl(); + + String playlistId(); +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java new file mode 100644 index 0000000000..6dba187222 --- /dev/null +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java @@ -0,0 +1,90 @@ +package org.schabi.newpipe.extractor.stream; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; +import org.schabi.newpipe.extractor.utils.Parser; + +import java.io.IOException; +import java.util.regex.Pattern; + +import javax.annotation.Nonnull; + +public final class SoundcloudHlsUtils { + private static final String TAG = HlsAudioStream.class.getSimpleName(); + private static final Pattern MP3_HLS_PATTERN = + Pattern.compile("https://cf-hls-media\\.sndcdn.com/playlist/" + + "([a-zA-Z0-9]+)\\.128\\.mp3/playlist\\.m3u8"); + private static final Pattern AAC_HLS_PATTERN = + Pattern.compile("https://playback\\.media-streaming\\.soundcloud\\.cloud/" + + "([a-zA-Z0-9]+)/aac_160k/[a-f0-9\\-]+/playlist\\.m3u8"); + private static final Pattern OPUS_HLS_PATTERN = + Pattern.compile("https://cf-hls-opus-media\\.sndcdn\\.com/" + + "playlist/([a-zA-Z0-9]+)\\.64\\.opus/playlist\\.m3u8"); + + private SoundcloudHlsUtils() { } + + /** + * Calls the API endpoint url for this stream to get the url for retrieving the + * actual byte data for playback (returns the m3u8 playlist url for HLS streams, + * and the url to get the full binary track for progressives streams)

+ * + * NOTE: this returns a different url every time! (for SoundCloud) + * @param apiStreamUrl The url to call to get the actual stream data url + * @return The url for playing the audio (e.g. playlist.m3u8) + * @throws IOException If there's a problem calling the endpoint + * @throws ExtractionException for the same reason + */ + public static String getStreamContentUrl(final String apiStreamUrl) + throws IOException, ExtractionException { + ExtractorLogger.d(TAG, "Fetching content url for " + apiStreamUrl); + final String response = NewPipe.getDownloader() + .get(apiStreamUrl) + .validateResponseCode() + .responseBody(); + final JsonObject urlObject; + try { + urlObject = JsonParser.object().from(response); + } catch (final JsonParserException e) { + // TODO: Improve error message. + throw new ParsingException("Could not parse stream content from URL (" + + response + ")", e); + } + + return urlObject.getString("url"); + } + + @Nonnull + public static String extractHlsPlaylistId(final String hlsPlaylistUrl, + final MediaFormat mediaFormat) + throws ExtractionException { + switch (mediaFormat) { + case MP3: return extractHlsMp3PlaylistId(hlsPlaylistUrl); + case M4A: return extractHlsAacPlaylistId(hlsPlaylistUrl); + case OPUS: return extractHlsOpusPlaylistId(hlsPlaylistUrl); + default: + throw new IllegalArgumentException("Unsupported media format: " + mediaFormat); + } + } + + private static String extractHlsMp3PlaylistId(final String hlsPlaylistUrl) + throws ExtractionException { + return Parser.matchGroup1(MP3_HLS_PATTERN, hlsPlaylistUrl); + } + + private static String extractHlsAacPlaylistId(final String hlsPlaylistUrl) + throws ExtractionException { + return Parser.matchGroup1(AAC_HLS_PATTERN, hlsPlaylistUrl); + } + + private static String extractHlsOpusPlaylistId(final String hlsPlaylistUrl) + throws ExtractionException { + return Parser.matchGroup1(OPUS_HLS_PATTERN, hlsPlaylistUrl); + } +} From ec51b718b2a8593e5c7b34145ad2509c08422958 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Thu, 20 Nov 2025 07:40:09 +0000 Subject: [PATCH 07/10] Logging syntax changes --- .../soundcloud/SoundcloudParsingHelper.java | 8 +- .../extractors/SoundcloudStreamExtractor.java | 20 +- .../extractor/stream/SoundcloudHlsUtils.java | 180 +++++++++--------- .../newpipe/extractor/stream/StreamInfo.java | 10 +- 4 files changed, 111 insertions(+), 107 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 61a80e6648..42876e7362 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -102,7 +102,7 @@ private SoundcloudParsingHelper() { public static synchronized String clientId() throws ExtractionException, IOException { if (!isNullOrEmpty(clientId)) { - ExtractorLogger.d(TAG, "Returning clientId=" + clientId); + ExtractorLogger.d(TAG, "Returning clientId={clientId}", clientId); return clientId; } @@ -124,11 +124,11 @@ public static synchronized String clientId() throws ExtractionException, IOExcep final String srcUrl = element.attr("src"); if (!isNullOrEmpty(srcUrl)) { try { - ExtractorLogger.d(TAG, "Searching for clientId in " + srcUrl); + ExtractorLogger.d(TAG, "Searching for clientId in {srcUrl}", srcUrl); clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers) .validateResponseCode() .responseBody()); - ExtractorLogger.d(TAG, "Found clientId=" + clientId); + ExtractorLogger.d(TAG, "Found clientId={clientId}", clientId); return clientId; } catch (final RegexException ignored) { // Ignore it and proceed to try searching other script @@ -164,7 +164,7 @@ public static DateWrapper parseDate(final String uploadDate) throws ParsingExcep // CHECKSTYLE:ON public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url) throws IOException, ExtractionException { - ExtractorLogger.d(TAG, "resolveFor(" + url + ")"); + ExtractorLogger.d(TAG, "resolveFor({url})", url); final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve" + "?url=" + Utils.encodeUrlUtf8(url) + "&client_id=" + clientId(); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java index eef01bc772..c2681d6e1c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java @@ -59,11 +59,11 @@ public SoundcloudStreamExtractor(final StreamingService service, public void onFetchPage(@Nonnull final Downloader downloader) throws IOException, ExtractionException { final var url = getUrl(); - ExtractorLogger.d(TAG, "onFetchPage(" + url + ")"); + ExtractorLogger.d(TAG, "onFetchPage({url}", url); track = SoundcloudParsingHelper.resolveFor(downloader, url); final String policy = track.getString("policy", ""); - ExtractorLogger.d(TAG, "policy is: " + policy); + ExtractorLogger.d(TAG, "policy is: {policy}", policy); if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) { isAvailable = false; @@ -168,7 +168,7 @@ public List getAudioStreams() throws ExtractionException { // For playing the track, it is only necessary to have a streamable track. // If this is not the case, this track might not be published yet. if (!track.getBoolean("streamable") || !isAvailable) { - ExtractorLogger.d(TAG, "Not streamable track: " + getUrl()); + ExtractorLogger.d(TAG, "Not streamable track: {url}", getUrl()); return audioStreams; } @@ -177,7 +177,7 @@ public List getAudioStreams() throws ExtractionException { .getArray("transcodings"); if (!isNullOrEmpty(transcodings)) { // Get information about what stream formats are available - ExtractorLogger.d(TAG, "Extracting audio streams for " + getName()); + ExtractorLogger.d(TAG, "Extracting audio streams for {name}", getName()); extractAudioStreams(transcodings, audioStreams); } } catch (final NullPointerException e) { @@ -237,7 +237,7 @@ public String toString() { private StreamBuildResult buildBaseAudioStream(final JsonObject transcoding, final AudioStream.Builder builder) throws ExtractionException, IOException { - ExtractorLogger.d(TAG, getName() + " Building base audio stream info"); + ExtractorLogger.d(TAG, "{name} Building base audio stream info", getName()); final var preset = transcoding.getString("preset", ID_UNKNOWN); final MediaFormat mediaFormat; if (preset.contains("mp3")) { @@ -266,7 +266,7 @@ private StreamBuildResult buildBaseAudioStream(final JsonObject transcoding, private HlsAudioStream buildHlsAudioStream(final JsonObject transcoding) throws ExtractionException, IOException { - ExtractorLogger.d(TAG, getName() + "Extracting hls audio stream"); + ExtractorLogger.d(TAG, "{name} Extracting hls audio stream", getName()); final var builder = new HlsAudioStream.Builder(); final StreamBuildResult buildResult = buildBaseAudioStream(transcoding, builder); if (buildResult == null) { @@ -282,7 +282,7 @@ private HlsAudioStream buildHlsAudioStream(final JsonObject transcoding) private AudioStream buildProgressiveAudioStream(final JsonObject transcoding) throws ExtractionException, IOException { - ExtractorLogger.d(TAG, getName() + "Extracting progressive audio stream"); + ExtractorLogger.d(TAG, "{name} Extracting progressive audio stream", getName()); final var builder = new AudioStream.Builder(); final StreamBuildResult buildResult = buildBaseAudioStream(transcoding, builder); return buildResult == null ? null : builder.build(); @@ -317,8 +317,10 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings, : buildProgressiveAudioStream(transcoding); if (audioStream != null && !Stream.containSimilarStream(audioStream, audioStreams)) { - ExtractorLogger.d(TAG, audioStream.getFormat().getName() + " " - + getName() + " " + audioStream.getContent()); + ExtractorLogger.d(TAG, "{format} {trackName} {url}", + audioStream.getFormat().getName(), + getName(), + audioStream.getContent()); audioStreams.add(audioStream); } } catch (final ExtractionException | IOException e) { diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java index 6dba187222..8f654d779c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java @@ -1,90 +1,90 @@ -package org.schabi.newpipe.extractor.stream; - -import com.grack.nanojson.JsonObject; -import com.grack.nanojson.JsonParser; -import com.grack.nanojson.JsonParserException; - -import org.schabi.newpipe.extractor.MediaFormat; -import org.schabi.newpipe.extractor.NewPipe; -import org.schabi.newpipe.extractor.exceptions.ExtractionException; -import org.schabi.newpipe.extractor.exceptions.ParsingException; -import org.schabi.newpipe.extractor.utils.ExtractorLogger; -import org.schabi.newpipe.extractor.utils.Parser; - -import java.io.IOException; -import java.util.regex.Pattern; - -import javax.annotation.Nonnull; - -public final class SoundcloudHlsUtils { - private static final String TAG = HlsAudioStream.class.getSimpleName(); - private static final Pattern MP3_HLS_PATTERN = - Pattern.compile("https://cf-hls-media\\.sndcdn.com/playlist/" - + "([a-zA-Z0-9]+)\\.128\\.mp3/playlist\\.m3u8"); - private static final Pattern AAC_HLS_PATTERN = - Pattern.compile("https://playback\\.media-streaming\\.soundcloud\\.cloud/" - + "([a-zA-Z0-9]+)/aac_160k/[a-f0-9\\-]+/playlist\\.m3u8"); - private static final Pattern OPUS_HLS_PATTERN = - Pattern.compile("https://cf-hls-opus-media\\.sndcdn\\.com/" - + "playlist/([a-zA-Z0-9]+)\\.64\\.opus/playlist\\.m3u8"); - - private SoundcloudHlsUtils() { } - - /** - * Calls the API endpoint url for this stream to get the url for retrieving the - * actual byte data for playback (returns the m3u8 playlist url for HLS streams, - * and the url to get the full binary track for progressives streams)

- * - * NOTE: this returns a different url every time! (for SoundCloud) - * @param apiStreamUrl The url to call to get the actual stream data url - * @return The url for playing the audio (e.g. playlist.m3u8) - * @throws IOException If there's a problem calling the endpoint - * @throws ExtractionException for the same reason - */ - public static String getStreamContentUrl(final String apiStreamUrl) - throws IOException, ExtractionException { - ExtractorLogger.d(TAG, "Fetching content url for " + apiStreamUrl); - final String response = NewPipe.getDownloader() - .get(apiStreamUrl) - .validateResponseCode() - .responseBody(); - final JsonObject urlObject; - try { - urlObject = JsonParser.object().from(response); - } catch (final JsonParserException e) { - // TODO: Improve error message. - throw new ParsingException("Could not parse stream content from URL (" - + response + ")", e); - } - - return urlObject.getString("url"); - } - - @Nonnull - public static String extractHlsPlaylistId(final String hlsPlaylistUrl, - final MediaFormat mediaFormat) - throws ExtractionException { - switch (mediaFormat) { - case MP3: return extractHlsMp3PlaylistId(hlsPlaylistUrl); - case M4A: return extractHlsAacPlaylistId(hlsPlaylistUrl); - case OPUS: return extractHlsOpusPlaylistId(hlsPlaylistUrl); - default: - throw new IllegalArgumentException("Unsupported media format: " + mediaFormat); - } - } - - private static String extractHlsMp3PlaylistId(final String hlsPlaylistUrl) - throws ExtractionException { - return Parser.matchGroup1(MP3_HLS_PATTERN, hlsPlaylistUrl); - } - - private static String extractHlsAacPlaylistId(final String hlsPlaylistUrl) - throws ExtractionException { - return Parser.matchGroup1(AAC_HLS_PATTERN, hlsPlaylistUrl); - } - - private static String extractHlsOpusPlaylistId(final String hlsPlaylistUrl) - throws ExtractionException { - return Parser.matchGroup1(OPUS_HLS_PATTERN, hlsPlaylistUrl); - } -} +package org.schabi.newpipe.extractor.stream; + +import com.grack.nanojson.JsonObject; +import com.grack.nanojson.JsonParser; +import com.grack.nanojson.JsonParserException; + +import org.schabi.newpipe.extractor.MediaFormat; +import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.exceptions.ExtractionException; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.utils.ExtractorLogger; +import org.schabi.newpipe.extractor.utils.Parser; + +import java.io.IOException; +import java.util.regex.Pattern; + +import javax.annotation.Nonnull; + +public final class SoundcloudHlsUtils { + private static final String TAG = HlsAudioStream.class.getSimpleName(); + private static final Pattern MP3_HLS_PATTERN = + Pattern.compile("https://cf-hls-media\\.sndcdn.com/playlist/" + + "([a-zA-Z0-9]+)\\.128\\.mp3/playlist\\.m3u8"); + private static final Pattern AAC_HLS_PATTERN = + Pattern.compile("https://playback\\.media-streaming\\.soundcloud\\.cloud/" + + "([a-zA-Z0-9]+)/aac_160k/[a-f0-9\\-]+/playlist\\.m3u8"); + private static final Pattern OPUS_HLS_PATTERN = + Pattern.compile("https://cf-hls-opus-media\\.sndcdn\\.com/" + + "playlist/([a-zA-Z0-9]+)\\.64\\.opus/playlist\\.m3u8"); + + private SoundcloudHlsUtils() { } + + /** + * Calls the API endpoint url for this stream to get the url for retrieving the + * actual byte data for playback (returns the m3u8 playlist url for HLS streams, + * and the url to get the full binary track for progressives streams)

+ * + * NOTE: this returns a different url every time! (for SoundCloud) + * @param apiStreamUrl The url to call to get the actual stream data url + * @return The url for playing the audio (e.g. playlist.m3u8) + * @throws IOException If there's a problem calling the endpoint + * @throws ExtractionException for the same reason + */ + public static String getStreamContentUrl(final String apiStreamUrl) + throws IOException, ExtractionException { + ExtractorLogger.d(TAG, "Fetching content url for {url}", apiStreamUrl); + final String response = NewPipe.getDownloader() + .get(apiStreamUrl) + .validateResponseCode() + .responseBody(); + final JsonObject urlObject; + try { + urlObject = JsonParser.object().from(response); + } catch (final JsonParserException e) { + // TODO: Improve error message. + throw new ParsingException("Could not parse stream content from URL (" + + response + ")", e); + } + + return urlObject.getString("url"); + } + + @Nonnull + public static String extractHlsPlaylistId(final String hlsPlaylistUrl, + final MediaFormat mediaFormat) + throws ExtractionException { + switch (mediaFormat) { + case MP3: return extractHlsMp3PlaylistId(hlsPlaylistUrl); + case M4A: return extractHlsAacPlaylistId(hlsPlaylistUrl); + case OPUS: return extractHlsOpusPlaylistId(hlsPlaylistUrl); + default: + throw new IllegalArgumentException("Unsupported media format: " + mediaFormat); + } + } + + private static String extractHlsMp3PlaylistId(final String hlsPlaylistUrl) + throws ExtractionException { + return Parser.matchGroup1(MP3_HLS_PATTERN, hlsPlaylistUrl); + } + + private static String extractHlsAacPlaylistId(final String hlsPlaylistUrl) + throws ExtractionException { + return Parser.matchGroup1(AAC_HLS_PATTERN, hlsPlaylistUrl); + } + + private static String extractHlsOpusPlaylistId(final String hlsPlaylistUrl) + throws ExtractionException { + return Parser.matchGroup1(OPUS_HLS_PATTERN, hlsPlaylistUrl); + } +} diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java index 7ecd6019d6..272cfde96a 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java @@ -190,12 +190,14 @@ private static void extractStreams(final StreamInfo streamInfo, final var url = streamInfo.getOriginalUrl(); final var name = streamInfo.getName(); if (errors.isEmpty()) { - ExtractorLogger.e(TAG, "Error extracting " + name + " " + url - + "\nCould not get any stream and didn't catch any errors"); + ExtractorLogger.e(TAG, "Error extracting {name} {url} \n" + + "Could not get any stream and didn't catch any errors", + name, url); } else { errors.forEach(m -> ExtractorLogger.e(TAG, - "Error for " + streamInfo.getOriginalUrl(), - m)); + m, + "Error for {url}", + url)); } throw new StreamExtractException( From 5b6c37217074a32cb83e936925d06dfdff16798b Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Thu, 29 Jan 2026 23:10:01 +0000 Subject: [PATCH 08/10] Fix checkstyle (curse you checkstyle) --- .../main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java index 31c937ea09..01a905d8aa 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java @@ -10,7 +10,7 @@ public final class HttpUtils { private HttpUtils() { // Utility class, no instances allowed } - + // CHECKSTYLE:OFF /** * Validates the response codes for the given {@link Response}, and throws From 5d590c6d2fad577ac0b1848f1360c8aead268308 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Thu, 29 Jan 2026 23:19:36 +0000 Subject: [PATCH 09/10] Fix Soundcloud HLS Opus Regex to include hyphen --- .../org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java index 8f654d779c..1edb89a539 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java @@ -26,7 +26,7 @@ public final class SoundcloudHlsUtils { + "([a-zA-Z0-9]+)/aac_160k/[a-f0-9\\-]+/playlist\\.m3u8"); private static final Pattern OPUS_HLS_PATTERN = Pattern.compile("https://cf-hls-opus-media\\.sndcdn\\.com/" - + "playlist/([a-zA-Z0-9]+)\\.64\\.opus/playlist\\.m3u8"); + + "playlist/([a-zA-Z0-9-]+)\\.64\\.opus/playlist\\.m3u8"); private SoundcloudHlsUtils() { } From 95408f3620006850a539095d832e1dca7825b276 Mon Sep 17 00:00:00 2001 From: AbsurdlyLongUsername <22662897+absurdlylongusername@users.noreply.github.com> Date: Fri, 30 Jan 2026 00:27:21 +0000 Subject: [PATCH 10/10] Fix javadoc so jitpack can build? --- .../java/org/schabi/newpipe/extractor/downloader/Response.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java index 87c3577ef4..05fe5477a5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java @@ -90,7 +90,7 @@ public String getHeader(final String name) { * Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid * @see HttpUtils#validateResponseCode(Response, int...) * @param validResponseCodes Expected valid response codes - * @return {@link this} response + * @return {@code this} response * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes}, * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error. */