+ * Call the endpoint "/resolve" of the API. *
- * See https://developers.soundcloud.com/docs/api/reference#resolve
+ * See https://web.archive.org/web/20170804051146/https://developers.soundcloud.com/docs/api/reference#resolve
*/
+ // CHECKSTYLE:ON
public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url)
throws IOException, ExtractionException {
+ ExtractorLogger.d(TAG, "resolveFor({url})", url);
final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve"
+ "?url=" + Utils.encodeUrlUtf8(url)
+ "&client_id=" + clientId();
@@ -178,10 +187,11 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final
public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException,
ReCaptchaException {
- final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
- + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()).responseBody();
-
- return Jsoup.parse(response).select("link[rel=\"canonical\"]").first()
+ final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization());
+ validateResponseCode(response);
+ final var responseBody = response.responseBody();
+ return Jsoup.parse(responseBody).select("link[rel=\"canonical\"]").first()
.attr("abs:href");
}
@@ -190,6 +200,7 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc
*
* @return the resolved id
*/
+ // TODO: what makes this method different from the others? Don' they all return the same?
public static String resolveIdWithWidgetApi(final String urlString) throws IOException,
ParsingException {
String fixedUrl = urlString;
@@ -225,9 +236,12 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc
final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url="
+ Utils.encodeUrlUtf8(url.toString())
+ "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
- final String response = NewPipe.getDownloader().get(widgetUrl,
- SoundCloud.getLocalization()).responseBody();
- final JsonObject o = JsonParser.object().from(response);
+
+ final var response = NewPipe.getDownloader().get(widgetUrl,
+ SoundCloud.getLocalization());
+
+ final var responseBody = response.validateResponseCode().responseBody();
+ final JsonObject o = JsonParser.object().from(responseBody);
return String.valueOf(JsonUtils.getValue(o, "id"));
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse JSON response", e);
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
index 4de7114de6..c2681d6e1c 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudStreamExtractor.java
@@ -11,12 +11,9 @@
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
-import com.grack.nanojson.JsonParser;
-import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Image;
import org.schabi.newpipe.extractor.MediaFormat;
-import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
@@ -30,11 +27,14 @@
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
import org.schabi.newpipe.extractor.stream.Description;
+import org.schabi.newpipe.extractor.stream.HlsAudioStream;
+import org.schabi.newpipe.extractor.stream.SoundcloudHlsUtils;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
@@ -46,6 +46,7 @@
import javax.annotation.Nullable;
public class SoundcloudStreamExtractor extends StreamExtractor {
+ public static final String TAG = SoundcloudStreamExtractor.class.getSimpleName();
private JsonObject track;
private boolean isAvailable = true;
@@ -57,9 +58,12 @@ public SoundcloudStreamExtractor(final StreamingService service,
@Override
public void onFetchPage(@Nonnull final Downloader downloader) throws IOException,
ExtractionException {
- track = SoundcloudParsingHelper.resolveFor(downloader, getUrl());
+ final var url = getUrl();
+ ExtractorLogger.d(TAG, "onFetchPage({url}", url);
+ track = SoundcloudParsingHelper.resolveFor(downloader, url);
final String policy = track.getString("policy", "");
+ ExtractorLogger.d(TAG, "policy is: {policy}", policy);
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
isAvailable = false;
@@ -164,6 +168,7 @@ public List
+ * Returns the stream content url if we support this type of transcoding, {@code null} otherwise
+ * @param transcoding The SoundCloud JSON transcoding object for this stream
+ * @param builder AudioStream builder to set the common values
+ * @return the stream content url if this transcoding is supported and common values were built
+ * {@code null} otherwise
+ */
+ @Nullable
+ private StreamBuildResult buildBaseAudioStream(final JsonObject transcoding,
+ final AudioStream.Builder builder)
+ throws ExtractionException, IOException {
+ ExtractorLogger.d(TAG, "{name} Building base audio stream info", getName());
+ final var preset = transcoding.getString("preset", ID_UNKNOWN);
+ final MediaFormat mediaFormat;
+ if (preset.contains("mp3")) {
+ mediaFormat = MediaFormat.MP3;
+ builder.setAverageBitrate(128);
+ } else if (preset.contains("opus")) {
+ mediaFormat = MediaFormat.OPUS;
+ builder.setAverageBitrate(64);
+ builder.setDeliveryMethod(DeliveryMethod.HLS);
+ } else if (preset.contains("aac_160k")) {
+ mediaFormat = MediaFormat.M4A;
+ builder.setAverageBitrate(160);
+ } else {
+ // Unknown format, return null to skip to the next audio stream
+ return null;
}
- return urlObject.getString("url");
+ builder.setMediaFormat(mediaFormat);
+
+ builder.setId(preset);
+ final var url = transcoding.getString("url");
+ final var hlsPlaylistUrl = SoundcloudHlsUtils.getStreamContentUrl(getApiStreamUrl(url));
+ builder.setContent(hlsPlaylistUrl, true);
+ return new StreamBuildResult(hlsPlaylistUrl, mediaFormat);
+ }
+
+ private HlsAudioStream buildHlsAudioStream(final JsonObject transcoding)
+ throws ExtractionException, IOException {
+ ExtractorLogger.d(TAG, "{name} Extracting hls audio stream", getName());
+ final var builder = new HlsAudioStream.Builder();
+ final StreamBuildResult buildResult = buildBaseAudioStream(transcoding, builder);
+ if (buildResult == null) {
+ return null;
+ }
+
+ builder.setApiStreamUrl(getApiStreamUrl(transcoding.getString("url")));
+ builder.setPlaylistId(SoundcloudHlsUtils.extractHlsPlaylistId(buildResult.contentUrl,
+ buildResult.mediaFormat));
+
+ return builder.build();
+ }
+
+ private AudioStream buildProgressiveAudioStream(final JsonObject transcoding)
+ throws ExtractionException, IOException {
+ ExtractorLogger.d(TAG, "{name} Extracting progressive audio stream", getName());
+ final var builder = new AudioStream.Builder();
+ final StreamBuildResult buildResult = buildBaseAudioStream(transcoding, builder);
+ return buildResult == null ? null : builder.build();
+ // TODO: anything else?
}
private void extractAudioStreams(@Nonnull final JsonArray transcodings,
@@ -213,47 +300,37 @@ private void extractAudioStreams(@Nonnull final JsonArray transcodings,
return;
}
- try {
- final String preset = transcoding.getString("preset", ID_UNKNOWN);
- final String protocol = transcoding.getObject("format")
- .getString("protocol");
-
- if (protocol.contains("encrypted")) {
- // Skip DRM-protected streams, which have encrypted in their protocol
- // name
- return;
- }
-
- final AudioStream.Builder builder = new AudioStream.Builder()
- .setId(preset);
+ final String protocol = transcoding.getObject("format")
+ .getString("protocol");
- if (protocol.equals("hls")) {
- builder.setDeliveryMethod(DeliveryMethod.HLS);
- }
-
- builder.setContent(getTranscodingUrl(url), true);
-
- if (preset.contains("mp3")) {
- builder.setMediaFormat(MediaFormat.MP3);
- builder.setAverageBitrate(128);
- } else if (preset.contains("opus")) {
- builder.setMediaFormat(MediaFormat.OPUS);
- builder.setAverageBitrate(64);
- } else if (preset.contains("aac_160k")) {
- builder.setMediaFormat(MediaFormat.M4A);
- builder.setAverageBitrate(160);
- } else {
- // Unknown format, skip to the next audio stream
- return;
- }
+ if (protocol.contains("encrypted")) {
+ // Skip DRM-protected streams, which have encrypted in their protocol
+ // name
+ return;
+ }
- final AudioStream audioStream = builder.build();
- if (!Stream.containSimilarStream(audioStream, audioStreams)) {
+ final AudioStream audioStream;
+ try {
+ // SoundCloud only has one progressive stream, the rest are HLS
+ audioStream = protocol.equals("hls")
+ ? buildHlsAudioStream(transcoding)
+ : buildProgressiveAudioStream(transcoding);
+ if (audioStream != null
+ && !Stream.containSimilarStream(audioStream, audioStreams)) {
+ ExtractorLogger.d(TAG, "{format} {trackName} {url}",
+ audioStream.getFormat().getName(),
+ getName(),
+ audioStream.getContent());
audioStreams.add(audioStream);
}
- } catch (final ExtractionException | IOException ignored) {
+ } catch (final ExtractionException | IOException e) {
// Something went wrong when trying to get and add this audio stream,
// skip to the next one
+ final var preset = transcoding.getString("preset", "unknown");
+ ExtractorLogger.e(TAG,
+ getName() + " Failed to extract audio stream for transcoding "
+ + '[' + protocol + '/' + preset + "] " + url,
+ e);
}
});
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java
index 410a20592f..935ff1ff73 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/AudioStream.java
@@ -28,7 +28,7 @@
import java.util.Locale;
import java.util.Objects;
-public final class AudioStream extends Stream {
+public class AudioStream extends Stream {
public static final int UNKNOWN_BITRATE = -1;
private final int averageBitrate;
@@ -60,7 +60,7 @@ public final class AudioStream extends Stream {
* Class to build {@link AudioStream} objects.
*/
@SuppressWarnings("checkstyle:hiddenField")
- public static final class Builder {
+ public static class Builder {
private String id;
private String content;
private boolean isUrl;
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/HlsAudioStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/HlsAudioStream.java
new file mode 100644
index 0000000000..d935abb419
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/HlsAudioStream.java
@@ -0,0 +1,60 @@
+package org.schabi.newpipe.extractor.stream;
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import java.io.IOException;
+
+import javax.annotation.Nonnull;
+
+public class HlsAudioStream extends AudioStream implements RefreshableStream {
+ private final String apiStreamUrl;
+ private final String playlistId;
+
+ HlsAudioStream(final Builder builder) {
+ super(builder);
+ apiStreamUrl = builder.apiStreamUrl;
+ playlistId = builder.playlistId;
+ }
+
+ @Nonnull
+ public String fetchLatestUrl() throws IOException, ExtractionException {
+ return SoundcloudHlsUtils.getStreamContentUrl(apiStreamUrl);
+ }
+
+ @Nonnull
+ public String initialUrl() {
+ return getContent();
+ }
+
+ @Override
+ public String playlistId() {
+ return playlistId;
+ }
+
+ @SuppressWarnings({"checkstyle:HiddenField", "UnusedReturnValue"})
+ public static class Builder extends AudioStream.Builder {
+ private String apiStreamUrl;
+ private String playlistId;
+
+ public Builder() {
+ setDeliveryMethod(DeliveryMethod.HLS);
+ }
+
+ @Override
+ @Nonnull
+ public HlsAudioStream build() {
+ validateBuild();
+ return new HlsAudioStream(this);
+ }
+
+ public Builder setApiStreamUrl(@Nonnull final String apiStreamUrl) {
+ this.apiStreamUrl = apiStreamUrl;
+ return this;
+ }
+
+ public Builder setPlaylistId(@Nonnull final String playlistId) {
+ this.playlistId = playlistId;
+ return this;
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/RefreshableStream.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/RefreshableStream.java
new file mode 100644
index 0000000000..5d5a456adb
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/RefreshableStream.java
@@ -0,0 +1,15 @@
+package org.schabi.newpipe.extractor.stream;
+
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+
+import javax.annotation.Nonnull;
+import java.io.IOException;
+
+@SuppressWarnings("checkstyle:LeftCurly")
+public interface RefreshableStream {
+ @Nonnull
+ String fetchLatestUrl() throws IOException, ExtractionException;
+ String initialUrl();
+
+ String playlistId();
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java
new file mode 100644
index 0000000000..1edb89a539
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/SoundcloudHlsUtils.java
@@ -0,0 +1,90 @@
+package org.schabi.newpipe.extractor.stream;
+
+import com.grack.nanojson.JsonObject;
+import com.grack.nanojson.JsonParser;
+import com.grack.nanojson.JsonParserException;
+
+import org.schabi.newpipe.extractor.MediaFormat;
+import org.schabi.newpipe.extractor.NewPipe;
+import org.schabi.newpipe.extractor.exceptions.ExtractionException;
+import org.schabi.newpipe.extractor.exceptions.ParsingException;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
+import org.schabi.newpipe.extractor.utils.Parser;
+
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nonnull;
+
+public final class SoundcloudHlsUtils {
+ private static final String TAG = HlsAudioStream.class.getSimpleName();
+ private static final Pattern MP3_HLS_PATTERN =
+ Pattern.compile("https://cf-hls-media\\.sndcdn.com/playlist/"
+ + "([a-zA-Z0-9]+)\\.128\\.mp3/playlist\\.m3u8");
+ private static final Pattern AAC_HLS_PATTERN =
+ Pattern.compile("https://playback\\.media-streaming\\.soundcloud\\.cloud/"
+ + "([a-zA-Z0-9]+)/aac_160k/[a-f0-9\\-]+/playlist\\.m3u8");
+ private static final Pattern OPUS_HLS_PATTERN =
+ Pattern.compile("https://cf-hls-opus-media\\.sndcdn\\.com/"
+ + "playlist/([a-zA-Z0-9-]+)\\.64\\.opus/playlist\\.m3u8");
+
+ private SoundcloudHlsUtils() { }
+
+ /**
+ * Calls the API endpoint url for this stream to get the url for retrieving the
+ * actual byte data for playback (returns the m3u8 playlist url for HLS streams,
+ * and the url to get the full binary track for progressives streams)
+ *
+ * NOTE: this returns a different url every time! (for SoundCloud)
+ * @param apiStreamUrl The url to call to get the actual stream data url
+ * @return The url for playing the audio (e.g. playlist.m3u8)
+ * @throws IOException If there's a problem calling the endpoint
+ * @throws ExtractionException for the same reason
+ */
+ public static String getStreamContentUrl(final String apiStreamUrl)
+ throws IOException, ExtractionException {
+ ExtractorLogger.d(TAG, "Fetching content url for {url}", apiStreamUrl);
+ final String response = NewPipe.getDownloader()
+ .get(apiStreamUrl)
+ .validateResponseCode()
+ .responseBody();
+ final JsonObject urlObject;
+ try {
+ urlObject = JsonParser.object().from(response);
+ } catch (final JsonParserException e) {
+ // TODO: Improve error message.
+ throw new ParsingException("Could not parse stream content from URL ("
+ + response + ")", e);
+ }
+
+ return urlObject.getString("url");
+ }
+
+ @Nonnull
+ public static String extractHlsPlaylistId(final String hlsPlaylistUrl,
+ final MediaFormat mediaFormat)
+ throws ExtractionException {
+ switch (mediaFormat) {
+ case MP3: return extractHlsMp3PlaylistId(hlsPlaylistUrl);
+ case M4A: return extractHlsAacPlaylistId(hlsPlaylistUrl);
+ case OPUS: return extractHlsOpusPlaylistId(hlsPlaylistUrl);
+ default:
+ throw new IllegalArgumentException("Unsupported media format: " + mediaFormat);
+ }
+ }
+
+ private static String extractHlsMp3PlaylistId(final String hlsPlaylistUrl)
+ throws ExtractionException {
+ return Parser.matchGroup1(MP3_HLS_PATTERN, hlsPlaylistUrl);
+ }
+
+ private static String extractHlsAacPlaylistId(final String hlsPlaylistUrl)
+ throws ExtractionException {
+ return Parser.matchGroup1(AAC_HLS_PATTERN, hlsPlaylistUrl);
+ }
+
+ private static String extractHlsOpusPlaylistId(final String hlsPlaylistUrl)
+ throws ExtractionException {
+ return Parser.matchGroup1(OPUS_HLS_PATTERN, hlsPlaylistUrl);
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
index 62fb6bbf74..272cfde96a 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/stream/StreamInfo.java
@@ -31,6 +31,7 @@
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.ExtractorHelper;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
import java.io.IOException;
import java.util.List;
@@ -44,7 +45,7 @@
* Info object for opened contents, i.e. the content ready to play.
*/
public class StreamInfo extends Info {
-
+ private static final String TAG = StreamInfo.class.getSimpleName();
public static class StreamExtractException extends ExtractionException {
StreamExtractException(final String message) {
super(message);
@@ -61,19 +62,36 @@ public StreamInfo(final int serviceId,
super(serviceId, id, url, originalUrl, name);
this.streamType = streamType;
this.ageLimit = ageLimit;
+ ExtractorLogger.d(TAG, "Created {}", this);
+ }
+
+ @Override
+ public String toString() {
+ return TAG + "["
+ + "serviceId=" + getServiceId()
+ + ", url='" + getUrl() + '\''
+ + ", originalUrl='" + getOriginalUrl() + '\''
+ + ", id='" + getId() + '\''
+ + ", name='" + getName() + '\''
+ + ", streamType=" + streamType
+ + ", ageLimit=" + ageLimit
+ + ']';
}
public static StreamInfo getInfo(final String url) throws IOException, ExtractionException {
+ ExtractorLogger.d(TAG, "getInfo({url})", url);
return getInfo(NewPipe.getServiceByUrl(url), url);
}
public static StreamInfo getInfo(@Nonnull final StreamingService service,
final String url) throws IOException, ExtractionException {
+ ExtractorLogger.d(TAG, "getInfo({service},{url})", service, url);
return getInfo(service.getStreamExtractor(url));
}
public static StreamInfo getInfo(@Nonnull final StreamExtractor extractor)
throws ExtractionException, IOException {
+ ExtractorLogger.d(TAG, "getInfo({extractor)", extractor);
extractor.fetchPage();
final StreamInfo streamInfo;
try {
@@ -168,6 +186,20 @@ private static void extractStreams(final StreamInfo streamInfo,
// Either audio or video has to be available, otherwise we didn't get a stream (since
// videoOnly are optional, they don't count).
if ((streamInfo.videoStreams.isEmpty()) && (streamInfo.audioStreams.isEmpty())) {
+ final var errors = streamInfo.getErrors();
+ final var url = streamInfo.getOriginalUrl();
+ final var name = streamInfo.getName();
+ if (errors.isEmpty()) {
+ ExtractorLogger.e(TAG, "Error extracting {name} {url} \n"
+ + "Could not get any stream and didn't catch any errors",
+ name, url);
+ } else {
+ errors.forEach(m -> ExtractorLogger.e(TAG,
+ m,
+ "Error for {url}",
+ url));
+ }
+
throw new StreamExtractException(
"Could not get any stream. See error variable to get further details.");
}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java
new file mode 100644
index 0000000000..eb1557946c
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ExtractorLogger.java
@@ -0,0 +1,193 @@
+package org.schabi.newpipe.extractor.utils;
+
+public final class ExtractorLogger {
+
+ private ExtractorLogger() { }
+
+ private static final Logger EMPTY_LOGGER = new EmptyLogger();
+ private static volatile Logger logger = EMPTY_LOGGER;
+
+ public static void setLogger(final Logger customLogger) {
+ logger = customLogger != null ? customLogger : EMPTY_LOGGER;
+ }
+
+ public enum Level { DEBUG, WARN, ERROR }
+
+ @SuppressWarnings("checkstyle:NeedBraces")
+ private static void log(final Level level,
+ final String tag,
+ final String message,
+ final Throwable t) {
+ if (logger == EMPTY_LOGGER) return;
+ switch (level) {
+ case DEBUG:
+ if (t == null) {
+ logger.debug(tag, message);
+ } else {
+ logger.debug(tag, message, t);
+ }
+ break;
+ case WARN:
+ if (t == null) {
+ logger.warn(tag, message);
+ } else {
+ logger.warn(tag, message, t);
+ }
+ break;
+ case ERROR:
+ if (t == null) {
+ logger.error(tag, message);
+ } else {
+ logger.error(tag, message, t);
+ }
+ break;
+ }
+ }
+
+ @SuppressWarnings("checkstyle:NeedBraces")
+ private static void logFormat(final Level level,
+ final String tag,
+ final Throwable t,
+ final String template,
+ final Object... args) {
+ if (logger == EMPTY_LOGGER) return;
+ log(level, tag, format(template, args), t);
+ }
+
+ // DEBUG
+ public static void d(final String tag, final String msg) {
+ log(Level.DEBUG, tag, msg, null);
+ }
+
+ public static void d(final String tag, final String msg, final Throwable t) {
+ log(Level.DEBUG, tag, msg, t);
+ }
+
+ public static void d(final String tag, final String template, final Object... args) {
+ logFormat(Level.DEBUG, tag, null, template, args);
+ }
+
+ public static void d(final String tag,
+ final Throwable t,
+ final String template,
+ final Object... args) {
+ logFormat(Level.DEBUG, tag, t, template, args);
+ }
+
+ // WARN
+ public static void w(final String tag, final String msg) {
+ log(Level.WARN, tag, msg, null);
+ }
+
+ public static void w(final String tag, final String msg, final Throwable t) {
+ log(Level.WARN, tag, msg, t);
+ }
+
+ public static void w(final String tag, final String template, final Object... args) {
+ logFormat(Level.WARN, tag, null, template, args);
+ }
+
+ public static void w(final String tag,
+ final Throwable t,
+ final String template,
+ final Object... args) {
+ logFormat(Level.WARN, tag, t, template, args);
+ }
+
+ // ERROR
+ public static void e(final String tag, final String msg) {
+ log(Level.ERROR, tag, msg, null);
+ }
+
+ public static void e(final String tag, final String msg, final Throwable t) {
+ log(Level.ERROR, tag, msg, t);
+ }
+
+ public static void e(final String tag, final String template, final Object... args) {
+ logFormat(Level.ERROR, tag, null, template, args);
+ }
+
+ public static void e(final String tag,
+ final Throwable t,
+ final String template,
+ final Object... args) {
+ logFormat(Level.ERROR, tag, t, template, args);
+ }
+
+ /**
+ * Simple string format method for easier logging in the form of
+ * {@code ExtractorLogger.d("Hello my name {Name} {}", name, surname)}
+ * @param template The template string to format
+ * @param args Arguments to replace identifiers with in {@code template}
+ * @return Formatted string with arguments replaced
+ */
+ private static String format(final String template, final Object... args) {
+ if (template == null || args == null || args.length == 0) {
+ return template;
+ }
+ final var out = new StringBuilder(template.length() + Math.min(32, 16 * args.length));
+ int cursorIndex = 0;
+ int argIndex = 0;
+ final int n = template.length();
+ while (cursorIndex < n) {
+ // Find first/next open brace
+ final int openBraceIndex = template.indexOf('{', cursorIndex);
+ if (openBraceIndex < 0) {
+ // If none found then there's no more arguments to replace
+ out.append(template, cursorIndex, n); break;
+ }
+
+ // Find matching closing brace
+ final int close = template.indexOf('}', openBraceIndex + 1);
+ if (close < 0) {
+ // If none found then there's no more arguments to replace
+ out.append(template, cursorIndex, n); break;
+ }
+ // Append everything from cursor up to before the open brace
+ out.append(template, cursorIndex, openBraceIndex);
+ // Append arguments in the brace
+ out.append(argIndex < args.length
+ ? String.valueOf(args[argIndex++])
+ : template.substring(openBraceIndex, close + 1));
+ cursorIndex = close + 1;
+ }
+ return out.toString();
+ }
+
+ private static final class EmptyLogger implements Logger {
+ public void debug(final String tag, final String msg) { }
+ public void debug(final String tag, final String msg, final Throwable throwable) { }
+ public void warn(final String tag, final String msg) { }
+ public void warn(final String tag, final String msg, final Throwable t) { }
+ public void error(final String tag, final String msg) { }
+ public void error(final String tag, final String msg, final Throwable t) { }
+ }
+
+ public static final class ConsoleLogger implements Logger {
+ public void debug(final String tag, final String msg) {
+ System.out.println("[DEBUG][" + tag + "] " + msg);
+ }
+
+ public void debug(final String tag, final String msg, final Throwable throwable) {
+ debug(tag, msg);
+ throwable.printStackTrace(System.err);
+ }
+ public void warn(final String tag, final String msg) {
+ System.out.println("[WARN ][" + tag + "] " + msg);
+ }
+
+ public void warn(final String tag, final String msg, final Throwable t) {
+ warn(tag, msg);
+ t.printStackTrace(System.err);
+ }
+
+ public void error(final String tag, final String msg) {
+ System.err.println("[ERROR][" + tag + "] " + msg);
+ }
+
+ public void error(final String tag, final String msg, final Throwable t) {
+ System.err.println("[ERROR][" + tag + "] " + msg);
+ t.printStackTrace(System.err);
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java
new file mode 100644
index 0000000000..01a905d8aa
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/HttpUtils.java
@@ -0,0 +1,36 @@
+package org.schabi.newpipe.extractor.utils;
+
+import java.util.Arrays;
+
+import org.schabi.newpipe.extractor.downloader.Response;
+import org.schabi.newpipe.extractor.exceptions.HttpResponseException;
+
+public final class HttpUtils {
+
+ private HttpUtils() {
+ // Utility class, no instances allowed
+ }
+
+ // CHECKSTYLE:OFF
+ /**
+ * Validates the response codes for the given {@link Response}, and throws
+ * a {@link HttpResponseException} if the code is invalid
+ * @param response The response to validate
+ * @param validResponseCodes Expected valid response codes
+ * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes},
+ * or when {@code validResponseCodes} is empty and the code is a 4xx or 5xx error.
+ */
+ // CHECKSTYLE:ON
+ public static void validateResponseCode(final Response response,
+ final int... validResponseCodes)
+ throws HttpResponseException {
+ final int code = response.responseCode();
+ final var throwError = (validResponseCodes == null || validResponseCodes.length == 0)
+ ? code >= 400 && code <= 599
+ : Arrays.stream(validResponseCodes).noneMatch(c -> c == code);
+
+ if (throwError) {
+ throw new HttpResponseException(response);
+ }
+ }
+}
diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Logger.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Logger.java
new file mode 100644
index 0000000000..c7cd9db49c
--- /dev/null
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/Logger.java
@@ -0,0 +1,10 @@
+package org.schabi.newpipe.extractor.utils;
+
+public interface Logger {
+ void debug(String tag, String message);
+ void debug(String tag, String message, Throwable throwable);
+ void warn(String tag, String message);
+ void warn(String tag, String message, Throwable throwable);
+ void error(String tag, String message);
+ void error(String tag, String message, Throwable t);
+}
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/LoggerExtension.java b/extractor/src/test/java/org/schabi/newpipe/extractor/LoggerExtension.java
new file mode 100644
index 0000000000..7ef509cebc
--- /dev/null
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/LoggerExtension.java
@@ -0,0 +1,16 @@
+package org.schabi.newpipe.extractor;
+
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
+
+public class LoggerExtension implements BeforeAllCallback {
+ private static boolean set = false;
+
+ @Override
+ public void beforeAll(ExtensionContext context) {
+ if (set) return;
+ set = true;
+ ExtractorLogger.setLogger(new ExtractorLogger.ConsoleLogger());
+ }
+}
\ No newline at end of file
diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ExtractorLoggerTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ExtractorLoggerTest.java
new file mode 100644
index 0000000000..3530e696ab
--- /dev/null
+++ b/extractor/src/test/java/org/schabi/newpipe/extractor/utils/ExtractorLoggerTest.java
@@ -0,0 +1,144 @@
+package org.schabi.newpipe.extractor.utils;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class ExtractorLoggerTest {
+
+ private CapturingLogger logger;
+
+ @BeforeEach
+ void setup() {
+ logger = new CapturingLogger();
+ ExtractorLogger.setLogger(logger);
+ }
+
+ @Test
+ void replacesSinglePlaceholder() {
+ ExtractorLogger.d("T", "Hello {Name}", "Alice");
+ assertEquals("Hello Alice", logger.lastDebug);
+ }
+
+ @Test
+ void replacesMultiplePlaceholdersSequentially() {
+ ExtractorLogger.d("T", "A={A} B={B} C={C}", 1, 2, 3);
+ assertEquals("A=1 B=2 C=3", logger.lastDebug);
+ }
+
+ @Test
+ void leavesExtraPlaceholdersWhenNotEnoughArgs() {
+ ExtractorLogger.d("T", "First={F} Second={S} Third={T}", "X", "Y");
+ assertEquals("First=X Second=Y Third={T}", logger.lastDebug);
+ }
+
+ @Test
+ void ignoresExtraArgs() {
+ ExtractorLogger.d("T", "Only {One}", "X", "Y", "Z");
+ assertEquals("Only X", logger.lastDebug);
+ }
+
+ @Test
+ void noArgsReturnsTemplateUnchanged() {
+ ExtractorLogger.d("T", "No placeholders {} here");
+ assertEquals("No placeholders {} here", logger.lastDebug);
+ }
+
+ @Test
+ void nullTemplatePrintsNull() {
+ ExtractorLogger.d("T", (String) null, "X");
+ assertNull(logger.lastDebug);
+ }
+
+ @Test
+ void unmatchedBraceLeavesRemainder() {
+ ExtractorLogger.d("T", "Value {Unclosed", "X");
+ assertEquals("Value {Unclosed", logger.lastDebug);
+ }
+
+ @Test
+ void debugFormatWithThrowable() {
+ RuntimeException ex = new RuntimeException("boom");
+ ExtractorLogger.d("T", ex, "Failure {Code} at {Step}", 500, "init");
+ assertEquals("Failure 500 at init", logger.lastDebug);
+ assertSame(ex, logger.lastDebugThrowable);
+ }
+
+ @Test
+ void warnFormatWithThrowable() {
+ IllegalStateException ex = new IllegalStateException("warned");
+ ExtractorLogger.w("T", ex, "Warn {What}", "disk");
+ assertEquals("Warn disk", logger.lastWarn);
+ assertSame(ex, logger.lastWarnThrowable);
+ }
+
+ @Test
+ void errorFormatWithThrowable() {
+ Exception ex = new Exception("fatal");
+ ExtractorLogger.e("T", ex, "Error {Type} code={Code}", "IO", 42);
+ assertEquals("Error IO code=42", logger.lastError);
+ assertSame(ex, logger.lastErrorThrowable);
+ }
+
+ @Test
+ void debugFormatWithThrowableNotEnoughArgsLeavesPlaceholder() {
+ RuntimeException ex = new RuntimeException("x");
+ ExtractorLogger.d("T", ex, "Only one {A} and leftover {B}", "arg1");
+ assertEquals("Only one arg1 and leftover {B}", logger.lastDebug);
+ assertSame(ex, logger.lastDebugThrowable);
+ }
+
+ @Test
+ void errorFormatWithThrowableExtraArgsIgnored() {
+ Exception ex = new Exception("x");
+ ExtractorLogger.e("T", ex, "Val {V}", 10, 20, 30);
+ assertEquals("Val 10", logger.lastError);
+ assertSame(ex, logger.lastErrorThrowable);
+ }
+
+ private static final class CapturingLogger implements Logger {
+ String lastDebug;
+ Throwable lastDebugThrowable;
+ String lastWarn;
+ Throwable lastWarnThrowable;
+ String lastError;
+ Throwable lastErrorThrowable;
+
+ @Override
+ public void debug(String tag, String message) {
+ lastDebug = message;
+ lastDebugThrowable = null;
+ }
+
+ @Override
+ public void debug(String tag, String message, Throwable throwable) {
+ lastDebug = message;
+ lastDebugThrowable = throwable;
+ }
+
+ @Override
+ public void warn(String tag, String message) {
+ lastWarn = message;
+ lastWarnThrowable = null;
+ }
+
+ @Override
+ public void warn(String tag, String message, Throwable throwable) {
+ lastWarn = message;
+ lastWarnThrowable = throwable;
+ }
+
+ @Override
+ public void error(String tag, String message) {
+ lastError = message;
+ lastErrorThrowable = null;
+ }
+
+ @Override
+ public void error(String tag, String message, Throwable t) {
+ lastError = message;
+ lastErrorThrowable = t;
+ }
+ }
+}
\ No newline at end of file
diff --git a/extractor/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension b/extractor/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension
new file mode 100644
index 0000000000..ad03392dbd
--- /dev/null
+++ b/extractor/src/test/resources/META-INF/services/org.junit.jupiter.api.extension.Extension
@@ -0,0 +1 @@
+org.schabi.newpipe.extractor.LoggerExtension
\ No newline at end of file
diff --git a/extractor/src/test/resources/junit-platform.properties b/extractor/src/test/resources/junit-platform.properties
new file mode 100644
index 0000000000..1cebb76d5a
--- /dev/null
+++ b/extractor/src/test/resources/junit-platform.properties
@@ -0,0 +1 @@
+junit.jupiter.extensions.autodetection.enabled = true
\ No newline at end of file