[SoundCloud] Fix SoundCloud HLS expiry

extractor/src/main/java/org/schabi/newpipe/extractor/Extractor.java

@@ -7,6 +7,7 @@
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.localization.TimeAgoParser;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -15,6 +16,8 @@
 import java.util.Objects;
 
 public abstract class Extractor {
+    private final String TAG = getClass().getSimpleName() + "@" + hashCode();
+
     /**
      * {@link StreamingService} currently related to this extractor.<br>
      * Useful for getting other things from a service (like the url handlers for
@@ -54,7 +57,9 @@ public LinkHandler getLinkHandler() {
      * @throws ExtractionException if the pages content is not understood
      */
     public void fetchPage() throws IOException, ExtractionException {
+        ExtractorLogger.d(TAG, "base fetchPage called");
         if (pageFetched) {
+            ExtractorLogger.d(TAG, "Page already fetched; returning");
             return;
         }
         onFetchPage(downloader);
@@ -151,4 +156,9 @@ public ContentCountry getExtractorContentCountry() {
     public TimeAgoParser getTimeAgoParser() {
         return getService().getTimeAgoParser(getExtractorLocalization());
     }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/Info.java

@@ -2,6 +2,7 @@
 
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
 
 import java.io.Serializable;
 import java.util.ArrayList;
@@ -10,6 +11,7 @@
 
 public abstract class Info implements Serializable {
 
+    private static final String TAG = "Info";
     private final int serviceId;
     /**
      * Id of this Info object <br>
@@ -52,6 +54,7 @@ public Info(final int serviceId,
         this.url = url;
         this.originalUrl = originalUrl;
         this.name = name;
+        ExtractorLogger.d(TAG, "Base Created {}", this);
     }
 
     public Info(final int serviceId, final LinkHandler linkHandler, final String name) {

extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java

@@ -24,6 +24,7 @@
 import org.schabi.newpipe.extractor.exceptions.ExtractionException;
 import org.schabi.newpipe.extractor.localization.ContentCountry;
 import org.schabi.newpipe.extractor.localization.Localization;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
 
 import java.util.List;
 
@@ -34,6 +35,7 @@
  * Provides access to streaming services supported by NewPipe.
  */
 public final class NewPipe {
+    private static final String TAG = NewPipe.class.getSimpleName();
     private static Downloader downloader;
     private static Localization preferredLocalization;
     private static ContentCountry preferredContentCountry;
@@ -42,15 +44,19 @@ private NewPipe() {
     }
 
     public static void init(final Downloader d) {
+        ExtractorLogger.d(TAG, "Default init called");
         init(d, Localization.DEFAULT);
     }
 
     public static void init(final Downloader d, final Localization l) {
+        ExtractorLogger.d(TAG, "Default init called with localization");
         init(d, l, l.getCountryCode().isEmpty()
                 ? ContentCountry.DEFAULT : new ContentCountry(l.getCountryCode()));
     }
 
     public static void init(final Downloader d, final Localization l, final ContentCountry c) {
+        ExtractorLogger.d(TAG, "Initializing with downloader={}, localization={}, country={}",
+                          d, l, c);
         downloader = d;
         preferredLocalization = l;
         preferredContentCountry = c;

extractor/src/main/java/org/schabi/newpipe/extractor/StreamingServiceId.java

@@ -0,0 +1,23 @@
+package org.schabi.newpipe.extractor;
+
+import java.util.Objects;
+
+public enum StreamingServiceId {
+    NO_SERVICE_ID,
+    YOUTUBE,
+    SOUNDCLOUD,
+    MEDIACCC,
+    PEERTUBE,
+    BANDCAMP;
+
+
+    private static final StreamingServiceId[] VALUES = values();
+
+    public static String nameFromId(final int serviceId) {
+        try {
+            return VALUES[Objects.checkIndex(serviceId + 1, VALUES.length)].name();
+        } catch (final IndexOutOfBoundsException e) {
+            throw new IllegalArgumentException("Invalid serviceId: " + serviceId, e);
+        }
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java

@@ -243,4 +243,9 @@ public Response postWithContentTypeJson(final String url,
      */
     public abstract Response execute(@Nonnull Request request)
             throws IOException, ReCaptchaException;
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Response.java

@@ -6,6 +6,9 @@
 import java.util.List;
 import java.util.Map;
 
+import org.schabi.newpipe.extractor.exceptions.HttpResponseException;
+import org.schabi.newpipe.extractor.utils.HttpUtils;
+
 /**
  * A Data class used to hold the results from requests made by the Downloader implementation.
  */
@@ -80,4 +83,21 @@ public String getHeader(final String name) {
 
         return null;
     }
+    // CHECKSTYLE:OFF
+    /**
+     * Helper function simply to make it easier to validate response code inline
+     * before getting the code/body/latestUrl/etc.
+     * Validates the response codes for the given {@link Response}, and throws a {@link HttpResponseException} if the code is invalid
+     * @see HttpUtils#validateResponseCode(Response, int...)
+     * @param validResponseCodes Expected valid response codes
+     * @return {@code this} response
+     * @throws HttpResponseException Thrown when the response code is not in {@code validResponseCodes},
+     * or when {@code  validResponseCodes} is empty and the code is a 4xx or 5xx error.
+     */
+    // CHECKSTYLE:ON
+    public Response validateResponseCode(final int... validResponseCodes)
+        throws HttpResponseException {
+        HttpUtils.validateResponseCode(this, validResponseCodes);
+        return this;
+    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/exceptions/HttpResponseException.java

@@ -0,0 +1,15 @@
+package org.schabi.newpipe.extractor.exceptions;
+
+import java.io.IOException;
+import org.schabi.newpipe.extractor.downloader.Response;
+
+public class HttpResponseException extends IOException {
+    public HttpResponseException(final Response response) {
+        this("Error in HTTP Response for " + response.latestUrl() + "\n\t"
+            + response.responseCode() + " - " + response.responseMessage());
+    }
+
+    public HttpResponseException(final String message) {
+        super(message);
+    }
+}

extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java

@@ -5,14 +5,14 @@
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
 import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
+import static org.schabi.newpipe.extractor.utils.HttpUtils.validateResponseCode;
 
 import com.grack.nanojson.JsonArray;
 import com.grack.nanojson.JsonObject;
 import com.grack.nanojson.JsonParser;
 import com.grack.nanojson.JsonParserException;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
 import org.schabi.newpipe.extractor.Image;
@@ -29,6 +29,7 @@
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudLikesInfoItemExtractor;
 import org.schabi.newpipe.extractor.services.soundcloud.extractors.SoundcloudStreamInfoItemExtractor;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
+import org.schabi.newpipe.extractor.utils.ExtractorLogger;
 import org.schabi.newpipe.extractor.utils.ImageSuffix;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Parser;
@@ -87,6 +88,7 @@ public final class SoundcloudParsingHelper {
     private static final List<ImageSuffix> VISUALS_IMAGE_SUFFIXES =
             List.of(new ImageSuffix("t1240x260", 1240, 260, MEDIUM),
                     new ImageSuffix("t2480x520", 2480, 520, MEDIUM));
+    public static final String TAG = SoundcloudParsingHelper.class.getSimpleName();
 
     private static String clientId;
     public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/";
@@ -100,13 +102,14 @@ private SoundcloudParsingHelper() {
 
     public static synchronized String clientId() throws ExtractionException, IOException {
         if (!isNullOrEmpty(clientId)) {
+            ExtractorLogger.d(TAG, "Returning clientId={clientId}", clientId);
             return clientId;
         }
 
         final Downloader dl = NewPipe.getDownloader();
 
-        final Response download = dl.get("https://soundcloud.com");
-        final String responseBody = download.responseBody();
+        final Response downloadResponse = dl.get("https://soundcloud.com").validateResponseCode();
+        final String responseBody = downloadResponse.responseBody();
         final String clientIdPattern = ",client_id:\"(.*?)\"";
 
         final Document doc = Jsoup.parse(responseBody);
@@ -117,12 +120,15 @@ public static synchronized String clientId() throws ExtractionException, IOExcep
 
         final var headers = Map.of("Range", List.of("bytes=0-50000"));
 
-        for (final Element element : possibleScripts) {
+        for (final var element : possibleScripts) {
             final String srcUrl = element.attr("src");
             if (!isNullOrEmpty(srcUrl)) {
                 try {
+                    ExtractorLogger.d(TAG, "Searching for clientId in {srcUrl}", srcUrl);
                     clientId = Parser.matchGroup1(clientIdPattern, dl.get(srcUrl, headers)
+                            .validateResponseCode()
                             .responseBody());
+                    ExtractorLogger.d(TAG, "Found clientId={clientId}", clientId);
                     return clientId;
                 } catch (final RegexException ignored) {
                     // Ignore it and proceed to try searching other script
@@ -149,13 +155,16 @@ public static DateWrapper parseDate(final String uploadDate) throws ParsingExcep
         }
     }
 
+     // CHECKSTYLE:OFF
     /**
-     * Call the endpoint "/resolve" of the API.<p>
+     * Call the endpoint "/resolve" of the API.
      * <p>
-     * See https://developers.soundcloud.com/docs/api/reference#resolve
+     * See https://web.archive.org/web/20170804051146/https://developers.soundcloud.com/docs/api/reference#resolve
      */
+     // CHECKSTYLE:ON
     public static JsonObject resolveFor(@Nonnull final Downloader downloader, final String url)
             throws IOException, ExtractionException {
+        ExtractorLogger.d(TAG, "resolveFor({url})", url);
         final String apiUrl = SOUNDCLOUD_API_V2_URL + "resolve"
                 + "?url=" + Utils.encodeUrlUtf8(url)
                 + "&client_id=" + clientId();
@@ -178,10 +187,11 @@ public static JsonObject resolveFor(@Nonnull final Downloader downloader, final
     public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOException,
             ReCaptchaException {
 
-        final String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
-                + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization()).responseBody();
-
-        return Jsoup.parse(response).select("link[rel=\"canonical\"]").first()
+        final var response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+                + Utils.encodeUrlUtf8(apiUrl), SoundCloud.getLocalization());
+        validateResponseCode(response);
+        final var responseBody = response.responseBody();
+        return Jsoup.parse(responseBody).select("link[rel=\"canonical\"]").first()
                 .attr("abs:href");
     }
 
@@ -190,6 +200,7 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc
      *
      * @return the resolved id
      */
+    // TODO: what makes this method different from the others? Don' they all return the same?
     public static String resolveIdWithWidgetApi(final String urlString) throws IOException,
             ParsingException {
         String fixedUrl = urlString;
@@ -225,9 +236,12 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc
             final String widgetUrl = "https://api-widget.soundcloud.com/resolve?url="
                     + Utils.encodeUrlUtf8(url.toString())
                     + "&format=json&client_id=" + SoundcloudParsingHelper.clientId();
-            final String response = NewPipe.getDownloader().get(widgetUrl,
-                    SoundCloud.getLocalization()).responseBody();
-            final JsonObject o = JsonParser.object().from(response);
+
+            final var response = NewPipe.getDownloader().get(widgetUrl,
+                    SoundCloud.getLocalization());
+
+            final var responseBody = response.validateResponseCode().responseBody();
+            final JsonObject o = JsonParser.object().from(responseBody);
             return String.valueOf(JsonUtils.getValue(o, "id"));
         } catch (final JsonParserException e) {
             throw new ParsingException("Could not parse JSON response", e);