diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5a3ef64..c45ee20 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.11, 3.12] + python-version: [3.9, 3.13] steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md index 435782b..4b94c57 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ When you download the actual images, you'll see that they're actually concatenat ![Grid of tiny 10x10 images showing the video's frames at a regular interval.](https://raw.githubusercontent.com/MarcAbonce/thumbframes_dl/master/docs/img/storyboard.webp) © Blender Foundation | [cloud.blender.org/spring](https://cloud.blender.org/films/spring) -I refer to this individual frames as thumbframes because I couldn't find a better, less confusing name. +I refer to this individual frames as thumbframes because I couldn't find a better, less confusing name. YouTube calls them storyboards, but that's technically a different thing. ## How to use Install the library: diff --git a/docs/extractors.md b/docs/extractors.md index 153bf13..6992936 100644 --- a/docs/extractors.md +++ b/docs/extractors.md @@ -1,124 +1,99 @@ # Table of Contents -* [extractors.\_base](#extractors._base) - * [ThumbFramesImage](#extractors._base.ThumbFramesImage) - * [get\_image](#extractors._base.ThumbFramesImage.get_image) - * [ThumbFramesFormat](#extractors._base.ThumbFramesFormat) - * [WebsiteFrames](#extractors._base.WebsiteFrames) - * [video\_id](#extractors._base.WebsiteFrames.video_id) - * [video\_url](#extractors._base.WebsiteFrames.video_url) - * [thumbframe\_formats](#extractors._base.WebsiteFrames.thumbframe_formats) - * [get\_thumbframe\_format](#extractors._base.WebsiteFrames.get_thumbframe_format) - * [download\_thumbframe\_info](#extractors._base.WebsiteFrames.download_thumbframe_info) - * [get\_thumbframes](#extractors._base.WebsiteFrames.get_thumbframes) - - -# extractors.\_base - - -## ThumbFramesImage Objects - -```python -class ThumbFramesImage(InfoExtractor) -``` - -Each ThumbFramesImage represents a single image with n_frames frames arranged in a cols*rows grid. -Note that different images may have different sizes and number of frames even if they're from the same video. - - -#### get\_image - -```python - | get_image() -> bytes -``` - -The raw image as bytes. -Raises an ExtractorError if download fails. - - -## ThumbFramesFormat Objects +* [WebsiteFrames](#thumbframes_dl.extractors.base.frames.WebsiteFrames) + * [video\_id](#thumbframes_dl.extractors.base.frames.WebsiteFrames.video_id) + * [video\_url](#thumbframes_dl.extractors.base.frames.WebsiteFrames.video_url) + * [thumbframe\_formats](#thumbframes_dl.extractors.base.frames.WebsiteFrames.thumbframe_formats) + * [get\_thumbframe\_format](#thumbframes_dl.extractors.base.frames.WebsiteFrames.get_thumbframe_format) + * [download\_thumbframe\_info](#thumbframes_dl.extractors.base.frames.WebsiteFrames.download_thumbframe_info) + * [get\_thumbframes](#thumbframes_dl.extractors.base.frames.WebsiteFrames.get_thumbframes) +* [ThumbFramesImage](#thumbframes_dl.extractors.base.image.ThumbFramesImage) + * [get\_image](#thumbframes_dl.extractors.base.image.ThumbFramesImage.get_image) +* [ThumbFramesFormat](#thumbframes_dl.extractors.base.format.ThumbFramesFormat) -```python -@total_ordering -class ThumbFramesFormat(object) -``` -Basic metadata to show the qualities of each set of ThumbFramesImages. -Useful when there's more than one list of images per video. -Can be compared and sorted to get the frames with the highest resolution. + - ## WebsiteFrames Objects ```python -class WebsiteFrames(abc.ABC, InfoExtractor) +class WebsiteFrames(abc.ABC, InfoExtractor) ``` Represents a video and contains its frames. A subclass of this class needs to be implemented for each supported website. - + + #### video\_id ```python - | @property - | @abc.abstractmethod - | video_id() -> str +@property +@abc.abstractmethod +def video_id() -> str ``` Any unique identifier for the video provided by the website. - + + #### video\_url ```python - | @property - | @abc.abstractmethod - | video_url() -> str +@property +@abc.abstractmethod +def video_url() -> str ``` The video's URL. If possible, this URL should be "normalized" to its most canonical form -and not a URL shortner, mirror, embedding or a URL with unnecessary query parameters. +and not a URL shortener, mirror, embedding or a URL with unnecessary query parameters. + + - #### thumbframe\_formats ```python - | @property - | thumbframe_formats() -> Optional[Sequence[ThumbFramesFormat]] +@property +def thumbframe_formats() -> Optional[Sequence[ThumbFramesFormat]] ``` Available thumbframe formats for the video. Sorted by highest resolution. - + + #### get\_thumbframe\_format ```python - | get_thumbframe_format(format_id: Optional[str] = None) -> Optional[ThumbFramesFormat] +def get_thumbframe_format( + format_id: Optional[str] = None) -> Optional[ThumbFramesFormat] ``` Get thumbframe format identified by format_id. Will return None if format_id is not found in video's thumbframe formats. If no format_id is passed, this will return the highest resolution thumbframe format. - + + #### download\_thumbframe\_info ```python - | @abc.abstractmethod - | download_thumbframe_info() -> Union[Dict[str, List[ThumbFramesImage]], List[ThumbFramesImage]] +@abc.abstractmethod +def download_thumbframe_info( +) -> Union[dict[str, list[ThumbFramesImage]], list[ThumbFramesImage]] ``` Get all the thumbframe's metadata from the video. The actual image files are downloaded later. If the page offers more than 1 thumbframe set (for example with different resolutions), then this method should return a dict so each set is listed separately. Otherwise, return a list. - + + #### get\_thumbframes ```python - | get_thumbframes(format_id: Optional[str] = None, lazy=True) -> List[ThumbFramesImage] +def get_thumbframes(format_id: Optional[str] = None, + lazy=True) -> list[ThumbFramesImage] ``` Get the video's ThumbFramesImages as a list. @@ -126,3 +101,42 @@ If a webpage has more than one thumbframe format, the format_id parameter needs knows which images to return. By default, the images are downloaded lazily until the image property is called for each object. If the lazy parameter is set to False, all the images will be downloaded right away. + + + +## ThumbFramesImage Objects + +```python +class ThumbFramesImage(InfoExtractor) +``` + +Each ThumbFramesImage represents a single image with n_frames frames arranged in a cols*rows grid. +Note that different images may have different sizes and number of frames even if they're from the same video. + + + +#### get\_image + +```python +def get_image() -> bytes +``` + +The raw image as bytes. + +Tries to download the image if it hasn't been already downloaded. + +:raises ExtractorError + + + +## ThumbFramesFormat Objects + +```python +@total_ordering +class ThumbFramesFormat(object) +``` + +Basic metadata to show the qualities of each set of ThumbFramesImages. +Useful when there's more than one list of images per video. +Can be compared and sorted to get the frames with the highest resolution. + diff --git a/docs/main.md b/docs/main.md index 5d2ef66..227affe 100644 --- a/docs/main.md +++ b/docs/main.md @@ -4,7 +4,7 @@ Each website handles thumbframes in a unique way, so for each supported website we need to implement a [WebsiteFrames](extractors.md#extractors._base.WebsiteFrames) subclass. ### YouTubeFrames -YouTube can return thumbframes images in different formats, such as: +YouTube can return thumbframes images (a.k.a. storyboards) in different formats, such as: * **L0**: A single small image with a 10x10 grid. * **L1**: A set of bigger images with a 10x10 grid each. * **L2**: A set of bigger images with a 5x5 grid each. diff --git a/requirements.txt b/requirements.txt index 7216ed8..4c56720 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -youtube_dl +youtube_dl==2021.12.17 diff --git a/test/test_youtube.py b/test/test_youtube.py index 7a46726..6581504 100644 --- a/test/test_youtube.py +++ b/test/test_youtube.py @@ -1,5 +1,6 @@ import re import os +import logging import unittest import httpretty # type: ignore @@ -22,6 +23,8 @@ class TestYouTubeFrames(unittest.TestCase): # Mock YoutubeDL's internal HTTP requests def setUp(self): + logging.disable(logging.CRITICAL) # comment out if needed for debugging a failed test + httpretty.reset() httpretty.enable(allow_net_connect=False) @@ -57,6 +60,7 @@ def setUp(self): def tearDown(self): httpretty.disable() + logging.disable(logging.NOTSET) # Assert that ThumbFramesImage objects look reasonably well def assertThumbFrames(self, tf_images): diff --git a/thumbframes_dl/extractors/base/format.py b/thumbframes_dl/extractors/base/format.py index 221ef8d..c271afd 100644 --- a/thumbframes_dl/extractors/base/format.py +++ b/thumbframes_dl/extractors/base/format.py @@ -19,20 +19,24 @@ def __init__(self, format_id: Optional[str], thumbframes: List[ThumbFramesImage] self.total_frames = reduce(lambda acum, x: acum + x.n_frames, thumbframes, 0) self.total_images = len(thumbframes) - def __hash__(self): + def __hash__(self) -> int: return hash(self.format_id) @property - def frame_size(self): + def frame_size(self) -> int: return self.frame_width * self.frame_height - def __eq__(self, other): + def __eq__(self, other) -> bool: + if not isinstance(other, ThumbFramesFormat): + return NotImplemented return self.frame_size == other.frame_size - def __lt__(self, other): + def __lt__(self, other) -> bool: + if not isinstance(other, ThumbFramesFormat): + return NotImplemented return self.frame_size < other.frame_size - def __repr__(self): + def __repr__(self) -> str: return "<%s %s: %s %sx%s frames in %s images>" % ( self.__class__.__name__, self.format_id, self.total_frames, self.frame_width, self.frame_height, self.total_images diff --git a/thumbframes_dl/extractors/base/frames.py b/thumbframes_dl/extractors/base/frames.py index e0eaf86..e642aee 100644 --- a/thumbframes_dl/extractors/base/frames.py +++ b/thumbframes_dl/extractors/base/frames.py @@ -1,5 +1,5 @@ import abc -from typing import Dict, List, Optional, Sequence, Union +from typing import Optional, Sequence, Union from youtube_dl.YoutubeDL import YoutubeDL from youtube_dl.extractor.common import InfoExtractor @@ -25,8 +25,9 @@ def __init__(self, video_url: str): @abc.abstractmethod def _validate(self) -> None: """ - Method that validates that self._input_url is a valid URL or id for this website. - If not, an ExtractorError should be thrown here. + Validates that self._input_url is a valid URL or id for this website. + + :raises ExtractorError """ pass @@ -44,7 +45,7 @@ def video_url(self) -> str: """ The video's URL. If possible, this URL should be "normalized" to its most canonical form - and not a URL shortner, mirror, embedding or a URL with unnecessary query parameters. + and not a URL shortener, mirror, embedding or a URL with unnecessary query parameters. """ pass @@ -82,7 +83,7 @@ def get_thumbframe_format(self, format_id: Optional[str] = None) -> Optional[Thu return None @abc.abstractmethod - def download_thumbframe_info(self) -> Union[Dict[str, List[ThumbFramesImage]], List[ThumbFramesImage]]: + def download_thumbframe_info(self) -> Union[dict[str, list[ThumbFramesImage]], list[ThumbFramesImage]]: """ Get all the thumbframe's metadata from the video. The actual image files are downloaded later. If the page offers more than 1 thumbframe set (for example with different resolutions), @@ -90,7 +91,7 @@ def download_thumbframe_info(self) -> Union[Dict[str, List[ThumbFramesImage]], L """ pass - def get_thumbframes(self, format_id: Optional[str] = None, lazy=True) -> List[ThumbFramesImage]: + def get_thumbframes(self, format_id: Optional[str] = None, lazy=True) -> list[ThumbFramesImage]: """ Get the video's ThumbFramesImages as a list. If a webpage has more than one thumbframe format, the format_id parameter needs to be set so this method @@ -114,7 +115,7 @@ def get_thumbframes(self, format_id: Optional[str] = None, lazy=True) -> List[Th return thumbframes_list - def __repr__(self): + def __repr__(self) -> str: return "<%s %s>" % ( self.__class__.__name__, self.video_id ) diff --git a/thumbframes_dl/extractors/base/image.py b/thumbframes_dl/extractors/base/image.py index 4c0b6c0..edf99f0 100644 --- a/thumbframes_dl/extractors/base/image.py +++ b/thumbframes_dl/extractors/base/image.py @@ -20,13 +20,15 @@ def __init__(self, url: str, width: int, height: int, cols: int, rows: int, n_fr self.cols = cols self.rows = rows self.n_frames = n_frames - self.mime_type = None + self.mime_type: Optional[str] = None self._image: Optional[bytes] = None def get_image(self) -> bytes: """ The raw image as bytes. - Raises an ExtractorError if download fails. + Tries to download the image if it hasn't been already downloaded. + + :raises ExtractorError """ if self._image is None: resp = self._request_webpage(self.url, self.url, fatal=True) @@ -35,7 +37,7 @@ def get_image(self) -> bytes: self._image = raw_image return self._image - def __repr__(self): + def __repr__(self) -> str: return "<%s: %sx%s image in a %sx%s grid>" % ( self.__class__.__name__, self.width, self.height, self.cols, self.rows ) diff --git a/thumbframes_dl/extractors/youtube.py b/thumbframes_dl/extractors/youtube.py index 960560b..b637ed2 100644 --- a/thumbframes_dl/extractors/youtube.py +++ b/thumbframes_dl/extractors/youtube.py @@ -1,5 +1,6 @@ import math -from typing import Dict, List + +from typing import Optional from youtube_dl.utils import try_get, int_or_none from youtube_dl.extractor.youtube import YoutubeIE @@ -10,23 +11,41 @@ class YouTubeFrames(WebsiteFrames, YoutubeIE): + """ + Extracts thumbframes (a.k.a. storyboards) from a YouTube video. + + YouTube can return thumbframes images in different formats, such as: + + * L0: A single small image with a 10x10 grid. + * L1: A set of bigger images with a 10x10 grid each. + * L2: A set of bigger images with a 5x5 grid each. + + The image sizes may vary per video. + Also, a video doesn't necessarily contain images in all the formats. + """ + _YOUTUBE_URL = 'https://www.youtube.com' _VIDEO_WEBPAGE_URL = _YOUTUBE_URL + '/watch?v={VIDEO_ID}' _VIDEO_INFO_URL = _YOUTUBE_URL + '/get_video_info?video_id={VIDEO_ID}&el=detailpage' - def _validate(self): + def _validate(self) -> None: + """:raises ExtractorError""" self._video_id = YoutubeIE.extract_id(self._input_url) @property - def video_id(self): + def video_id(self) -> str: return self._video_id @property - def video_url(self): + def video_url(self) -> str: return self._VIDEO_WEBPAGE_URL.format(VIDEO_ID=self.video_id) - # Try to extract storyboard spec from player_response object - def _get_storyboard_spec(self): + def _get_storyboard_spec(self) -> Optional[str]: + """ + Tries to extract storyboard spec from player_response object. + Storyboard spec is downloaded from video's page or an API endpoint. + """ + video_id = self.video_id webpage_url = self._VIDEO_WEBPAGE_URL.format(VIDEO_ID=video_id) @@ -47,9 +66,15 @@ def _get_storyboard_spec(self): lambda x: x['storyboards']['playerStoryboardSpecRenderer']['spec'], str) - # Extract information of each storyboard - def _get_storyboards_from_spec(self, video_id, sb_spec): - storyboards = dict() + return None # storyboard spec not found anywhere in page + + def _get_storyboards_from_spec(self, sb_spec: str) -> dict[str, list[ThumbFramesImage]]: + """ + Tries to extract information for each storyboard + by parsing the extracted storyboard spec. + """ + + storyboards: dict[str, list[ThumbFramesImage]] = {} s_parts = sb_spec.split('|') base_url = s_parts[0] @@ -75,8 +100,8 @@ def _get_storyboards_from_spec(self, video_id, sb_spec): logger.warning('Unable to extract thumbframe from spec {}'.format(params)) continue + storyboard_set: list[ThumbFramesImage] = [] storyboards_url = base_url.replace('$L', str(i)) + '&' - storyboard_set = [] for j in range(n_images): url = storyboards_url.replace('$N', filename).replace('$M', str(j)) + 'sigh=' + sigh if j == n_images - 1: @@ -102,10 +127,10 @@ def _get_storyboards_from_spec(self, video_id, sb_spec): return storyboards - def download_thumbframe_info(self) -> Dict[str, List[ThumbFramesImage]]: + def download_thumbframe_info(self) -> dict[str, list[ThumbFramesImage]]: sb_spec = self._get_storyboard_spec() if not sb_spec: logger.warning('Could not find thumbframes for video {}'.format(self.video_id)) return dict() - return self._get_storyboards_from_spec(self.video_id, sb_spec) + return self._get_storyboards_from_spec(sb_spec) diff --git a/thumbframes_dl/version.py b/thumbframes_dl/version.py index 9e78220..9da2f8f 100644 --- a/thumbframes_dl/version.py +++ b/thumbframes_dl/version.py @@ -1 +1 @@ -__version__ = "0.14.0" +__version__ = "0.15.0"