diff --git a/base/base_crawler.py b/base/base_crawler.py index 4fe9eaa26..419ebe7a4 100644 --- a/base/base_crawler.py +++ b/base/base_crawler.py @@ -116,6 +116,12 @@ async def store_video(self, video_content_item: Dict): pass +class AbstractStoreAudio(ABC): + # TODO: support all platform + async def store_audio(self, audio_content_item: Dict): + pass + + class AbstractApiClient(ABC): @abstractmethod diff --git a/config/base_config.py b/config/base_config.py index 28a852e08..b95ffba83 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -107,6 +107,17 @@ # Whether to enable crawling media mode (including image or video resources), crawling media is not enabled by default ENABLE_GET_MEIDAS = False +# Whether to enable audio download/extraction mode. +# When True, the crawler will automatically extract audio from downloaded videos and save it as a separate file. +ENABLE_GET_AUDIO = False + +# Audio output format, supports: mp3, aac +AUDIO_FORMAT = "mp3" + +# Whether to keep the original video file after extracting audio. +# Set to False to delete the video and keep only the audio, saving disk space. +KEEP_ORIGINAL_VIDEO = True + # Whether to enable comment crawling mode. Comment crawling is enabled by default. ENABLE_GET_COMMENTS = True diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index 14b509977..7caf6f0a9 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -607,6 +607,11 @@ async def get_bilibili_video(self, video_item: Dict, semaphore: asyncio.Semaphor extension_file_name = f"video.mp4" await bilibili_store.store_video(aid, content, extension_file_name) + if config.ENABLE_GET_AUDIO: + video_path = f"{bilibili_store.BilibiliVideo().video_store_path}/{aid}/{extension_file_name}" + audio_extension_file_name = f"{extension_file_name.rsplit('.', 1)[0]}.{config.AUDIO_FORMAT}" + await bilibili_store.store_audio(aid, video_path, audio_extension_file_name) + async def get_all_creator_details(self, creator_url_list: List[str]): """ creator_url_list: get details for creator from creator URL list diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index c0e9372aa..03192f12d 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -468,3 +468,8 @@ async def get_aweme_video(self, aweme_item: Dict): return extension_file_name = f"video.mp4" await douyin_store.update_dy_aweme_video(aweme_id, content, extension_file_name) + + if config.ENABLE_GET_AUDIO: + video_path = f"{douyin_store.DouYinVideo().video_store_path}/{aweme_id}/{extension_file_name}" + audio_extension_file_name = f"{extension_file_name.rsplit('.', 1)[0]}.{config.AUDIO_FORMAT}" + await douyin_store.update_dy_aweme_audio(aweme_id, video_path, audio_extension_file_name) diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py index 334fef395..f6863d507 100644 --- a/media_platform/xhs/core.py +++ b/media_platform/xhs/core.py @@ -520,3 +520,8 @@ async def get_notice_video(self, note_item: Dict): extension_file_name = f"{videoNum}.mp4" videoNum += 1 await xhs_store.update_xhs_note_video(note_id, content, extension_file_name) + + if config.ENABLE_GET_AUDIO: + video_path = f"{xhs_store.XiaoHongShuVideo().video_store_path}/{note_id}/{extension_file_name}" + audio_extension_file_name = f"{extension_file_name.rsplit('.', 1)[0]}.{config.AUDIO_FORMAT}" + await xhs_store.update_xhs_note_audio(note_id, video_path, audio_extension_file_name) diff --git a/store/bilibili/__init__.py b/store/bilibili/__init__.py index b5eb9215a..f34eaf5e9 100644 --- a/store/bilibili/__init__.py +++ b/store/bilibili/__init__.py @@ -30,6 +30,7 @@ from ._store_impl import * from .bilibilli_store_media import * +from .bilibili_store_audio import * class BiliStoreFactory: @@ -131,6 +132,24 @@ async def store_video(aid, video_content, extension_file_name): }) +async def store_audio(aid, video_path, extension_file_name): + """ + Extract and save Bilibili video audio from a downloaded video. + Args: + aid: + video_path: + extension_file_name: + + Returns: + Path to the extracted audio file, or None if failed. + """ + return await BilibiliAudio().store_audio({ + "aid": aid, + "video_path": video_path, + "extension_file_name": extension_file_name, + }) + + async def batch_update_bilibili_creator_fans(creator_info: Dict, fans_list: List[Dict]): # 教学版：不再采集/存储粉丝列表(其他用户的个人信息)，防骚扰。 return diff --git a/store/bilibili/bilibili_store_audio.py b/store/bilibili/bilibili_store_audio.py new file mode 100644 index 000000000..b55554755 --- /dev/null +++ b/store/bilibili/bilibili_store_audio.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/bilibili/bilibili_store_audio.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对目标平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from pathlib import Path +from typing import Dict, Optional + +from base.base_crawler import AbstractStoreAudio +from tools import utils +from tools.media_util import extract_audio_from_video +import config + + +class BilibiliAudio(AbstractStoreAudio): + def __init__(self): + if config.SAVE_DATA_PATH: + self.audio_store_path = f"{config.SAVE_DATA_PATH}/bili/audios" + else: + self.audio_store_path = "data/bili/audios" + + def make_save_file_name(self, aid: str, extension_file_name: str) -> str: + """ + make save file name by store type + + Args: + aid: aid + extension_file_name: audio filename with extension + + Returns: + + """ + return f"{self.audio_store_path}/{aid}/{extension_file_name}" + + async def store_audio(self, audio_content_item: Dict) -> Optional[str]: + """ + Extract audio from a downloaded video and save it to the audio store path. + + Args: + audio_content_item: dict with keys: + - aid: aid + - video_path: path to the downloaded video file + - extension_file_name: audio filename with extension + + Returns: + Path to the extracted audio file, or None if failed. + """ + aid: str = str(audio_content_item.get("aid")) + video_path: str = audio_content_item.get("video_path") + extension_file_name: str = audio_content_item.get("extension_file_name") + + if not video_path or not Path(video_path).exists(): + utils.logger.warning(f"[BilibiliAudio.store_audio] Video not found: {video_path}") + return None + + Path(self.audio_store_path + "/" + aid).mkdir(parents=True, exist_ok=True) + audio_path = self.make_save_file_name(aid, extension_file_name) + + return await extract_audio_from_video( + video_path=video_path, + audio_path=audio_path, + audio_format=config.AUDIO_FORMAT, + ) diff --git a/store/douyin/__init__.py b/store/douyin/__init__.py index a1e920737..e875d9305 100644 --- a/store/douyin/__init__.py +++ b/store/douyin/__init__.py @@ -29,6 +29,7 @@ from ._store_impl import * from .douyin_store_media import * +from .douyin_store_audio import * class DouyinStoreFactory: @@ -249,3 +250,18 @@ async def update_dy_aweme_video(aweme_id, video_content, extension_file_name): """ await DouYinVideo().store_video({"aweme_id": aweme_id, "video_content": video_content, "extension_file_name": extension_file_name}) + + +async def update_dy_aweme_audio(aweme_id, video_path, extension_file_name): + """ + Extract and save Douyin aweme audio from a downloaded video. + Args: + aweme_id: + video_path: + extension_file_name: + + Returns: + Path to the extracted audio file, or None if failed. + """ + + return await DouYinAudio().store_audio({"aweme_id": aweme_id, "video_path": video_path, "extension_file_name": extension_file_name}) diff --git a/store/douyin/douyin_store_audio.py b/store/douyin/douyin_store_audio.py new file mode 100644 index 000000000..7c10588e3 --- /dev/null +++ b/store/douyin/douyin_store_audio.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/douyin/douyin_store_audio.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对目标平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from pathlib import Path +from typing import Dict, Optional + +from base.base_crawler import AbstractStoreAudio +from tools import utils +from tools.media_util import extract_audio_from_video +import config + + +class DouYinAudio(AbstractStoreAudio): + def __init__(self): + if config.SAVE_DATA_PATH: + self.audio_store_path = f"{config.SAVE_DATA_PATH}/douyin/audios" + else: + self.audio_store_path = "data/douyin/audios" + + def make_save_file_name(self, aweme_id: str, extension_file_name: str) -> str: + """ + make save file name by store type + + Args: + aweme_id: aweme id + extension_file_name: audio filename with extension + + Returns: + + """ + return f"{self.audio_store_path}/{aweme_id}/{extension_file_name}" + + async def store_audio(self, audio_content_item: Dict) -> Optional[str]: + """ + Extract audio from a downloaded video and save it to the audio store path. + + Args: + audio_content_item: dict with keys: + - aweme_id: aweme id + - video_path: path to the downloaded video file + - extension_file_name: audio filename with extension + + Returns: + Path to the extracted audio file, or None if failed. + """ + aweme_id: str = audio_content_item.get("aweme_id") + video_path: str = audio_content_item.get("video_path") + extension_file_name: str = audio_content_item.get("extension_file_name") + + if not video_path or not Path(video_path).exists(): + utils.logger.warning(f"[DouYinAudio.store_audio] Video not found: {video_path}") + return None + + Path(self.audio_store_path + "/" + aweme_id).mkdir(parents=True, exist_ok=True) + audio_path = self.make_save_file_name(aweme_id, extension_file_name) + + return await extract_audio_from_video( + video_path=video_path, + audio_path=audio_path, + audio_format=config.AUDIO_FORMAT, + ) diff --git a/store/xhs/__init__.py b/store/xhs/__init__.py index 5ef8e3c81..974679abb 100644 --- a/store/xhs/__init__.py +++ b/store/xhs/__init__.py @@ -28,6 +28,7 @@ from tools.user_hash import anonymize_user_id, mask_nickname from .xhs_store_media import * +from .xhs_store_audio import * from ._store_impl import * @@ -220,3 +221,18 @@ async def update_xhs_note_video(note_id, video_content, extension_file_name): """ await XiaoHongShuVideo().store_video({"notice_id": note_id, "video_content": video_content, "extension_file_name": extension_file_name}) + + +async def update_xhs_note_audio(note_id, video_path, extension_file_name): + """ + Extract and save Xiaohongshu note audio from a downloaded video. + Args: + note_id: + video_path: + extension_file_name: + + Returns: + Path to the extracted audio file, or None if failed. + """ + + return await XiaoHongShuAudio().store_audio({"notice_id": note_id, "video_path": video_path, "extension_file_name": extension_file_name}) diff --git a/store/xhs/xhs_store_audio.py b/store/xhs/xhs_store_audio.py new file mode 100644 index 000000000..0c9d5be7b --- /dev/null +++ b/store/xhs/xhs_store_audio.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/xhs/xhs_store_audio.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对目标平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +from pathlib import Path +from typing import Dict, Optional + +from base.base_crawler import AbstractStoreAudio +from tools import utils +from tools.media_util import extract_audio_from_video +import config + + +class XiaoHongShuAudio(AbstractStoreAudio): + def __init__(self): + if config.SAVE_DATA_PATH: + self.audio_store_path = f"{config.SAVE_DATA_PATH}/xhs/audios" + else: + self.audio_store_path = "data/xhs/audios" + + def make_save_file_name(self, notice_id: str, extension_file_name: str) -> str: + """ + make save file name by store type + + Args: + notice_id: notice id + extension_file_name: audio filename with extension + + Returns: + + """ + return f"{self.audio_store_path}/{notice_id}/{extension_file_name}" + + async def store_audio(self, audio_content_item: Dict) -> Optional[str]: + """ + Extract audio from a downloaded video and save it to the audio store path. + + Args: + audio_content_item: dict with keys: + - notice_id: note id + - video_path: path to the downloaded video file + - extension_file_name: audio filename with extension + + Returns: + Path to the extracted audio file, or None if failed. + """ + notice_id: str = audio_content_item.get("notice_id") + video_path: str = audio_content_item.get("video_path") + extension_file_name: str = audio_content_item.get("extension_file_name") + + if not video_path or not Path(video_path).exists(): + utils.logger.warning(f"[XiaoHongShuAudio.store_audio] Video not found: {video_path}") + return None + + Path(self.audio_store_path + "/" + notice_id).mkdir(parents=True, exist_ok=True) + audio_path = self.make_save_file_name(notice_id, extension_file_name) + + return await extract_audio_from_video( + video_path=video_path, + audio_path=audio_path, + audio_format=config.AUDIO_FORMAT, + ) diff --git a/tools/media_util.py b/tools/media_util.py new file mode 100644 index 000000000..b28367aea --- /dev/null +++ b/tools/media_util.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tools/media_util.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对目标平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +import asyncio +import os +import shutil +from pathlib import Path +from typing import Optional + +import config +from tools import utils + + +def _find_ffmpeg() -> Optional[str]: + """Find ffmpeg executable, with common Windows install paths fallback.""" + ffmpeg_cmd = shutil.which("ffmpeg") + if ffmpeg_cmd: + return ffmpeg_cmd + + # Common Windows winget install path for Gyan.FFmpeg + user_profile = os.environ.get("USERPROFILE", "") + candidates = [ + r"C:\ffmpeg\bin\ffmpeg.exe", + r"C:\ProgramData\chocolatey\bin\ffmpeg.exe", + ] + if user_profile: + base = Path(user_profile) / "AppData" / "Local" / "Microsoft" / "WinGet" / "Packages" + if base.exists(): + for folder in base.glob("Gyan.FFmpeg*"): + candidate = folder / "ffmpeg-*-full_build" / "bin" / "ffmpeg.exe" + candidates.extend(sorted(candidate.parent.glob("ffmpeg.exe"), reverse=True)) + + for candidate in candidates: + if Path(candidate).exists(): + return str(candidate) + return None + + +async def extract_audio_from_video( + video_path: str, + audio_path: Optional[str] = None, + audio_format: Optional[str] = None, + sample_rate: int = 16000, + channels: int = 1, +) -> Optional[str]: + """ + Extract audio from a video file using ffmpeg. + + Args: + video_path: Path to the source video file. + audio_path: Path to save the extracted audio. If None, inferred from video_path. + audio_format: Audio output format, e.g. 'mp3' or 'aac'. Defaults to config.AUDIO_FORMAT. + sample_rate: Audio sample rate. Default 16000, suitable for ASR. + channels: Number of audio channels. Default 1 (mono). + + Returns: + The path of the extracted audio file, or None if extraction failed. + """ + video_file = Path(video_path) + if not video_file.exists(): + utils.logger.error(f"[media_util.extract_audio_from_video] Video file not found: {video_path}") + return None + + fmt = (audio_format or config.AUDIO_FORMAT).lower() + if fmt not in ("mp3", "aac"): + utils.logger.warning(f"[media_util.extract_audio_from_video] Unsupported audio format '{fmt}', falling back to mp3") + fmt = "mp3" + + if audio_path is None: + audio_path = str(video_file.with_suffix(f".{fmt}")) + + audio_file = Path(audio_path) + audio_file.parent.mkdir(parents=True, exist_ok=True) + + if fmt == "mp3": + codec = "libmp3lame" + else: + codec = "aac" + + ffmpeg_cmd = _find_ffmpeg() + if not ffmpeg_cmd: + utils.logger.error("[media_util.extract_audio_from_video] ffmpeg not found in PATH, please install ffmpeg first") + return None + + cmd = [ + ffmpeg_cmd, + "-y", + "-i", str(video_file), + "-vn", + "-ar", str(sample_rate), + "-ac", str(channels), + "-c:a", codec, + str(audio_file), + ] + + try: + utils.logger.info(f"[media_util.extract_audio_from_video] Extracting audio: {video_file.name} -> {audio_file.name}") + process = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await process.communicate() + if process.returncode != 0: + error_msg = stderr.decode("utf-8", errors="ignore")[-500:] + utils.logger.error(f"[media_util.extract_audio_from_video] ffmpeg failed: {error_msg}") + return None + + utils.logger.info(f"[media_util.extract_audio_from_video] Audio saved: {audio_file}") + + if not config.KEEP_ORIGINAL_VIDEO and video_file.exists(): + video_file.unlink() + utils.logger.info(f"[media_util.extract_audio_from_video] Deleted original video: {video_file}") + + return str(audio_file) + except Exception as e: + utils.logger.error(f"[media_util.extract_audio_from_video] Exception: {e}") + return None