diff --git a/python/02-use-cases/video_breakdown_agent/.gitignore b/python/02-use-cases/video_breakdown_agent/.gitignore new file mode 100644 index 00000000..4816631a --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.gitignore @@ -0,0 +1,37 @@ +# 密钥和运行时配置（绝不可提交） +# 注意：避免把任何 .env* 模板误提交（其中经常包含真实密钥/Token） +.env +.env.* +.env* + +# 媒体缓存（视频处理临时文件） +.media-cache/ +.media-uploads/ +.media-downloads/ + +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +dist/ +build/ +.pytest_cache/ + +# VeADK / ADK +.adk/ +.agentkit/ +.veadk-ignore +uv.lock +.uv-cache/ + +# 虚拟环境 +.venv/ +venv/ + +# IDE +.idea/ +.vscode/ +.DS_Store + +# 日志 +*.log diff --git a/python/02-use-cases/video_breakdown_agent/.scripts/smoke_test.py b/python/02-use-cases/video_breakdown_agent/.scripts/smoke_test.py new file mode 100644 index 00000000..67fc8630 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.scripts/smoke_test.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +""" +Video Breakdown Agent — 最小联调/冒烟测试脚本 + +用法（从项目根目录运行）： + # 交互式对话（默认） + uv run python .scripts/smoke_test.py + + # 直接发送一条消息 + uv run python .scripts/smoke_test.py "你好，介绍一下你的功能" + + # 运行 pipeline 回归用例 + uv run python .scripts/smoke_test.py --pipeline-cases + +依赖：需要在项目根目录下已有 config.yaml 或 .env 配置。 +""" + +import asyncio +import sys +import os +from pathlib import Path + +# 确保项目根目录在 Python 路径中 +PROJECT_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + + +async def run_single(message: str) -> str: + """发送单条消息并返回 Agent 最终输出""" + # 延迟导入，让 sys.path 先生效 + from agent import runner # noqa: E402 + + print(f"\n{'=' * 60}") + print(f"📤 发送: {message}") + print(f"{'=' * 60}") + + result = await runner.run( + messages=message, + user_id="smoke_test_user", + session_id="smoke_test_session", + ) + + print(f"\n{'=' * 60}") + print("📥 回复:") + print(f"{'=' * 60}") + print(result) + return result + + +async def run_interactive() -> None: + """交互式多轮对话""" + from agent import runner # noqa: E402 + + session_id = f"smoke_test_{os.getpid()}" + user_id = "smoke_test_user" + turn = 0 + + print("=" * 60) + print("Video Breakdown Agent — 交互式测试") + print("输入消息后回车发送，输入 q/quit/exit 退出") + print("=" * 60) + + while True: + try: + message = input("\n🧑 你: ").strip() + except (EOFError, KeyboardInterrupt): + print("\n👋 再见！") + break + + if not message: + continue + if message.lower() in ("q", "quit", "exit"): + print("👋 再见！") + break + + turn += 1 + print(f"\n⏳ 正在处理（第 {turn} 轮）...") + + try: + result = await runner.run( + messages=message, + user_id=user_id, + session_id=session_id, + ) + print(f"\n🤖 小视: {result}") + except Exception as e: + print(f"\n❌ 出错: {e}") + + +def _looks_like_raw_json(text: str) -> bool: + stripped = (text or "").strip() + return (stripped.startswith("{") and stripped.endswith("}")) or ( + stripped.startswith("[") and stripped.endswith("]") + ) + + +def _assert_case_output( + case_name: str, output: str, expected_keywords: list[str] +) -> None: + lowered = (output or "").lower() + if "<[plhd" in lowered or "transfer_to_agent" in lowered: + raise AssertionError(f"{case_name}: 检测到内部占位/转移片段泄露") + if _looks_like_raw_json(output): + raise AssertionError(f"{case_name}: 输出仍是原始 JSON") + if "我是search_agent" in (output or ""): + raise AssertionError(f"{case_name}: 对话未切回 root，仍由 search_agent 输出") + for keyword in expected_keywords: + if keyword not in output: + raise AssertionError(f"{case_name}: 未命中预期关键词 `{keyword}`") + + +async def run_pipeline_cases() -> None: + """本地/云端一致性回归用例（需要真实视频 URL 或本地测试视频）。""" + from agent import runner # noqa: E402 + + session_id = f"pipeline_case_{os.getpid()}" + user_id = "smoke_test_user" + + # 使用实际测试视频（如果项目内有 .media-uploads 中的测试样本） + test_video = os.getenv( + "TEST_VIDEO_URL", "https://tos-cn-beijing.volces.com/obj/video-demo/sample.mp4" + ) + + # 检查本地测试视频 + local_test_videos = list(Path(PROJECT_ROOT / ".media-uploads").glob("*.mp4")) + if local_test_videos: + test_video = str(local_test_videos[0]) + print(f"Using local test video: {test_video}") + + cases = [ + ( + "case1_full_pipeline", + f"请对这个视频做完整分析并给出报告：{test_video}", + ["钩子分析", "报告"], + ), + ( + "case2_hook_only", + f"请分析这个视频前三秒钩子：{test_video}", + ["前三秒钩子分析", "综合评分"], + ), + ( + "case3_greeting", + "你好，介绍一下你的功能", + ["分镜", "钩子", "报告"], + ), + ( + "case4_search_then_identity", + "搜一下杭州这两天天气，然后回答：你是谁？", + ["小视"], + ), + ] + + print("=" * 60) + print("Video Breakdown Agent — Pipeline Cases") + print("=" * 60) + print(f"Test video: {test_video}") + print(f"Session: {session_id}") + + passed = 0 + failed = 0 + + for case_name, message, expected_keywords in cases: + print(f"\n[RUN] {case_name}") + print(f"Input: {message[:80]}...") + try: + output = await runner.run( + messages=message, + user_id=user_id, + session_id=session_id, + ) + output_str = str(output) + print(f"Output preview: {output_str[:300]}...") + _assert_case_output(case_name, output_str, expected_keywords) + print(f"✅ [PASS] {case_name}") + passed += 1 + except Exception as e: + print(f"❌ [FAIL] {case_name}: {e}") + failed += 1 + + print("\n" + "=" * 60) + print(f"Results: {passed} passed, {failed} failed") + print("=" * 60) + + if failed > 0: + sys.exit(1) + + +async def main() -> None: + # 切换到项目根目录，确保 config.yaml 能被 VeADK 读取 + os.chdir(PROJECT_ROOT) + + if len(sys.argv) > 1 and sys.argv[1] == "--pipeline-cases": + await run_pipeline_cases() + elif len(sys.argv) > 1: + # 单条消息模式 + message = " ".join(sys.argv[1:]) + await run_single(message) + else: + # 交互模式 + await run_interactive() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python/02-use-cases/video_breakdown_agent/.skills/README.md b/python/02-use-cases/video_breakdown_agent/.skills/README.md new file mode 100644 index 00000000..1cffdb56 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.skills/README.md @@ -0,0 +1,68 @@ +# Video Breakdown Agent - VeADK Skills + +基于 VeADK 原生 Skills 格式的视频分镜拆解技能集。自包含模式，无需外部后端服务。 + +## Skills 列表 + +| Skill | 目录 | 描述 | +|-------|------|------| +| **video-breakdown** | `video-breakdown-skill/` | 视频分镜拆解：FFmpeg 预处理 → 分段 → 帧提取 | +| **hook-analyzer** | `hook-analyzer-skill/` | 前三秒钩子分析：从分镜数据中提取前3秒，构造多模态分析上下文 | +| **report-generator** | `report-generator-skill/` | 分析报告生成：整合分镜+钩子数据，输出 Markdown 报告 | + +## 目录结构 + +``` +skills/ +├── README.md +├── video-breakdown-skill/ +│ ├── SKILL.md +│ └── scripts/ +│ ├── process_video.py # 视频预处理（FFmpeg，自包含） +│ └── video_upload.py # 上传本地视频到 TOS +├── hook-analyzer-skill/ +│ ├── SKILL.md +│ └── scripts/ +│ └── analyze_hook_segments.py # 提取前三秒分镜数据 +└── report-generator-skill/ + ├── SKILL.md + └── scripts/ + └── generate_report.py # 生成 Markdown 分析报告 +``` + +## 典型工作流 + +```bash +# 1. 视频预处理 +python video-breakdown-skill/scripts/process_video.py "https://example.com/video.mp4" > breakdown.json + +# 2. 提取前三秒钩子数据 +python hook-analyzer-skill/scripts/analyze_hook_segments.py breakdown.json > hook_context.json + +# 3. (LLM 评分后得到 hook_analysis.json) + +# 4. 生成完整报告 +python report-generator-skill/scripts/generate_report.py breakdown.json hook_analysis.json > report.md +``` + +## 前置要求 + +- FFmpeg：`brew install ffmpeg` +- Python 3.12+ +- httpx：`pip install httpx` + +## 环境变量 + +| 变量 | 用途 | 必需 | +|------|------|------| +| `FFMPEG_BIN` | FFmpeg 路径 | 否（默认 `ffmpeg`） | +| `FFPROBE_BIN` | FFprobe 路径 | 否（默认 `ffprobe`） | +| `VOLCENGINE_ACCESS_KEY` | 火山引擎 AK | 上传视频时 | +| `VOLCENGINE_SECRET_KEY` | 火山引擎 SK | 上传视频时 | +| `DATABASE_TOS_BUCKET` | TOS 存储桶 | 上传时（可选） | +| `DATABASE_TOS_REGION` | TOS 区域 | 上传时（可选） | + +## 参考 + +- [VeADK 内置工具文档](https://volcengine.github.io/veadk-python/tools/builtin/) +- [AgentKit Skills 示例](https://github.com/volcengine/agentkit-samples/tree/main/02-use-cases/agent_skills) diff --git a/python/02-use-cases/video_breakdown_agent/.skills/hook-analyzer-skill/SKILL.md b/python/02-use-cases/video_breakdown_agent/.skills/hook-analyzer-skill/SKILL.md new file mode 100644 index 00000000..8421df7b --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.skills/hook-analyzer-skill/SKILL.md @@ -0,0 +1,111 @@ +--- +name: hook-analyzer +description: 视频前三秒钩子分析技能。从分镜拆解结果中提取前三秒分镜数据，为钩子吸引力评估提供结构化上下文。运行脚本 `python scripts/analyze_hook_segments.py ""` 或通过 stdin 传入 JSON。输出包含前三秒分镜数量、总时长和每个分镜的详细信息（含关键帧 URL）。 +--- + +# 视频前三秒钩子分析 (Hook Analyzer) + +## 概述 + +视频前三秒钩子分析技能从分镜拆解结果中提取前三秒的分镜数据，并构造结构化的分析上下文，包含视觉内容描述、关键帧图片 URL、运镜方式等信息，为后续的 LLM 多维度评分分析提供输入。 + +## 适用场景 + +1. **短视频优化**：评估视频开头的吸引力，提升 3 秒留存率 +2. **创意评审**：从5个维度量化分析钩子质量 +3. **竞品对标**：对比不同视频开头的钩子策略 + +## 分析维度 + +本技能提取的数据供以下 5 个维度评分（由 LLM 完成评分）： + +| 维度 | 权重 | 评估要点 | +|------|------|----------| +| 视觉冲击力 | 30% | 画面构图、色彩、光影、运镜动感 | +| 语言钩子 | 25% | 文案与画面配合度、悬念制造 | +| 情绪唤起 | 15% | 画面情绪、人物表情、氛围营造 | +| 信息密度 | 15% | 有效信息量、核心价值传达 | +| 节奏掌控 | 15% | 分镜切换节奏、平台特性适配 | + +## 使用步骤 + +### 方式一：从文件读取分镜数据 + +```bash +# 1. 先使用 video-breakdown skill 处理视频并获取分镜结果 +python ../video-breakdown-skill/scripts/process_video.py "https://example.com/video.mp4" > breakdown.json + +# 2. 提取前三秒分镜数据 +python scripts/analyze_hook_segments.py breakdown.json +``` + +### 方式二：通过 stdin 管道传入 + +```bash +cat breakdown.json | python scripts/analyze_hook_segments.py - +``` + +## 输出格式 + +```json +{ + "segment_count": 3, + "total_duration": 2.8, + "total_video_segments": 15, + "analysis_mode": "multimodal", + "segments": [ + { + "index": 0, + "start_time": 0.0, + "end_time": 1.0, + "duration": 1.0, + "visual_content": "画面描述", + "speech_text": "语音文字", + "shot_type": "特写", + "camera_movement": "推镜头", + "function_tag": "开场", + "frame_images": [ + {"type": "image_url", "image_url": {"url": "https://..."}} + ], + "frame_count": 3 + } + ] +} +``` + +## 钩子类型分类 + +分析结果可用于识别以下钩子类型： + +- **痛点型**：直击用户痛点，引发共鸣 +- **好奇型**：设置悬念，引发好奇 +- **冲突型**：制造对比或冲突吸引注意 +- **价值型**：直接展示价值承诺 +- **情感型**：以情感共鸣打动用户 +- **视觉冲击型**：通过强烈视觉效果吸引 +- **悬念型**：留下悬念引导继续观看 + +## 评分标准 + +| 分数段 | 等级 | 描述 | +|--------|------|------| +| 9-10 | 顶级 | 极强的吸引力和创意 | +| 7-8 | 优秀 | 具备良好的吸引力 | +| 5-6 | 一般 | 有改进空间 | +| 3-4 | 较弱 | 需要重大改进 | +| 1-2 | 很差 | 基本没有吸引力 | + +## 注意事项 + +1. 此脚本只做**数据提取**，不做 LLM 评分（评分由 Agent 的 LLM 完成） +2. 提取的 `frame_images` 字段中包含关键帧 URL，可供 Vision 模型直接分析 +3. 每个分镜最多取前 3 帧关键帧，避免 token 超限 +4. 输入必须是 `process_video` 返回的完整 JSON 数据 + +## 故障排除 + +| 问题 | 解决方案 | +|------|----------| +| 输出为空 segments | 确认输入 JSON 包含 `segments` 字段且不为空 | +| 前三秒无分镜 | 视频可能从静止画面开始，检查原始数据 | +| 关键帧 URL 失效 | TOS 签名 URL 可能已过期，重新拆解获取 | diff --git a/python/02-use-cases/video_breakdown_agent/.skills/hook-analyzer-skill/scripts/analyze_hook_segments.py b/python/02-use-cases/video_breakdown_agent/.skills/hook-analyzer-skill/scripts/analyze_hook_segments.py new file mode 100644 index 00000000..898baad9 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.skills/hook-analyzer-skill/scripts/analyze_hook_segments.py @@ -0,0 +1,115 @@ +""" +从分镜拆解结果中提取前三秒分镜数据，构造钩子分析上下文。 + +Usage: + python scripts/analyze_hook_segments.py + cat breakdown.json | python scripts/analyze_hook_segments.py - + +Examples: + python scripts/analyze_hook_segments.py breakdown_result.json + echo '{"segments":[...]}' | python scripts/analyze_hook_segments.py - +""" + +import json +import sys + + +def analyze_hook_segments(breakdown_data: dict) -> dict: + """ + 提取并分析视频前三秒的分镜数据，为钩子分析提供结构化上下文。 + + Args: + breakdown_data: 完整的分镜拆解结果（包含 segments 字段） + + Returns: + dict: 前三秒分镜的结构化分析上下文 + """ + segments = breakdown_data.get("segments", []) + + if not segments: + return { + "error": "没有分镜数据", + "segment_count": 0, + "total_duration": 0, + "segments": [], + } + + # 提取前三秒的分镜 + first_segments = [] + cumulative_time = 0 + + for seg in segments: + end_time = seg.get("end_time", 0) + if cumulative_time >= 3.0 and first_segments: + break + first_segments.append(seg) + cumulative_time = end_time + + # 构造分析上下文（支持多模态） + context = { + "segment_count": len(first_segments), + "total_duration": cumulative_time, + "total_video_segments": len(segments), + "analysis_mode": "multimodal", + "segments": [], + } + + # 为每个分镜构造详细信息 + for s in first_segments: + frame_urls = s.get("frame_urls", []) + + segment_info = { + "index": s.get("segment_index", 0), + "start_time": s.get("start_time", 0), + "end_time": s.get("end_time", 0), + "duration": s.get("duration", 0), + "visual_content": s.get("visual_content", ""), + "speech_text": s.get("speech_text", ""), + "shot_type": s.get("shot_type", ""), + "camera_movement": s.get("camera_movement", ""), + "function_tag": s.get("function_tag", ""), + "headline": s.get("headline", ""), + "content_tags": s.get("content_tags", []), + "voice_type": s.get("voice_type", ""), + "clip_url": s.get("clip_url", ""), + } + + # 关键帧图片（供 vision 模型使用），每个分镜最多取前3帧 + if frame_urls: + segment_info["frame_images"] = [ + {"type": "image_url", "image_url": {"url": url}} + for url in frame_urls[:3] + ] + segment_info["frame_count"] = len(frame_urls) + else: + segment_info["frame_images"] = [] + segment_info["frame_count"] = 0 + + context["segments"].append(segment_info) + + total_frames = sum(s.get("frame_count", 0) for s in context["segments"]) + print( + f"前三秒提取完成: {len(first_segments)}个分镜, " + f"总时长{cumulative_time:.1f}s, 关键帧{total_frames}张", + file=sys.stderr, + ) + + return context + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python analyze_hook_segments.py ") + print(" cat breakdown.json | python analyze_hook_segments.py -") + sys.exit(1) + + source = sys.argv[1] + + if source == "-": + data = json.load(sys.stdin) + else: + with open(source, "r", encoding="utf-8") as f: + data = json.load(f) + + result = analyze_hook_segments(data) + print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/python/02-use-cases/video_breakdown_agent/.skills/report-generator-skill/SKILL.md b/python/02-use-cases/video_breakdown_agent/.skills/report-generator-skill/SKILL.md new file mode 100644 index 00000000..db2d13b3 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.skills/report-generator-skill/SKILL.md @@ -0,0 +1,140 @@ +--- +name: report-generator +description: 视频分析报告生成技能。整合分镜拆解数据和钩子分析结果，生成 Markdown 格式的专业视频分析报告。运行脚本 `python scripts/generate_report.py [hook_analysis_json]`，输出完整的 Markdown 报告，包含基本信息、前三秒钩子分析、分镜概览、BGM分析、场景分析等章节。 +--- + +# 视频分析报告生成 (Report Generator) + +## 概述 + +视频分析报告生成技能将分镜拆解数据和（可选的）钩子分析结果整合为一份专业的 Markdown 格式分析报告。报告包含视频基本信息、前三秒钩子评分、分镜概览表格、BGM 分析、场景分析和平台推荐等章节。 + +## 适用场景 + +1. **视频分析交付**：为客户或团队生成完整的视频分析文档 +2. **创意复盘**：生成结构化的视频内容复盘报告 +3. **竞品报告**：批量生成竞品视频分析报告 + +## 使用步骤 + +### 完整报告（分镜 + 钩子分析） + +```bash +# 1. 准备分镜拆解数据和钩子分析数据（JSON 文件） +# 2. 生成报告 +python scripts/generate_report.py breakdown.json hook_analysis.json + +# 3. 保存到文件 +python scripts/generate_report.py breakdown.json hook_analysis.json > report.md +``` + +### 仅分镜报告（无钩子分析） + +```bash +python scripts/generate_report.py breakdown.json +``` + +## 报告结构 + +生成的报告包含以下章节： + +```markdown +# 视频分析报告 + +## 基本信息 +- 视频时长、分镜数量、分辨率 + +## 前三秒钩子分析（核心） +- 综合评分 +- 5维度评分表格 +- 钩子类型 +- 优势/不足/优化建议 +- 留存预测 + +## 分镜概览 +- 前10个分镜的概览表格 + +## BGM 分析 +- 音乐风格、情绪基调、节拍 + +## 场景分析 +- 主要场景、视频风格、目标受众 +- 平台推荐 + +报告生成时间 +``` + +## 输入格式 + +### breakdown.json（必需） + +```json +{ + "duration": 30.5, + "segment_count": 12, + "resolution": "1920x1080", + "segments": [...], + "bgm_analysis": { + "music_style": {"primary": "流行"}, + "emotion": {"primary": "欢快"}, + "tempo": {"bpm_estimate": 120, "pace": "中速"} + }, + "scene_analysis": { + "primary_scene": "室内", + "video_style": {"overall": "生活方式", "target_audience": ["年轻人"]}, + "platform_recommendations": [...] + } +} +``` + +### hook_analysis.json（可选） + +```json +{ + "overall_score": 7.5, + "visual_impact": 8.0, + "visual_comment": "评价...", + "language_hook": 7.0, + "language_comment": "评价...", + "emotion_trigger": 7.5, + "emotion_comment": "评价...", + "information_density": 7.0, + "info_comment": "评价...", + "rhythm_control": 8.0, + "rhythm_comment": "评价...", + "hook_type": "好奇型", + "strengths": ["优点1", "优点2"], + "weaknesses": ["不足1"], + "suggestions": ["建议1", "建议2"], + "retention_prediction": "中：50-70%，因为..." +} +``` + +## 输出格式 + +Markdown 格式的完整报告文本，直接输出到 stdout。 + +## 示例 + +```bash +# 完整流程 +python ../video-breakdown-skill/scripts/process_video.py "https://example.com/video.mp4" > breakdown.json +cat breakdown.json | python ../hook-analyzer-skill/scripts/analyze_hook_segments.py - > hooks.json +# (hooks.json 需经 LLM 评分后得到 hook_analysis.json) +python scripts/generate_report.py breakdown.json hook_analysis.json > report.md +``` + +## 注意事项 + +1. 如果缺少 hook_analysis 数据，报告中钩子分析章节将显示"暂无数据" +2. 分镜概览最多展示前 10 个分镜 +3. 报告自动添加生成时间戳 +4. 画面内容描述超过 40 字符会自动截断 + +## 故障排除 + +| 问题 | 解决方案 | +|------|----------| +| 钩子分析为空 | 确认传入了 hook_analysis.json 文件 | +| 分镜表格为空 | 确认 breakdown.json 包含 segments 字段 | +| BGM/场景显示 N/A | 分镜拆解服务可能未返回此数据 | diff --git a/python/02-use-cases/video_breakdown_agent/.skills/report-generator-skill/scripts/generate_report.py b/python/02-use-cases/video_breakdown_agent/.skills/report-generator-skill/scripts/generate_report.py new file mode 100644 index 00000000..f5b4a704 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.skills/report-generator-skill/scripts/generate_report.py @@ -0,0 +1,208 @@ +""" +整合分镜拆解数据和钩子分析结果，生成 Markdown 视频分析报告。 + +Usage: + python scripts/generate_report.py [hook_analysis_json] + +Examples: + python scripts/generate_report.py breakdown.json hook_analysis.json + python scripts/generate_report.py breakdown.json + python scripts/generate_report.py breakdown.json hook.json > report.md +""" + +import json +import sys +from datetime import datetime + + +def generate_video_report(breakdown_data: dict, hook_analysis: dict = None) -> str: + """ + 整合分镜拆解结果和钩子分析结果，生成 Markdown 视频分析报告。 + + Args: + breakdown_data: 分镜拆解的完整结果数据 + hook_analysis: 前三秒钩子分析结果（可选） + + Returns: + str: Markdown 格式报告 + """ + if hook_analysis is None: + hook_analysis = {} + + duration = breakdown_data.get("duration", 0) + segment_count = breakdown_data.get("segment_count", 0) + resolution = breakdown_data.get("resolution", "N/A") + bgm = breakdown_data.get("bgm_analysis") or {} + scene = breakdown_data.get("scene_analysis") or {} + + # BGM 信息 + music_style = bgm.get("music_style", {}).get("primary", "N/A") if bgm else "N/A" + emotion = bgm.get("emotion", {}).get("primary", "N/A") if bgm else "N/A" + tempo = bgm.get("tempo", {}).get("bpm_estimate", "N/A") if bgm else "N/A" + tempo_pace = bgm.get("tempo", {}).get("pace", "N/A") if bgm else "N/A" + + # 场景信息 + primary_scene = scene.get("primary_scene", "N/A") if scene else "N/A" + video_style = scene.get("video_style", {}).get("overall", "N/A") if scene else "N/A" + target_audience = ( + ", ".join(scene.get("video_style", {}).get("target_audience", [])) + if scene + else "N/A" + ) + + hook_section = _build_hook_section(hook_analysis) + platform_section = _build_platform_section(scene) + segments_section = _build_segments_overview(breakdown_data.get("segments", [])) + + report = f"""# 视频分析报告 + +## 基本信息 +- **视频时长**: {duration:.1f}秒 +- **分镜数量**: {segment_count}个 +- **分辨率**: {resolution} + +--- + +{hook_section} + +--- + +## 分镜概览 + +{segments_section} + +--- + +## BGM 分析 +- **音乐风格**: {music_style} +- **情绪基调**: {emotion} +- **节拍**: {tempo} BPM（{tempo_pace}节奏） + +--- + +## 场景分析 +- **主要场景**: {primary_scene} +- **视频风格**: {video_style} +- **目标受众**: {target_audience} + +{platform_section} + +--- + +**报告生成时间**: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")} +""" + return report + + +def _build_hook_section(hook_analysis: dict) -> str: + if not hook_analysis: + return "## 前三秒钩子分析\n\n暂无钩子分析数据。" + + overall = hook_analysis.get("overall_score", 0) + hook_type = hook_analysis.get("hook_type", "N/A") + retention = hook_analysis.get("retention_prediction", "N/A") + + scores_table = f"""| 维度 | 得分 | 评价 | +|------|------|------| +| 视觉冲击力 | {hook_analysis.get("visual_impact", 0)}/10 | {hook_analysis.get("visual_comment", "")} | +| 语言钩子 | {hook_analysis.get("language_hook", 0)}/10 | {hook_analysis.get("language_comment", "")} | +| 情绪唤起 | {hook_analysis.get("emotion_trigger", 0)}/10 | {hook_analysis.get("emotion_comment", "")} | +| 信息密度 | {hook_analysis.get("information_density", 0)}/10 | {hook_analysis.get("info_comment", "")} | +| 节奏掌控 | {hook_analysis.get("rhythm_control", 0)}/10 | {hook_analysis.get("rhythm_comment", "")} |""" + + strengths = hook_analysis.get("strengths", []) + weaknesses = hook_analysis.get("weaknesses", []) + suggestions = hook_analysis.get("suggestions", []) + + strengths_text = "\n".join(f"- {s}" for s in strengths) if strengths else "- 暂无" + weaknesses_text = ( + "\n".join(f"- {w}" for w in weaknesses) if weaknesses else "- 暂无" + ) + suggestions_text = ( + "\n".join(f"{i + 1}. {s}" for i, s in enumerate(suggestions)) + if suggestions + else "1. 暂无" + ) + + return f"""## 前三秒钩子分析（核心） + +### 综合评分: {overall}/10 + +{scores_table} + +### 钩子类型 +**{hook_type}** + +### 优势 +{strengths_text} + +### 不足 +{weaknesses_text} + +### 优化建议 +{suggestions_text} + +### 留存预测 +**{retention}**""" + + +def _build_platform_section(scene: dict) -> str: + if not scene: + return "" + recommendations = scene.get("platform_recommendations", []) + if not recommendations: + return "" + lines = ["### 平台推荐"] + for rec in recommendations: + platform = rec.get("platform", "N/A") + suitability = rec.get("suitability", "N/A") + reason = rec.get("reason", "") + lines.append(f"- **{platform}**（适合度: {suitability}）: {reason}") + return "\n".join(lines) + + +def _build_segments_overview(segments: list) -> str: + if not segments: + return "暂无分镜数据。" + + lines = [ + "| 镜号 | 时间 | 景别 | 运镜 | 功能标签 | 画面内容 |", + "|------|------|------|------|----------|----------|", + ] + + for seg in segments[:10]: + index = seg.get("segment_index", "-") + start = seg.get("start_time", 0) + end = seg.get("end_time", 0) + time_range = f"{start:.1f}s-{end:.1f}s" + shot_type = seg.get("shot_type", "-") + camera = seg.get("camera_movement", "-") + func_tag = seg.get("function_tag", "-") + visual = seg.get("visual_content", "-") + if len(visual) > 40: + visual = visual[:37] + "..." + lines.append( + f"| {index} | {time_range} | {shot_type} | {camera} | {func_tag} | {visual} |" + ) + + if len(segments) > 10: + lines.append(f"\n*（仅展示前10个分镜，共{len(segments)}个）*") + + return "\n".join(lines) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python generate_report.py [hook_analysis_json]") + sys.exit(1) + + with open(sys.argv[1], "r", encoding="utf-8") as f: + bd_data = json.load(f) + + hook_data = None + if len(sys.argv) > 2: + with open(sys.argv[2], "r", encoding="utf-8") as f: + hook_data = json.load(f) + + report = generate_video_report(bd_data, hook_data) + print(report) diff --git a/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/SKILL.md b/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/SKILL.md new file mode 100644 index 00000000..5a953b8f --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/SKILL.md @@ -0,0 +1,70 @@ +--- +name: video-breakdown +description: 视频分镜拆解技能（自包含，无需外部后端）。使用 FFmpeg 预处理视频并生成分镜数据。(1) 视频 URL 直接处理 `python scripts/process_video.py ""`；(2) 本地文件先上传 `python scripts/video_upload.py ""` 获取 URL 后再处理。需要本机安装 FFmpeg。 +--- + +# 视频分镜拆解 (Video Breakdown) + +## 概述 + +视频分镜拆解技能通过本机 FFmpeg 将视频自动拆解为逐帧分镜，提供每个镜头的关键帧图片和时间信息。无需外部后端服务。 + +## 前置要求 + +- 本机安装 FFmpeg：`brew install ffmpeg` + +## 使用步骤 + +### 方式一：视频 URL 直接处理 + +```bash +python scripts/process_video.py "https://example.com/video.mp4" +``` + +### 方式二：本地文件上传后处理 + +```bash +# 1. 上传本地视频到 TOS +python scripts/video_upload.py "/path/to/video.mp4" + +# 2. 用返回的 video_url 处理 +python scripts/process_video.py "" +``` + +## 环境变量 + +| 变量名 | 必需 | 描述 | +|--------|------|------| +| `FFMPEG_BIN` | 否 | FFmpeg 路径，默认 `ffmpeg` | +| `FFPROBE_BIN` | 否 | FFprobe 路径，默认 `ffprobe` | +| `VOLCENGINE_ACCESS_KEY` | 上传时需要 | 火山引擎 Access Key | +| `VOLCENGINE_SECRET_KEY` | 上传时需要 | 火山引擎 Secret Key | +| `DATABASE_TOS_BUCKET` | 否 | TOS 存储桶名称 | +| `DATABASE_TOS_REGION` | 否 | TOS 区域，默认 `cn-beijing` | + +## 输出格式 + +```json +{ + "task_id": "xxx", + "duration": 30.5, + "resolution": "1920x1080", + "segment_count": 12, + "segments": [ + { + "index": 1, + "start": 0.0, + "end": 3.0, + "frame_paths": ["path/to/frame.jpg"] + } + ] +} +``` + +## 故障排除 + +| 问题 | 解决方案 | +|------|----------| +| FFmpeg not found | 安装 FFmpeg: `brew install ffmpeg` | +| 上传失败 | 检查 AK/SK 配置和 TOS 存储桶 | +| 视频下载失败 | 确认 URL 有效且可公开访问 | diff --git a/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/scripts/process_video.py b/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/scripts/process_video.py new file mode 100644 index 00000000..864e340a --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/scripts/process_video.py @@ -0,0 +1,177 @@ +""" +视频预处理脚本（独立可执行）。 +自包含视频预处理，无需外部后端服务。 +需要本机安装 FFmpeg。 + +Usage: + python scripts/process_video.py "" + +Env: + FFMPEG_BIN, FFPROBE_BIN, VOLCENGINE_ACCESS_KEY, VOLCENGINE_SECRET_KEY, + VOLC_ASR_APP_ID, VOLC_ASR_ACCESS_KEY, VOLC_ASR_RESOURCE_ID, + DATABASE_TOS_BUCKET (或 TOS_BUCKET), DATABASE_TOS_REGION (或 TOS_REGION) +""" + +import asyncio +import json +import os +import shutil +import subprocess +import sys +import tempfile +import uuid +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List + +import httpx + + +def _run_command(cmd: List[str]) -> str: + process = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True + ) + return process.stdout.strip() + + +def _probe_video(ffprobe_bin: str, video_path: Path) -> Dict[str, Any]: + cmd = [ + ffprobe_bin, + "-v", + "error", + "-print_format", + "json", + "-show_format", + "-show_streams", + str(video_path), + ] + result = _run_command(cmd) + info = json.loads(result) if result else {} + fmt = info.get("format") or {} + duration = float(fmt.get("duration", 0)) + w, h, fr = None, None, None + for s in info.get("streams", []): + if s.get("codec_type") == "video": + w, h, fr = s.get("width"), s.get("height"), s.get("r_frame_rate") + break + return { + "duration": duration, + "width": w, + "height": h, + "frame_rate": fr, + "size": fmt.get("size"), + } + + +def _build_segments(duration: float): + segments = [] + idx = 1 + bps = [0.0, 3.0, 5.0, 10.0, 20.0] + for i in range(len(bps) - 1): + if duration <= bps[i]: + break + end = min(bps[i + 1], duration) + if end - bps[i] < 0.5: + break + segments.append({"index": idx, "start": bps[i], "end": end}) + idx += 1 + if duration > 20.0: + c = 20.0 + while c < duration: + e = min(duration, c + 10.0) + if e - c < 0.5: + break + segments.append({"index": idx, "start": c, "end": e}) + idx += 1 + c = e + return segments + + +def _extract_frames(ffmpeg_bin: str, video_path: Path, segments, fps=3): + frames_dir = video_path.parent / "frames" + frames_dir.mkdir(exist_ok=True) + for seg in segments: + seg["frame_paths"] = [] + dur = seg["end"] - seg["start"] + for i in range(fps): + ratio = i / max(fps - 1, 1) + offset = min(seg["start"] + ratio * dur, seg["end"] - 0.1) + offset = max(offset, seg["start"]) + out = frames_dir / f"seg{seg['index']:03d}_frame_{i}.jpg" + cmd = [ + ffmpeg_bin, + "-y", + "-ss", + f"{offset:.2f}", + "-i", + str(video_path), + "-frames:v", + "1", + "-q:v", + "5", + str(out), + ] + try: + _run_command(cmd) + if out.exists(): + seg["frame_paths"].append(str(out)) + except Exception: + pass + + +async def main(video_url: str): + ffmpeg_bin = os.getenv("FFMPEG_BIN", "ffmpeg") + ffprobe_bin = os.getenv("FFPROBE_BIN", "ffprobe") + temp_dir = Path(tempfile.mkdtemp(prefix="vba_")) + task_id = datetime.now().strftime("%Y%m%d_%H%M%S") + "_" + uuid.uuid4().hex[:8] + local_video = temp_dir / f"{task_id}.mp4" + + try: + # Download + print("[1/4] 下载视频...", file=sys.stderr) + async with httpx.AsyncClient(timeout=300, follow_redirects=True) as c: + r = await c.get(video_url) + r.raise_for_status() + local_video.write_bytes(r.content) + + # Metadata + print("[2/4] 提取元数据...", file=sys.stderr) + meta = _probe_video(ffprobe_bin, local_video) + dur = meta["duration"] + if dur <= 0: + print(json.dumps({"error": "无法获取视频时长"})) + return + + # Segments + print(f"[3/4] 构建分镜 ({dur:.1f}s)...", file=sys.stderr) + segments = _build_segments(dur) + + # Frames + print("[4/4] 提取关键帧...", file=sys.stderr) + _extract_frames(ffmpeg_bin, local_video, segments) + + result = { + "task_id": task_id, + "duration": round(dur, 2), + "resolution": f"{meta.get('width')}x{meta.get('height')}", + "segment_count": len(segments), + "segments": [ + { + "index": s["index"], + "start": round(s["start"], 2), + "end": round(s["end"], 2), + "frame_paths": s.get("frame_paths", []), + } + for s in segments + ], + } + print(json.dumps(result, ensure_ascii=False, indent=2)) + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python process_video.py ", file=sys.stderr) + sys.exit(1) + asyncio.run(main(sys.argv[1])) diff --git a/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/scripts/video_upload.py b/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/scripts/video_upload.py new file mode 100644 index 00000000..f593a66c --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/.skills/video-breakdown-skill/scripts/video_upload.py @@ -0,0 +1,138 @@ +""" +上传本地视频文件到火山引擎 TOS 对象存储，返回签名 URL。 + +Usage: + python scripts/video_upload.py "" [bucket_name] + +Examples: + python scripts/video_upload.py "/path/to/video.mp4" + python scripts/video_upload.py "/path/to/video.mp4" "my-bucket" +""" + +import json +import os +import sys +from datetime import datetime + +import tos +from tos import HttpMethodType + +DEFAULT_BUCKET = "video-breakdown-uploads" +DEFAULT_REGION = "cn-beijing" + + +def video_upload_to_tos(file_path: str, bucket_name: str = None) -> dict: + """ + 将本地视频文件上传到 TOS，返回签名 URL。 + + Args: + file_path: 本地视频文件路径 + bucket_name: TOS 存储桶名称（可选） + + Returns: + dict: 包含 video_url 或 error + """ + if bucket_name is None: + bucket_name = os.getenv("DATABASE_TOS_BUCKET") or os.getenv( + "TOS_BUCKET", DEFAULT_BUCKET + ) + region = os.getenv("DATABASE_TOS_REGION") or os.getenv("TOS_REGION", DEFAULT_REGION) + + # 检查文件 + if not os.path.exists(file_path): + return {"error": f"文件不存在: {file_path}"} + if not os.path.isfile(file_path): + return {"error": f"路径不是文件: {file_path}"} + + file_size = os.path.getsize(file_path) + max_size = 2 * 1024 * 1024 * 1024 # 2GB + if file_size > max_size: + return {"error": f"文件过大（{file_size / 1024 / 1024:.0f}MB），最大支持 2GB"} + + # 获取凭证 + access_key = os.getenv("VOLCENGINE_ACCESS_KEY", "") + secret_key = os.getenv("VOLCENGINE_SECRET_KEY", "") + session_token = "" + + if not access_key or not secret_key: + try: + from veadk.auth.veauth.utils import get_credential_from_vefaas_iam + + cred = get_credential_from_vefaas_iam() + access_key = cred.access_key_id + secret_key = cred.secret_access_key + session_token = cred.session_token + except Exception: + pass + + if not access_key or not secret_key: + return { + "error": "缺少 TOS 访问凭证，请设置 VOLCENGINE_ACCESS_KEY 和 VOLCENGINE_SECRET_KEY" + } + + # 自动生成 object_key + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = os.path.basename(file_path) + object_key = f"video_breakdown/upload/{timestamp}_{filename}" + + # 上传 + client = None + try: + endpoint = f"tos-{region}.volces.com" + client = tos.TosClientV2( + ak=access_key, + sk=secret_key, + security_token=session_token, + endpoint=endpoint, + region=region, + ) + + # 检查桶 + try: + client.head_bucket(bucket_name) + except tos.exceptions.TosServerError as e: + if e.status_code == 404: + return {"error": f"TOS 存储桶 {bucket_name} 不存在"} + raise + + print(f"上传中: {file_path} -> {bucket_name}/{object_key}", file=sys.stderr) + client.put_object_from_file( + bucket=bucket_name, key=object_key, file_path=file_path + ) + + # 生成签名 URL（7天有效） + signed_url_output = client.pre_signed_url( + http_method=HttpMethodType.Http_Method_Get, + bucket=bucket_name, + key=object_key, + expires=604800, + ) + + return { + "video_url": signed_url_output.signed_url, + "bucket": bucket_name, + "object_key": object_key, + "file_size_mb": round(file_size / 1024 / 1024, 2), + "message": "上传成功！使用 video_url 调用 process_video.py 进行视频分镜分析", + } + + except tos.exceptions.TosClientError as e: + return {"error": f"TOS 客户端错误: {e}"} + except tos.exceptions.TosServerError as e: + return {"error": f"TOS 服务端错误: {e.message}"} + except Exception as e: + return {"error": f"上传失败: {str(e)}"} + finally: + if client: + client.close() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python video_upload.py [bucket_name]") + sys.exit(1) + + path = sys.argv[1] + bucket = sys.argv[2] if len(sys.argv) > 2 else None + result = video_upload_to_tos(path, bucket) + print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/python/02-use-cases/video_breakdown_agent/LICENSE b/python/02-use-cases/video_breakdown_agent/LICENSE new file mode 100644 index 00000000..67db8588 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/LICENSE @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. diff --git a/python/02-use-cases/video_breakdown_agent/README.md b/python/02-use-cases/video_breakdown_agent/README.md new file mode 100644 index 00000000..9590f3fe --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/README.md @@ -0,0 +1,504 @@ +# 智能短视频拆解分析助手 + +## 概述 + +这是一个基于火山引擎 VeADK & AgentKit 构建的智能短视频分析系统。本系统采用 Multi-Agent 架构，集成 FFmpeg 视频处理、火山 ASR 语音识别、LiteLLM 多模态视觉分析和 TOS 对象存储，能够对短视频进行全方位的专业分析。 + +## 核心功能 + +本项目提供以下核心功能： + +- **视频分镜拆解**：基于 FFmpeg 自动识别视频分镜，提取关键帧并进行画面内容分析，输出结构化分镜数据 +- **前三秒钩子分析**：从视觉冲击力、语言钩子、情绪唤起、信息密度、节奏掌控 5 个维度进行专业评分 +- **专业报告生成**：整合分镜数据和钩子分析结果，生成 Markdown 格式的完整分析报告 +- **联网搜索**：实时获取短视频行业最新资讯、平台规则和热门趋势 + +## 🎯 核心解决场景 + +本系统致力于解决短视频创作与运营中的以下痛点： + +- **📉 前三秒流失率高**：通过 **Hook Analyzer** 深度拆解开头黄金3秒，量化视觉/听觉/内容钩子质量，提供针对性优化建议。 +- **😵 卖点传递不清**：通过 **分镜拆解** 梳理叙事逻辑，确保产品核心卖点在关键时间窗内有效传递。 +- **🐢 竞品分析效率低**：自动化提取竞品分镜结构与拍摄手法，快速复刻爆款视频公式。 +- **📝 脚本创作没灵感**：基于优秀视频反推分镜脚本，为新视频创作提供结构化参考。 + +## 使用案例 + +以下展示了从联网搜索到视频深度拆解的典型使用流程。 + +### 1. 联网搜索（天气/行业资讯） + +**用户：** 杭州天气 + +**小视（Root Agent）：** 根据搜索结果，杭州当前天气及未来几天预报如下： + +📍 **杭州市天气（2026年2月11日 星期三）** + +**当前实况** +- **温度**：13℃（当前），全天5~15℃ +- **天气**：晴 +- **风向**：北风4级（午后转西风） +- **湿度**：36% +- **空气质量**：AQI 64（良），主要污染物PM2.5 + +**📅 未来三天预报** + +| 日期 | 天气 | 气温范围 | 风向风力 | +| :--- | :--- | :--- | :--- | +| 2月11日（周三） | 晴 → 晴 | 5~15℃ | 北风转西风 2-4级 | +| 2月12日（周四） | 晴 → 晴 | 6~16℃ | 东南风 2级 | +| 2月13日（周五） | 多云 | 8~20℃ | 南风 2级 | + +**🌡️ 生活指数参考** +- **洗车指数**：今日适宜（未来两天无雨） +- **运动指数**：今日不适宜剧烈运动（气压偏低），明日适宜 +- **紫外线**：弱（建议做好基础防晒） + +--- + +### 2. 视频分镜拆解 + +**用户：** http://example.com/sample-video.mp4 + +**小视（Root Agent）：** 好的，我将为您拆解该视频的分镜结构。 + +#### 🎥 视频分镜拆解结果 + +**视频信息** +- **时长**：17.07秒 +- **分辨率**：720x960 +- **帧速率**：60fps + +**分镜列表** + +| 序号 | 时间段 | 景别 | 运镜 | 画面内容描述 | 语音内容 | 功能标签 | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| 1 | 0.0-3.0s | 特写 | 固定 | 单手持米白色乐扣水杯展示整体，背景有日历、英文装饰画；过渡到双手操作杯盖提手。 | 无 | 产品展示 | +| 2 | 3.0-5.0s | 特写 | 固定 | 美甲双手打开杯盖，水流注入杯中；背景从英文装饰牌切换为日历场景。 | 无 | 产品展示 | +| 3 | 5.0-10.0s | 近景 | 固定 | 水杯接水后，美甲手握住盖好的水杯；背景含卡通图案布、日历等。 | 无 | 产品展示 | +| 4 | 10.0-17.07s | 近景 | 固定 | 先展示白色乐扣保温杯（带提手），后切换为透明条纹水杯，背景风格一致。 | 无 | 产品展示 | + +**BGM 分析** +- **是否有 BGM**：是 +- **风格**：轻音乐（纯音乐/氛围），标签：舒缓、治愈、简约 +- **情绪**：轻松愉悦（正面），强度中等 +- **乐器**：钢琴（主导）、小提琴、合成器、轻柔打击乐 +- **节奏**：60-80 BPM（慢节奏），平稳4/4拍 + +**完整语音文本** +- 无语音内容（ASR未识别到文字） + +etc. + +--- + +### 3. 前三秒钩子评测 + +**用户：** 详细分析评测视频前三秒钩子 + +**小视（Root Agent）：** + +#### 🪝 视频前三秒钩子分析评测 + +**钩子核心要素拆解** +- **视觉钩子（0-3秒）**：特写镜头聚焦产品提手设计，通过“握持-提动”动作强化产品便捷性认知，背景生活化，美甲细节增加精致感。 +- **听觉钩子（0-3秒）**：舒缓钢琴+小提琴轻音乐（60-80 BPM），营造治愈氛围，降低用户跳出率。 +- **内容钩子（0-3秒）**：3秒内完成“是什么→怎么用”的认知闭环，直击“开合便利性”需求。 + +**钩子有效性评测** + +| 维度 | 评分（1-10） | 分析 | +| :--- | :--- | :--- | +| 视觉吸引力 | 8.5 | 特写聚焦产品，动作连贯无冗余。 | +| 信息传递效率 | 9.0 | 3秒内完成核心信息传递，符合认知习惯。 | +| 情绪感染力 | 7.5 | 氛围舒缓但记忆点略显不足。 | + +--- + +etc. + +### 4. 生成视频洞察报告 + +**用户：** 给出详细的视频洞察报告 + +**小视（Root Agent）：** + +#### 📊 乐扣水杯产品展示视频洞察报告 + +**一、内容结构分析** +视频采用“整体→细节→场景→延伸”的递进逻辑，4个分镜形成完整产品认知链。 + +**二、用户注意力分析** +- **高注意力时段**：0-5秒（动作变化频繁，视觉焦点明确）。 +- **风险点**：10-17秒（产品切换缺乏过渡，易导致认知断层）。 + +**三、转化潜力评估** +- **优势**：符合“黄金3秒”法则，多产品展示覆盖不同需求。 +- **不足**：未明确材质安全、保温时长等核心卖点。 + +**四、优化建议** +- **视觉层**：增加“防漏测试”特写。 +- **听觉层**：加入杯盖开合的清脆音效增强记忆。 +- **内容层**：叠加“食品级304”等文字标签。 + +etc. + +## Agent 能力 + +![Video Breakdown Agent with AgentKit Runtime](assets/architecture_video_breakdown_agent.jpg) + +```text +用户输入（视频URL/本地文件） + ↓ +AgentKit 运行时 + ↓ +Root Agent（小视 - 主编排器） + ├── Breakdown Agent（分镜拆解） + │ ├── FFmpeg 视频预处理 + │ ├── 火山 ASR 语音识别 + │ ├── LiteLLM 视觉分析 + │ └── BGM 分析 + ├── Hook Analyzer Agent（钩子分析） + │ ├── 前三秒分镜提取 + │ ├── 多模态视觉评分 + │ └── JSON 格式化 + ├── Report Generator Agent（报告生成） + └── Search Agent（联网搜索） +``` + +主要的火山引擎产品或 Agent 组件： + +- 方舟大模型： + - doubao-seed-1-6-251015（主推理模型） + - doubao-seed-1-6-vision-250815（视觉分析模型） +- TOS 对象存储 +- 火山 ASR 语音识别（可选） +- Web Search 联网搜索 +- AgentKit +- APMPlus（可选可观测性） + +第三方依赖： + +- FFmpeg（通过 imageio-ffmpeg 自动打包，无需手动安装） +- LiteLLM（支持 Gemini、豆包、GPT-4o 等多种视觉模型） + +## 目录结构说明 + +```bash +video_breakdown_agent/ +├── README.md # 项目说明文档 +├── README_CONFIG.md # 详细配置指南 +├── project.toml # 应用广场元数据 +├── agent.py # AgentKit 部署入口 +├── requirements.txt # pip 依赖清单 +├── pyproject.toml # uv 项目配置 +├── config.yaml # 配置文件（示例，实际密钥通过环境变量注入） +├── config.yaml.example # 配置模板 +├── deploy.sh # 部署脚本 +├── video_breakdown_agent/ # Python 包（核心代码） +│ ├── agent.py # Root Agent 定义 +│ ├── prompt.py # 主编排 Prompt +│ ├── sub_agents/ # 子 Agent +│ │ ├── breakdown_agent/ # 分镜拆解 Agent +│ │ ├── hook_analyzer_agent/# 钩子分析 Agent（SequentialAgent） +│ │ └── report_generator_agent/ # 报告生成 Agent +│ ├── tools/ # 工具函数 +│ │ ├── process_video.py # 视频预处理（FFmpeg + ASR） +│ │ ├── analyze_segments_vision.py # 视觉分析 +│ │ ├── analyze_bgm.py # BGM 分析 +│ │ ├── analyze_hook_segments.py # 钩子分镜提取 +│ │ ├── report_generator.py # 报告生成 +│ │ └── video_upload.py # TOS 视频上传 +│ ├── hook/ # Callback 钩子 +│ │ ├── format_hook.py # JSON 修复 +│ │ └── video_upload_hook.py# 文件上传拦截 +│ └── utils/ # 工具类 +│ └── types.py # Pydantic 数据模型 +└── img/ # 架构图和截图 +``` + +## 本地运行 + +### 前置准备 + +**Python 版本：** + +- Python 3.12 或更高版本 + +**1. 开通火山方舟模型服务：** + +- 访问 [火山方舟控制台](https://console.volcengine.com/ark/region:ark+cn-beijing/overview) +- 进入"开通管理" → "语言模型" → 找到以下模型 → 点击"开通服务" + - `doubao-seed-1-6-251015`（主推理模型） + - `doubao-seed-1-6-vision-250815`（视觉分析模型，可选） +- 确认开通，等待服务生效（通常 1-2 分钟） + +**2. 获取火山引擎访问凭证：** + +- 登录 [火山引擎控制台](https://console.volcengine.com) +- 进入"访问控制" → "用户" → 选择用户 → "密钥" → 新建密钥或复制已有 AK/SK +- 为用户配置权限： + - `AgentKitFullAccess`（AgentKit 全量权限） + - `APMPlusServerFullAccess`（APMPlus 全量权限，可选） + +**3. 获取火山方舟模型 API Key：** + +- 登录 [火山方舟控制台](https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey) +- 进入"API Key 管理" → 创建或复制已有 API Key + +**4. 创建 TOS 存储桶：** + +- 访问 [TOS 控制台](https://console.volcengine.com/tos/bucket) +- 点击"创建桶" → 填写桶名称 → 选择区域（建议 cn-beijing）→ 创建 +- 记录桶名称，后续在环境变量 `DATABASE_TOS_BUCKET` 中填入 + +**5. 可选：开通火山 ASR 语音识别服务：** + +> 如未配置 ASR，系统会跳过语音识别，仍可完成分镜拆解和视觉分析 + +- 访问 [火山语音服务控制台](https://console.volcengine.com/speech/service/list) +- 创建应用并获取 `APP_ID` 和 `ACCESS_KEY` + +### 依赖安装 + +#### 1. 安装 uv 包管理器 + +```bash +# macOS / Linux（官方安装脚本） +curl -LsSf https://astral.sh/uv/install.sh | sh + +# 或使用 Homebrew（macOS） +brew install uv +``` + +#### 2. 初始化项目依赖 + +```bash +# 进入项目目录 +cd 02-use-cases/video_breakdown_agent +``` + +您可以通过 `pip` 工具来安装本项目依赖： + +```bash +pip install -r requirements.txt +``` + +或者使用 `uv` 工具来安装本项目依赖（推荐）： + +```bash +# 如果没有 `uv` 虚拟环境，可以使用命令先创建一个虚拟环境 +uv venv --python 3.12 + +# 使用 `pyproject.toml` 管理依赖 +uv sync --index-url https://pypi.tuna.tsinghua.edu.cn/simple + +# 激活虚拟环境 +source .venv/bin/activate +``` + +### 环境准备 + +设置以下环境变量： + +```bash +# 火山方舟模型 API Key（必需） +export MODEL_AGENT_API_KEY= + +# 火山引擎访问凭证（必需） +export VOLCENGINE_ACCESS_KEY= +export VOLCENGINE_SECRET_KEY= + +# TOS 存储桶名称（必需） +export DATABASE_TOS_BUCKET= +export DATABASE_TOS_REGION=cn-beijing + +# 可选：火山 ASR 配置（未配置时跳过语音识别） +export ASR_APP_ID= +export ASR_ACCESS_KEY= + +# 可选：切换视觉分析模型（默认使用豆包 Vision） +export MODEL_VISION_NAME=doubao-seed-1-6-vision-250815 +# 或切换为 Google Gemini（需配置 GEMINI_API_KEY） +# export MODEL_VISION_NAME=gemini/gemini-2.5-pro +# export GEMINI_API_KEY= +``` + +> 详细配置说明请参考 [README_CONFIG.md](README_CONFIG.md) + +### 调试方法 + +使用 `veadk web` 进行本地调试： + +```bash +# 进入 02-use-cases 目录 +cd 02-use-cases + +# 启动 VeADK Web 界面 +veadk web --port 8080 + +# 在浏览器访问：http://127.0.0.1:8080 +``` + +Web 界面提供图形化对话测试环境，支持实时查看消息流和调试信息。 + +### 示例提示词 + +```text +帮我拆解这个视频的分镜结构 https://example.com/video.mp4 +分析这个视频前三秒的钩子吸引力，给出专业评分 +生成完整的视频分析报告 +搜一下抖音最新推荐算法有什么变化 +``` + +### 效果展示 + +> 建议补充实际运行截图展示分镜拆解、钩子分析和报告生成效果 + +## AgentKit 部署 + +### 前置准备 + +**重要提示**：在运行本示例之前，请先访问 [AgentKit 控制台授权页面](https://console.volcengine.com/agentkit/region:agentkit+cn-beijing/auth?projectName=default) 对所有依赖服务进行授权，确保案例能够正常执行。 + +参考"本地运行"部分的"前置准备"。 + +### 依赖安装 + +> 如果您本地已经安装了该依赖，跳过此步骤。 + +使用 `pip` 安装 AgentKit 命令行工具： + +```bash +pip install agentkit-sdk-python==0.5.1 +``` + +或者使用 `uv` 安装 AgentKit 命令行工具： + +```bash +uv pip install agentkit-sdk-python==0.5.1 +``` + +### 设置环境变量 + +```bash +# 火山引擎访问凭证（必需） +export VOLCENGINE_ACCESS_KEY= +export VOLCENGINE_SECRET_KEY= +``` + +### AgentKit 云上部署 + +```bash +# 1. 进入项目目录 +cd 02-use-cases/video_breakdown_agent + +# 2. 配置 Agentkit 部署配置 +agentkit config \ + --agent_name video_breakdown_agent \ + --entry_point 'agent.py' \ + --launch_type cloud + +# 3. 配置 AgentKit Runtime 环境变量（应用级） +# 以下环境变量均为必填项，参考"前置准备"部分获取相应的值 +agentkit config \ + -e MODEL_AGENT_API_KEY= \ + -e DATABASE_TOS_BUCKET= \ + -e DATABASE_TOS_REGION=cn-beijing + +# 可选：配置 ASR 服务 +agentkit config \ + -e ASR_APP_ID= \ + -e ASR_ACCESS_KEY= + +# 4. 启动云端服务 +agentkit launch + +# 5. 测试部署的 Agent +agentkit invoke "分析这个视频 https://example.com/video.mp4" +``` + +### 测试已部署的智能体 + +在 AgentKit 控制台"智能体运行时"页面找到已部署的智能体 `video_breakdown_agent`，点击在线测评，输入提示词进行测试。 + +## 主要特性 + +### Multi-Agent 协作架构 + +Root Agent 作为主编排器，根据用户意图自动调度 4 个专业子 Agent： + +- **Breakdown Agent**：视频预处理 + ASR + 视觉分析 + BGM 分析 +- **Hook Analyzer Agent**：SequentialAgent 模式，先视觉分析后格式化 +- **Report Generator Agent**：整合数据生成 Markdown 报告 +- **Search Agent**：联网搜索行业信息 + +### 优雅降级机制 + +系统具备完善的容错能力： + +- TOS 上传失败 → 自动回退 base64 编码 +- ASR 未配置 → 跳过语音识别，仍可完成分镜拆解 +- FFmpeg 未安装 → 自动使用 imageio-ffmpeg 打包版本 + +### 多模型灵活切换 + +通过 LiteLLM 统一路由，支持一行配置切换视觉模型： + +- `doubao-seed-1-6-vision-250815`（火山方舟豆包） +- `gemini/gemini-2.5-pro`（Google Gemini） +- `gpt-4o`（OpenAI） + +### Session State 数据共享 + +子 Agent 之间通过 Session State 共享数据，避免重复序列化大 JSON： + +- `process_video` → `breakdown_result` +- `hook_analyzer` → `hook_analysis` +- `report_generator` → `final_report` + +## 常见问题 + +**错误：`VOLCENGINE_ACCESS_KEY not set`** + +- 请确保已设置火山引擎访问凭证环境变量 +- 参考"环境准备"章节设置 `VOLCENGINE_ACCESS_KEY` 和 `VOLCENGINE_SECRET_KEY` + +**错误：`TOS 存储桶不存在`** + +- 请在 TOS 控制台创建存储桶 +- 确认 `DATABASE_TOS_BUCKET` 和 `DATABASE_TOS_REGION` 配置正确 +- 系统会自动回退 base64 编码，不影响核心功能 + +**FFmpeg 未找到：** + +- 系统会自动使用 imageio-ffmpeg 打包版本，无需手动安装 +- 如需使用系统 FFmpeg，请确保已安装并在 PATH 中 + +**ASR 语音识别失败：** + +- 检查 `ASR_APP_ID` 和 `ASR_ACCESS_KEY` 是否正确 +- 未配置 ASR 时系统会跳过语音识别，仍可完成分镜拆解 + +**视觉模型切换：** + +- 修改 `MODEL_VISION_NAME` 环境变量即可切换模型 +- 使用 Gemini 时需配置 `GEMINI_API_KEY` +- 详细配置说明请参考 [README_CONFIG.md](README_CONFIG.md) + +**视频文件过大：** + +- 当前限制视频文件大小为 2GB +- 建议压缩视频后重试 + +## 参考资料 + +- [VeADK 官方文档](https://volcengine.github.io/veadk-python/) +- [AgentKit 开发指南](https://volcengine.github.io/agentkit-sdk-python/) +- [火山方舟模型服务](https://console.volcengine.com/ark/region:ark+cn-beijing/overview?briefPage=0&briefType=introduce&type=new&projectName=default) +- [TOS 对象存储](https://www.volcengine.com/product/TOS) +- [火山 ASR 语音识别](https://www.volcengine.com/product/voice-tech) + +## 代码许可 + +本工程遵循 Apache 2.0 License diff --git a/python/02-use-cases/video_breakdown_agent/README_en.md b/python/02-use-cases/video_breakdown_agent/README_en.md new file mode 100644 index 00000000..43e0b1b1 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/README_en.md @@ -0,0 +1,513 @@ +# Video Breakdown Analyzer + +## Overview + +This is an intelligent short-video analysis system built on Volcengine VeADK & AgentKit. The system adopts a Multi-Agent architecture, integrating FFmpeg video processing, Volcengine ASR speech recognition, LiteLLM multimodal vision analysis, and TOS object storage, capable of comprehensive professional analysis of short videos. + +## Core Features + +This project provides the following core capabilities: + +- **Video Scene Segmentation**: Automatically identifies video scenes based on FFmpeg, extracts key frames and analyzes visual content, outputting structured scene data +- **First 3-Second Hook Analysis**: Professional scoring from 5 dimensions: visual impact, language hooks, emotional arousal, information density, and rhythm control +- **Professional Report Generation**: Integrates scene data and hook analysis results to generate complete analysis reports in Markdown format +- **Web Search**: Real-time access to the latest short-video industry information, platform rules, and trending topics + +## Agent Architecture + +![Video Breakdown Agent with AgentKit Runtime](assets/architecture_video_breakdown_agent.jpg) + +```text +User Input (Video URL/Local File) + ↓ +AgentKit Runtime + ↓ +Root Agent (XiaoShi - Main Orchestrator) + ├── Breakdown Agent (Scene Segmentation) + │ ├── FFmpeg Video Preprocessing + │ ├── Volcengine ASR Speech Recognition + │ ├── LiteLLM Vision Analysis + │ └── BGM Analysis + ├── Hook Analyzer Agent (Hook Analysis) + │ ├── First 3-Second Scene Extraction + │ ├── Multimodal Vision Scoring + │ └── JSON Formatting + ├── Report Generator Agent (Report Generation) + └── Search Agent (Web Search) +``` + +Main Volcengine products and Agent components: + +- Ark LLM Platform: + - doubao-seed-1-6-251015 (primary reasoning model) + - doubao-seed-1-6-vision-250815 (vision analysis model) +- TOS Object Storage +- Volcengine ASR Speech Recognition (optional) +- Web Search +- AgentKit +- APMPlus (optional observability) + +Third-party dependencies: + +- FFmpeg (auto-packaged via imageio-ffmpeg, no manual installation required) +- LiteLLM (supports Gemini, Doubao, GPT-4o and other vision models) + +## Usage Examples + +Below is a typical workflow from web search to in-depth video breakdown. + +### 1. Web Search (Weather/Industry Info) + +**User:** Weather in Hangzhou + +**XiaoShi (Root Agent):** Based on the search results, here is the current weather and forecast for Hangzhou: + +📍 **Hangzhou Weather (Wednesday, Feb 11, 2026)** + +**Current Conditions** +- **Temperature**: 13°C (Current), 5~15°C for the day +- **Condition**: Sunny +- **Wind**: North 4 (turning West in the afternoon) +- **Humidity**: 36% +- **Air Quality**: AQI 64 (Good) + +**📅 3-Day Forecast** + +| Date | Condition | Temp Range | Wind | +| :--- | :--- | :--- | :--- | +| Feb 11 (Wed) | Sunny | 5~15°C | North to West 2-4 | +| Feb 12 (Thu) | Sunny | 6~16°C | SE 2 | +| Feb 13 (Fri) | Cloudy | 8~20°C | South 2 | + +--- + +### 2. Video Breakdown + +**User:** http://example.com/sample-video.mp4 + +**XiaoShi (Root Agent):** Sure, I'll break down the scenes for you. + +#### 🎥 Video Breakdown Results + +**Video Info** +- **Duration**: 17.07s +- **Resolution**: 720x960 +- **Frame Rate**: 60fps + +**Scene List** + +| ID | Time | Shot | Movement | Content Description | Audio | Tags | +| :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| 1 | 0.0-3.0s | CU | Static | Showing a beige LocknLock bottle; transitioning to opening the lid. | None | Product Display | +| 2 | 3.0-5.0s | CU | Static | Pouring water into the bottle; home lifestyle background. | None | Product Display | + +**BGM Analysis** +- **BGM**: Yes +- **Style**: Soft Piano/Ambient (Healing, Minimalist) +- **Mood**: Relaxing (Positive) + +--- + +### 3. Hook Analysis + +**User:** Analyze the first 3 seconds hook in detail. + +**XiaoShi (Root Agent):** + +#### 🪝 First 3s Hook Analysis & Rating + +**Hook Elements** +- **Visual Hook**: Close-up focusing on the handle design; "Grip-and-Lift" action reinforces convenience. +- **Auditory Hook**: Soothing piano/violin (60-80 BPM) creates a cozy atmosphere. +- **Content Hook**: Completes the "What it is -> How to use" loop within 3s. + +**Rating** + +| Dimension | Score (1-10) | Analysis | +| :--- | :--- | :--- | +| Visual Attraction | 8.5 | Sharp focus, smooth and purposeful movements. | +| Info Efficiency | 9.0 | Delivers core value proposition within 3s. | + +--- + +### 4. Generate Insight Report + +**User:** Give me a detailed video insight report. + +**XiaoShi (Root Agent):** + +#### 📊 LocknLock Product Display Insight Report + +**1. Content Structure** +The video follows a "General -> Detail -> Scenario -> Extension" logic, forming a complete product perception chain. + +**2. User Attention** +- **Peak Attention**: 0-5s (Frequent actions, clear visual focus). +- **Risk Point**: 10-17s (Product transition lacks smooth bridge). + +**3. Optimization Suggestions** +- **Visual**: Add a "Leak-proof test" close-up. +- **Audio**: Add crisp sound effects for lid opening. +- **Content**: Overlay text tags like "Food-grade 304 Stainless Steel". + +## Directory Structure + +```bash +video_breakdown_agent/ +├── README.md # Project documentation (Chinese) +├── README_en.md # Project documentation (English) +├── README_CONFIG.md # Detailed configuration guide +├── project.toml # Application marketplace metadata +├── agent.py # AgentKit deployment entry +├── requirements.txt # pip dependencies +├── pyproject.toml # uv project configuration +├── config.yaml # Configuration file (example, actual keys injected via env vars) +├── config.yaml.example # Configuration template +├── deploy.sh # Deployment script +├── video_breakdown_agent/ # Python package (core code) +│ ├── agent.py # Root Agent definition +│ ├── prompt.py # Main orchestration prompt +│ ├── sub_agents/ # Sub Agents +│ │ ├── breakdown_agent/ # Scene segmentation Agent +│ │ ├── hook_analyzer_agent/# Hook analysis Agent (SequentialAgent) +│ │ └── report_generator_agent/ # Report generation Agent +│ ├── tools/ # Tool functions +│ │ ├── process_video.py # Video preprocessing (FFmpeg + ASR) +│ │ ├── analyze_segments_vision.py # Vision analysis +│ │ ├── analyze_bgm.py # BGM analysis +│ │ ├── analyze_hook_segments.py # Hook scene extraction +│ │ ├── report_generator.py # Report generation +│ │ └── video_upload.py # TOS video upload +│ ├── hook/ # Callback hooks +│ │ ├── format_hook.py # JSON repair +│ │ └── video_upload_hook.py# File upload interceptor +│ └── utils/ # Utility classes +│ └── types.py # Pydantic data models +└── img/ # Architecture diagrams and screenshots +``` + +## Local Development + +### Prerequisites + +**Python Version:** + +- Python 3.12 or higher + +**1. Enable Volcengine Ark Model Service:** + +- Visit [Volcengine Ark Console](https://console.volcengine.com/ark/region:ark+cn-beijing/overview) +- Enable model inference service +- Create API Key (used for `MODEL_AGENT_API_KEY`) + +**2. Create TOS Bucket (for video upload):** + +- Visit [TOS Console](https://console.volcengine.com/tos/bucket) +- Create a new bucket (e.g., `video-breakdown-uploads`) +- Set region to `cn-beijing` +- Configure public read permissions (or use pre-signed URLs) + +**3. Obtain Volcengine Access Keys:** + +- Visit [IAM Key Management](https://console.volcengine.com/iam/keymanage/) +- Create Access Key/Secret Key (used for `VOLCENGINE_ACCESS_KEY` and `VOLCENGINE_SECRET_KEY`) + +**4. (Optional) Configure ASR Service:** + +- Visit [Speech Service](https://console.volcengine.com/speech/service/list) to obtain App ID and Access Key +- If not configured, the system will gracefully degrade (skip speech recognition) + +### Dependency Installation + +**Method 1: Using pip** + +```bash +pip install -r requirements.txt +``` + +**Method 2: Using uv (recommended)** + +```bash +# Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh + +# Install dependencies +uv sync --index-url https://pypi.tuna.tsinghua.edu.cn/simple +``` + +### Environment Configuration + +**Method 1: Create `.env` file (recommended for local development)** + +```bash +# Copy configuration template +cp config.yaml.example config.yaml + +# Edit .env file and fill in the following required environment variables: +MODEL_AGENT_API_KEY=your_ark_api_key +VOLCENGINE_ACCESS_KEY=your_volcengine_ak +VOLCENGINE_SECRET_KEY=your_volcengine_sk +DATABASE_TOS_BUCKET=your_tos_bucket_name +DATABASE_TOS_REGION=cn-beijing + +# Optional: ASR configuration (graceful degradation if not configured) +ASR_APP_ID=your_asr_app_id +ASR_ACCESS_KEY=your_asr_access_key + +# Optional: Vision model configuration (defaults to Doubao if not configured) +MODEL_VISION_NAME=doubao-seed-1-6-vision-250815 +# Or use Gemini: +# MODEL_VISION_NAME=gemini/gemini-2.5-pro +# GEMINI_API_KEY=your_gemini_api_key +``` + +**Method 2: Use environment variables directly** + +```bash +export MODEL_AGENT_API_KEY=your_ark_api_key +export VOLCENGINE_ACCESS_KEY=your_volcengine_ak +export VOLCENGINE_SECRET_KEY=your_volcengine_sk +export DATABASE_TOS_BUCKET=your_tos_bucket_name +``` + +**Priority**: System environment variables > `.env` file > `config.yaml` + +For detailed configuration instructions, see [README_CONFIG.md](README_CONFIG.md). + +### Running the Application + +**Method 1: Local debugging with veadk web (recommended)** + +```bash +# veadk automatically discovers video_breakdown_agent/ package +uv run veadk web +``` + +Access `http://localhost:8000` to interact with the Agent. + +**Method 2: Run directly** + +```bash +python agent.py +``` + +**Method 3: Smoke test** + +```bash +# Quick test +uv run python .scripts/smoke_test.py "Hello" + +# Full pipeline test +uv run python .scripts/smoke_test.py --pipeline-cases +``` + +## AgentKit Deployment + +### Prerequisites + +**Install AgentKit CLI:** + +```bash +pip install agentkit-sdk-python==0.5.1 +``` + +Or using `uv`: + +```bash +uv pip install agentkit-sdk-python==0.5.1 +``` + +### One-Click Deployment + +**1. Initialize configuration:** + +```bash +# Configure AgentKit credentials +agentkit config --account-id YOUR_ACCOUNT_ID --access-key YOUR_AK --secret-key YOUR_SK +``` + +**2. Deploy to cloud:** + +```bash +# Deploy (automatically creates Runtime, builds image, deploys) +agentkit launch + +# View deployment status +agentkit status + +# View Runtime logs +agentkit logs +``` + +**3. Configure environment variables in console:** + +After deployment, you need to configure the following environment variables in the [AgentKit Console](https://console.volcengine.com/agentkit): + +- `MODEL_AGENT_API_KEY`: Ark API Key +- `VOLCENGINE_ACCESS_KEY`: Volcengine Access Key +- `VOLCENGINE_SECRET_KEY`: Volcengine Secret Key +- `DATABASE_TOS_BUCKET`: TOS bucket name +- `DATABASE_TOS_REGION`: TOS region (default: `cn-beijing`) + +**4. Test deployment:** + +```bash +# Test using agentkit CLI +agentkit run "Analyze this video: https://example.com/video.mp4" +``` + +### Advanced Deployment Options + +**Custom Docker build:** + +```bash +# Build image locally +docker build -t video-breakdown-agent:latest . + +# Run container +docker run -p 8000:8000 \ + -e MODEL_AGENT_API_KEY=your_key \ + -e VOLCENGINE_ACCESS_KEY=your_ak \ + -e VOLCENGINE_SECRET_KEY=your_sk \ + video-breakdown-agent:latest +``` + +**Deployment script:** + +```bash +# Use built-in deployment script +./deploy.sh +``` + +For detailed deployment instructions, see [DEPLOY_GUIDE.md](DEPLOY_GUIDE.md). + +## Key Features + +### 1. Multi-Agent Architecture + +- **Root Agent**: Main orchestrator, responsible for understanding user intent and dispatching sub-agents +- **Breakdown Agent**: Video preprocessing + vision analysis + BGM analysis +- **Hook Analyzer Agent**: SequentialAgent (vision scoring → JSON formatting) +- **Report Generator Agent**: Markdown report generation +- **Search Agent**: Real-time web search + +### 2. Powerful Vision Analysis + +- LiteLLM unified routing supporting multiple vision models: + - Volcengine Doubao Vision + - Google Gemini 2.5 Pro + - OpenAI GPT-4o +- Switch models with one line of configuration, no code changes required + +### 3. Graceful Degradation + +- **TOS upload failure**: Automatically falls back to base64 encoding +- **ASR not configured**: Automatically skips speech recognition +- **Vision analysis failure**: Attempts model fallback +- **Hook analysis failure**: Still generates basic scene reports + +### 4. Production-Ready + +- Complete error handling and logging +- OpenTelemetry observability support (APMPlus/CozeLoop/TLS) +- Docker containerization +- AgentKit one-click deployment + +## Sample Prompts + +**Basic scene segmentation:** +``` +Analyze the scene structure of this video +``` + +**Hook analysis:** +``` +Analyze the hook effectiveness of the first 3 seconds of this video and provide professional scoring +``` + +**Complete analysis:** +``` +Generate a complete video analysis report +``` + +**Search information:** +``` +What are the latest Douyin recommendation algorithm rules? +``` + +## Demo + +[Demo screenshots or videos will be placed in the `img/` directory] + +## FAQ + +**Q1: Does FFmpeg need to be installed manually?** + +A: No. This project uses `imageio-ffmpeg` which automatically downloads and packages FFmpeg binaries. If you have system FFmpeg installed, it will be used preferentially. + +**Q2: Does the system support platforms like Douyin/Xiaohongshu/Bilibili links?** + +A: Currently only public HTTP/HTTPS video download links are supported. Platform-specific links need to be extracted to download URLs first. + +**Q3: What should I do if TOS upload fails?** + +A: The system will automatically fall back to base64 encoding to continue analysis. Check: +- Whether `VOLCENGINE_ACCESS_KEY` and `VOLCENGINE_SECRET_KEY` are correctly configured +- Whether TOS bucket permissions are correct (recommend public read or pre-signed URLs) + +**Q4: What should I do if vision analysis fails?** + +A: Check: +- Whether `MODEL_AGENT_API_KEY` is correctly configured +- Whether the vision model name is correct (must include date suffix, e.g., `doubao-seed-1-6-vision-250815`) +- Whether network can access the model endpoint + +**Q5: What should I do if ASR speech recognition fails?** + +A: ASR is optional. The system will automatically skip speech recognition and continue with vision analysis if not configured. To enable ASR, configure: +```bash +ASR_APP_ID=your_app_id +ASR_ACCESS_KEY=your_access_key +``` + +**Q6: How to switch vision models?** + +A: Modify environment variable `MODEL_VISION_NAME`: + +```bash +# Use Doubao (default) +MODEL_VISION_NAME=doubao-seed-1-6-vision-250815 + +# Use Gemini +MODEL_VISION_NAME=gemini/gemini-2.5-pro +GEMINI_API_KEY=your_gemini_api_key + +# Use GPT-4o +MODEL_VISION_NAME=gpt-4o +OPENAI_API_KEY=your_openai_api_key +``` + +**Q7: Are there any limits on video duration?** + +A: Recommended: +- Video duration: 15 seconds to 3 minutes +- File size: <100MB +- Resolution: 720p or 1080p + +Longer videos will require more processing time and model tokens. + +## References + +- [VeADK Documentation](https://volcengine.github.io/veadk-python/) +- [AgentKit Documentation](https://www.volcengine.com/docs/6459) +- [Volcengine Ark Platform](https://console.volcengine.com/ark) +- [LiteLLM Documentation](https://docs.litellm.ai/) +- [FFmpeg Documentation](https://ffmpeg.org/documentation.html) + +## Contributing + +Contributions are welcome! Please refer to [CONTRIBUTING.md](../../CONTRIBUTING.md) for contribution guidelines. + +## License + +This project is licensed under the Apache 2.0 License - see the [LICENSE](LICENSE) file for details. diff --git a/python/02-use-cases/video_breakdown_agent/agent.py b/python/02-use-cases/video_breakdown_agent/agent.py new file mode 100644 index 00000000..f0e9e2eb --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/agent.py @@ -0,0 +1,87 @@ +""" +Video Breakdown Agent — AgentKit 部署入口 + +Agent 定义位于 video_breakdown_agent/agent.py（veadk web 的唯一真相来源）。 +本文件仅添加 Runner、可观测性、AgentkitAgentServerApp 等部署层。 + +运行方式： + 1. uv run veadk web — 本地开发（veadk web 自动发现 video_breakdown_agent/） + 2. python agent.py — AgentKit 部署 +""" + +import logging +import os + +from veadk import Runner +from veadk.memory.short_term_memory import ShortTermMemory +from agentkit.apps import AgentkitAgentServerApp + +# 从包中导入唯一的 root_agent 定义 +from video_breakdown_agent.agent import root_agent + +# ==================== 日志配置 ==================== + +log_level = os.getenv("LOG_LEVEL", "INFO").upper() +logging.basicConfig(level=getattr(logging, log_level, logging.INFO)) +logger = logging.getLogger(__name__) + +# ==================== 短期记忆配置 ==================== + +app_name = "video_breakdown_agent" +short_term_memory = ShortTermMemory(backend="local") + +# ==================== Runner (支持 veadk web / smoke_test) ==================== + +runner = Runner( + agent=root_agent, + short_term_memory=short_term_memory, + app_name=app_name, +) + +# ==================== 可观测性（Observability） ==================== +# 通过环境变量启用 OpenTelemetry Tracer： +# VEADK_TRACER_APMPLUS=true — APMPlus +# VEADK_TRACER_COZELOOP=true — CozeLoop +# VEADK_TRACER_TLS=true — TLS + + +def _load_tracer() -> None: + """加载可观测性 Tracer（仅在对应环境变量为 true 时激活）""" + try: + from veadk.tracing.telemetry.exporters.apmplus_exporter import APMPlusExporter + from veadk.tracing.telemetry.exporters.cozeloop_exporter import CozeloopExporter + from veadk.tracing.telemetry.exporters.tls_exporter import TLSExporter + from veadk.tracing.telemetry.opentelemetry_tracer import OpentelemetryTracer + except ImportError: + logger.debug("Tracing 依赖未安装，跳过可观测性初始化") + return + + EXPORTER_REGISTRY = { + "VEADK_TRACER_APMPLUS": APMPlusExporter, + "VEADK_TRACER_COZELOOP": CozeloopExporter, + "VEADK_TRACER_TLS": TLSExporter, + } + + exporters = [] + for env_var, exporter_cls in EXPORTER_REGISTRY.items(): + if os.getenv(env_var, "").lower() == "true": + exporters.append(exporter_cls()) + logger.info(f"可观测性: 启用 {exporter_cls.__name__}") + + if exporters: + tracer = OpentelemetryTracer(name="veadk_tracer", exporters=exporters) + root_agent.tracers = list(root_agent.tracers or []) + [tracer] + logger.info(f"可观测性: 已加载 {len(exporters)} 个 Exporter") + + +_load_tracer() + +# ==================== Server App (支持 AgentKit 部署) ==================== + +agent_server_app = AgentkitAgentServerApp( + agent=root_agent, + short_term_memory=short_term_memory, +) + +if __name__ == "__main__": + agent_server_app.run(host="0.0.0.0", port=8000) diff --git a/python/02-use-cases/video_breakdown_agent/assets/README.md b/python/02-use-cases/video_breakdown_agent/assets/README.md new file mode 100644 index 00000000..6c013ce1 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/assets/README.md @@ -0,0 +1,158 @@ +# 技术架构图 + +## Mermaid 流程图代码 + +您可以使用以下 Mermaid 代码生成技术架构图，或使用绘图工具创建自定义架构图。 + +### Multi-Agent 架构流程图 + +```mermaid +graph TD + User[用户输入视频URL/本地文件] --> RootAgent[Root Agent 主编排器小视] + + RootAgent --> BreakdownAgent[Breakdown Agent
分镜拆解] + RootAgent --> HookAnalyzer[Hook Analyzer Agent
钩子分析] + RootAgent --> ReportGen[Report Generator Agent
报告生成] + RootAgent --> SearchAgent[Search Agent
联网搜索] + + BreakdownAgent --> ProcessVideo[process_video
FFmpeg预处理+ASR] + BreakdownAgent --> AnalyzeVision[analyze_segments_vision
LiteLLM视觉分析] + BreakdownAgent --> AnalyzeBGM[analyze_bgm
背景音乐分析] + BreakdownAgent --> VideoUpload[video_upload_to_tos
TOS上传] + + ProcessVideo --> SessionState[(Session State
breakdown_result)] + + SessionState --> HookAnalyzer + HookAnalyzer --> HookAnalysis[hook_analysis_agent
Vision模型多模态分析] + HookAnalysis --> HookFormat[hook_format_agent
JSON格式化校验] + HookFormat --> SessionState2[(Session State
hook_analysis)] + + SessionState --> ReportGen + SessionState2 --> ReportGen + ReportGen --> GenerateReport[generate_video_report
Markdown报告生成] + GenerateReport --> FinalReport[(Session State
final_report)] + + SearchAgent --> WebSearch[web_search
实时信息搜索] + + style RootAgent fill:#e1f5ff + style BreakdownAgent fill:#fff4e1 + style HookAnalyzer fill:#ffe1f5 + style ReportGen fill:#e1ffe1 + style SearchAgent fill:#f5e1ff + style SessionState fill:#ffd700 + style SessionState2 fill:#ffd700 + style FinalReport fill:#ffd700 +``` + +### 数据流转图 + +```mermaid +sequenceDiagram + participant User as 用户 + participant Root as Root Agent + participant Breakdown as Breakdown Agent + participant Hook as Hook Analyzer + participant Report as Report Generator + participant State as Session State + + User->>Root: 上传视频/提供URL + Root->>Breakdown: 委派分镜拆解任务 + + Breakdown->>Breakdown: FFmpeg视频预处理 + Breakdown->>Breakdown: 火山ASR语音识别 + Breakdown->>Breakdown: 提取关键帧 + Breakdown->>Breakdown: LiteLLM视觉分析 + Breakdown->>State: 写入 breakdown_result + Breakdown->>Root: 返回分镜数据 + + Root->>Hook: 委派钩子分析任务 + Hook->>State: 读取 breakdown_result + Hook->>Hook: 提取前3秒分镜 + Hook->>Hook: Vision模型多模态分析 + Hook->>Hook: JSON格式化校验 + Hook->>State: 写入 hook_analysis + Hook->>Root: 返回钩子评分 + + Root->>Report: 委派报告生成任务 + Report->>State: 读取 breakdown_result + Report->>State: 读取 hook_analysis + Report->>Report: 整合数据生成Markdown + Report->>State: 写入 final_report + Report->>Root: 返回完整报告 + + Root->>User: 展示分析结果 +``` + +### 火山引擎组件集成图 + +```mermaid +graph LR + Agent[Video Breakdown Agent] --> Ark[火山方舟模型] + Agent --> TOS[TOS对象存储] + Agent --> ASR[火山ASR语音识别] + Agent --> AgentKit[AgentKit运行时] + Agent --> WebSearch[Web Search搜索] + + Ark --> Model1[doubao-seed-1-6-251015
主推理模型] + Ark --> Model2[doubao-seed-1-6-vision-250815
视觉分析模型] + + TOS --> Upload[视频/图片上传] + TOS --> Storage[媒体资源存储] + + ASR --> SpeechToText[语音转文字] + ASR --> Timeline[时间轴分段] + + AgentKit --> MultiAgent[Multi-Agent编排] + AgentKit --> Memory[Session State管理] + + WebSearch --> Industry[行业资讯] + WebSearch --> Trends[平台规则] + + style Agent fill:#e1f5ff + style Ark fill:#fff4e1 + style TOS fill:#ffe1f5 + style ASR fill:#e1ffe1 + style AgentKit fill:#f5e1ff + style WebSearch fill:#ffe1e1 +``` + +## 生成架构图的方法 + +### 方法1：使用在线 Mermaid 编辑器 + +1. 访问 [Mermaid Live Editor](https://mermaid.live/) +2. 复制上述任一 Mermaid 代码 +3. 粘贴到编辑器中 +4. 导出为 PNG/SVG 图片 +5. 保存为 `architecture.jpg` 或其他格式 + +### 方法2：使用绘图工具 + +使用 Draw.io、Figma、Excalidraw 等工具绘制自定义架构图： + +**建议包含的元素：** +- Root Agent 和 4 个子 Agent +- 每个 Agent 的主要工具函数 +- Session State 数据流转 +- 火山引擎组件（方舟、TOS、ASR） +- 第三方组件（FFmpeg、LiteLLM） + +### 方法3：使用 VS Code 插件 + +1. 安装 VS Code 插件：`Markdown Preview Mermaid Support` +2. 在 Markdown 文件中查看 Mermaid 图表 +3. 截图保存 + +## 架构图说明 + +架构图应展示以下关键信息： + +1. **Multi-Agent 协作关系**：Root Agent 如何协调 4 个子 Agent +2. **数据流转路径**：Session State 如何在 Agent 之间传递数据 +3. **工具调用链路**：每个 Agent 调用哪些工具函数 +4. **火山引擎集成**：使用了哪些火山产品和服务 +5. **优雅降级机制**：TOS/ASR 失败时的回退路径 + +## 当前状态 + +> 架构图文件待补充。建议使用上述 Mermaid 代码生成或自行绘制后保存为 `img/architecture.jpg`。 diff --git a/python/02-use-cases/video_breakdown_agent/assets/architecture_video_breakdown_agent.jpg b/python/02-use-cases/video_breakdown_agent/assets/architecture_video_breakdown_agent.jpg new file mode 100644 index 00000000..2adbc5e9 Binary files /dev/null and b/python/02-use-cases/video_breakdown_agent/assets/architecture_video_breakdown_agent.jpg differ diff --git a/python/02-use-cases/video_breakdown_agent/config.yaml.example b/python/02-use-cases/video_breakdown_agent/config.yaml.example new file mode 100644 index 00000000..20340325 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/config.yaml.example @@ -0,0 +1,115 @@ +# ============================================================ +# Video Breakdown Agent — config.yaml 配置模板 +# 复制此文件为 config.yaml 并填入真实值： +# cp config.yaml.example config.yaml +# +# 注意：推荐使用 .env 文件或系统环境变量（优先级更高） +# 优先级：系统环境变量 > .env 文件 > config.yaml +# 参考文档：https://volcengine.github.io/veadk-python/configuration/ +# ============================================================ + +# ======================== 模型配置 ======================== +# VeADK 自动映射为标准环境变量： +# model.agent.name → MODEL_AGENT_NAME +# model.agent.provider → MODEL_AGENT_PROVIDER +# model.agent.api_base → MODEL_AGENT_API_BASE +# model.agent.api_key → MODEL_AGENT_API_KEY + +model: + agent: + provider: openai + name: doubao-seed-1-6-251015 # 主 Agent 推理模型 + api_base: https://ark.cn-beijing.volces.com/api/v3/ + api_key: # 填入方舟 API Key + + # ==================== 视觉模型（LiteLLM 多 Provider 支持） ==================== + # 用于分镜画面分析，通过 LiteLLM 统一路由 + # → MODEL_VISION_NAME / MODEL_VISION_API_KEY / MODEL_VISION_API_BASE + # + # LiteLLM 模型名格式速查： + # gemini/gemini-2.5-pro → 直连 Google (需 GEMINI_API_KEY) + # gemini-2.5-pro + api_base → 通过 OneRouter 等 OpenAI 兼容代理 + # doubao-seed-1-6-251015 → 火山方舟 (需 api_base + api_key) + # gpt-4o → OpenAI (需 OPENAI_API_KEY) + vision: + # 方案 A：直连 Google Gemini（推荐，1M+ context，无 tokens 超限问题） + # name: gemini/gemini-2.5-pro + # api_key: # 填入 Google AI Studio API Key + + # 方案 B：通过 OneRouter 代理 + # name: gemini-2.5-pro + # api_base: https://llm.onerouter.pro/v1 + # api_key: # 填入 OneRouter API Key + + # 方案 C：火山方舟豆包（tokens 限制较小，复杂视频可能超限） + name: doubao-seed-1-6-vision + + # ==================== BGM 分析模型 ==================== + # 用于背景音乐分析，需模型支持音频输入 + # → MODEL_BGM_NAME / MODEL_BGM_API_KEY / MODEL_BGM_API_BASE + bgm: + # name: gemini/gemini-2.0-flash # 推荐（原版配置） + # api_key: # 填入 Google AI Studio API Key + name: doubao-seed-1-6-251015 # 回退方案 + + # ==================== 格式化模型 ==================== + # → MODEL_FORMAT_NAME + # 小模型，用于 JSON 格式校验，降低成本 + format: + name: doubao-seed-1-6-250615 + +# ==================== 火山引擎凭证 ==================== +# VeADK 自动映射： +# volcengine.access_key → VOLCENGINE_ACCESS_KEY +# volcengine.secret_key → VOLCENGINE_SECRET_KEY + +volcengine: + access_key: # 火山引擎 Access Key + secret_key: # 火山引擎 Secret Key + +# ==================== TOS 对象存储 ==================== +# 用于上传本地视频文件到 TOS +# 对应标准环境变量：DATABASE_TOS_BUCKET / DATABASE_TOS_REGION +# 注意：config.yaml 中通过 getenv("DATABASE_TOS_BUCKET") 读取 + +database: + tos: + bucket: video-breakdown-uploads # TOS 存储桶名称 + region: cn-beijing # TOS 区域 + +# ==================== FFmpeg 配置 ==================== +# FFmpeg 已通过 imageio-ffmpeg 自动打包在 Python 依赖中，无需手动安装 +# 如有系统 FFmpeg 会优先使用；以下配置可覆盖自动检测 +# 对应环境变量：FFMPEG_BIN / FFMPEG_FFPROBE_BIN / FFMPEG_FRAMES_PER_SEGMENT / FFMPEG_MEDIA_TEMP_DIR + +ffmpeg: + bin: ffmpeg # FFmpeg 可执行文件路径 + ffprobe_bin: ffprobe # FFprobe 可执行文件路径 + frames_per_segment: 3 # 每个分镜采样帧数量 + media_temp_dir: ./.media-cache # 临时文件目录 + +# ==================== 火山 ASR 配置 ==================== +# 语音识别（未配置时跳过，优雅降级） +# 对应环境变量：VOLC_ASR_APP_ID / VOLC_ASR_ACCESS_KEY / VOLC_ASR_RESOURCE_ID + +asr: + app_id: # 火山引擎 ASR APP ID + access_key: # 火山引擎 ASR Access Key + resource_id: volc.bigasr.auc # ASR 资源 ID + +# ==================== Thinking 配置 ==================== +# 控制每个 Agent 的推理模式：disabled / enabled +# 对应环境变量：THINKING_ROOT_AGENT / THINKING_BREAKDOWN_AGENT 等 + +thinking: + root_agent: disabled + breakdown_agent: disabled + hook_analyzer_agent: disabled + hook_format_agent: disabled + report_agent: disabled + +# ==================== 日志级别 ==================== +# DEBUG / INFO / WARNING / ERROR + +logging: + level: INFO diff --git a/python/02-use-cases/video_breakdown_agent/dataset.json b/python/02-use-cases/video_breakdown_agent/dataset.json new file mode 100644 index 00000000..daa2e8f5 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/dataset.json @@ -0,0 +1,14 @@ +[ + { + "question": "帮我拆解这个视频的分镜结构：https://v.douyin.com/example/", + "answer": "好的，我已经为您启动了分镜拆解流程。我将首先使用 FFmpeg 进行关键帧提取，并结合多模态模型分析画面内容。您可以稍等片刻查看完整的分镜脚本。" + }, + { + "question": "分析这个视频前三秒的钩子吸引力，给出专业评分", + "answer": "明白。我将针对视频的前三秒进行深度分析，从视觉冲击力、语言钩子、情绪唤起、信息密度和节奏掌控五个维度进行评分，并为您提供改进建议。" + }, + { + "question": "生成一份完整的视频分析报告", + "answer": "没问题。我将整合已有的分镜拆解数据和钩子分析结果，为您生成一份包含视频元信息、详细分镜表及专业优化建议的 Markdown 报告。" + } +] diff --git a/python/02-use-cases/video_breakdown_agent/deploy.sh b/python/02-use-cases/video_breakdown_agent/deploy.sh new file mode 100755 index 00000000..232b8a4c --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/deploy.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Video Breakdown Agent 部署脚本 + +set -e + +# 确保在项目根目录执行 +if [ ! -f "agent.py" ]; then + echo "请在 video_breakdown_agent 项目根目录执行此脚本" + exit 1 +fi + +AGENT_NAME="video_breakdown_agent" +ENTRY_POINT="agent.py" + +echo "🚀 配置 AgentKit 部署参数..." +agentkit config \ + --agent_name "${AGENT_NAME}" \ + --entry_point "${ENTRY_POINT}" \ + --launch_type cloud + +echo "🔧 配置环境变量..." +agentkit config \ + -e DATABASE_TOS_BUCKET="${DATABASE_TOS_BUCKET:-video-breakdown-uploads}" \ + -e DATABASE_TOS_REGION="${DATABASE_TOS_REGION:-cn-beijing}" \ + -e FFMPEG_BIN="${FFMPEG_BIN:-ffmpeg}" \ + -e FFPROBE_BIN="${FFPROBE_BIN:-ffprobe}" + +echo "✅ 配置完成。现在可以执行 'agentkit launch' 进行部署。" +echo "" +echo "注意事项：" +echo " 1. 确保已配置 VOLCENGINE_ACCESS_KEY 和 VOLCENGINE_SECRET_KEY" +echo " 2. 确保 TOS 存储桶已创建" +echo " 3. FFmpeg 已通过 imageio-ffmpeg 打包在 Python 依赖中，无需单独安装" +echo " 4. 如需 ASR 功能，请配置 ASR_APP_ID 和 ASR_ACCESS_KEY" diff --git a/python/02-use-cases/video_breakdown_agent/docs/DEPLOY_GUIDE.md b/python/02-use-cases/video_breakdown_agent/docs/DEPLOY_GUIDE.md new file mode 100644 index 00000000..33af4c70 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/docs/DEPLOY_GUIDE.md @@ -0,0 +1,132 @@ +# 部署指南 + +## 🚀 云端重新部署步骤 + +由于更新了 `requirements.txt`（添加了 `imageio-ffmpeg` 依赖），需要重新构建并部署应用。 + +### 1️⃣ 确认环境变量已配置 + +在 AgentKit 控制台确认以下环境变量已正确填写： + +**必需变量**： +```bash +MODEL_AGENT_API_KEY=<您的豆包 API Key> +VOLCENGINE_ACCESS_KEY=<您的火山引擎 AK> +VOLCENGINE_SECRET_KEY=<您的火山引擎 SK> +DATABASE_TOS_BUCKET=video-breakdown-uploads +DATABASE_TOS_REGION=cn-beijing +``` + +### 2️⃣ 重新部署 + +在项目根目录执行： + +```bash +# 确保在正确的目录 +cd /Users/edy/Downloads/agentkit-samples-main/02-use-cases/video_breakdown_agent + +# 激活虚拟环境（如果使用 conda） +source ~/.venv/bin/activate # 或 conda activate video-breakdown-agent + +# 重新部署到云端 +agentkit launch +``` + +### 3️⃣ 等待构建完成 + +部署过程包括： +1. 📦 打包项目代码 +2. 🔧 上传到 TOS +3. 🏗️ 云端构建 Docker 镜像（安装 imageio-ffmpeg） +4. 🚢 部署到 Runtime +5. ✅ 服务就绪 + +预计耗时：3-5 分钟 + +### 4️⃣ 验证部署 + +部署成功后，在控制台测试： + +1. **基础对话测试**： + ``` + 你好，介绍一下你的功能 + ``` + +2. **视频分析测试**： + ``` + 帮我分析这个视频的分镜 + ``` + 然后上传一个短视频文件（< 50MB） + +如果配置正确，Agent 应该能够： +- ✅ 接收并处理视频文件 +- ✅ 使用 FFmpeg 进行分镜拆解 +- ✅ 返回完整的分析结果 + +--- + +## 🐛 常见问题 + +### Q1: 部署时提示 CR 配额超限？ + +**解决方案**：使用已有的 CR 实例 + +```bash +agentkit config --cr_instance_name nodesk-center +agentkit launch +``` + +### Q2: 运行时报错"FFmpeg 缺失"？ + +**原因**：旧版本的镜像没有包含 `imageio-ffmpeg` + +**解决方案**：重新部署（按照上述步骤） + +### Q3: 视频分析失败，提示 TOS 权限错误？ + +**原因**：环境变量 `VOLCENGINE_ACCESS_KEY` 或 `VOLCENGINE_SECRET_KEY` 未配置或错误 + +**解决方案**： +1. 在控制台检查环境变量配置 +2. 重启 Runtime 使配置生效 + +### Q4: ASR 语音识别不工作？ + +**说明**：ASR 是可选功能，未配置时会跳过语音识别，不影响视频分析 + +**解决方案**（如需启用）： +```bash +# 在控制台添加环境变量 +ASR_APP_ID=<您的 ASR App ID> +ASR_ACCESS_KEY=<您的 ASR Access Key> +``` + +--- + +## 📊 部署后检查清单 + +- [ ] Runtime 状态为 "运行中" +- [ ] 环境变量已配置（至少 3 个必需变量） +- [ ] 能够进行基础对话 +- [ ] 能够上传视频文件 +- [ ] 视频分析功能正常 +- [ ] TOS 存储桶可访问 + +--- + +## 🔗 相关链接 + +- AgentKit 控制台：https://console.volcengine.com/agentkit +- TOS 存储桶管理：https://console.volcengine.com/tos/bucket +- 火山方舟 API Key：https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey +- IAM 密钥管理：https://console.volcengine.com/iam/keymanage/ + +--- + +## 💬 获取帮助 + +如遇到问题，请提供： +1. Runtime ID 或名称 +2. 错误信息截图 +3. 环境变量配置（脱敏后） +4. 部署日志 diff --git a/python/02-use-cases/video_breakdown_agent/docs/README_CONFIG.md b/python/02-use-cases/video_breakdown_agent/docs/README_CONFIG.md new file mode 100644 index 00000000..aa8ae078 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/docs/README_CONFIG.md @@ -0,0 +1,251 @@ +# Video Breakdown Agent 配置指南 + +本文档说明 Video Breakdown Agent 所需的全部配置项、优先级规则、可观测性集成方式及常见问题排查。 + +## 快速开始 + +```bash +# 1. 复制环境变量模板 +cp .env.example .env + +# 2. 编辑 .env，至少填写以下必需项 +# MODEL_AGENT_API_KEY — 方舟模型 API Key +# VOLCENGINE_ACCESS_KEY — 火山引擎 AK（上传视频需要） +# VOLCENGINE_SECRET_KEY — 火山引擎 SK + +# 3. 启动 +uv run veadk web +``` + +## 配置优先级 + +VeADK 按以下优先级（从高到低）读取配置： + +| 优先级 | 来源 | 说明 | +|--------|------|------| +| 1 | 系统环境变量 | 通过 `export` 或 CI/CD 注入 | +| 2 | `.env` 文件 | 项目根目录下，推荐用于本地开发 | +| 3 | `config.yaml` | 项目根目录下，作为备选 | + +> 参考：[VeADK 配置项文档](https://volcengine.github.io/veadk-python/configuration/) + +## 工具返回数据优化策略 + +为避免工具返回数据过大导致 LLM context 超限，本项目对视觉分析工具返回数据做了精简处理： + +### 优化细节 + +- **完整数据存储位置**：`tool_context.state["vision_analysis_result"]`（包含完整的 base64 frame_urls） +- **工具返回数据**：移除 base64 图片 URL，替换为占位符 `"（base64图片已省略）"` +- **数据量减少**：从 ~2200KB 精简到 ~3KB（减少 99.8%） +- **后续工具使用**：钩子分析、报告生成等工具从 `tool_context.state` 读取完整数据，不受影响 + +### 为什么需要精简？ + +- **Token 限制**：豆包模型的 context window 相对较小，大量 base64 图片数据会导致 `Total tokens exceed max message tokens` 错误 +- **性能优化**：减少 LLM 处理的数据量，提升响应速度 +- **成本控制**：减少 token 消耗，降低 API 调用成本 + +### 如何获取完整数据？ + +如果你的自定义工具需要访问完整的帧图 URL（含 base64），可以这样读取： + +```python +from google.adk.tools import ToolContext + +async def my_custom_tool(tool_context: ToolContext) -> str: + # 从 session state 读取完整的视觉分析结果 + vision_result = tool_context.state.get("vision_analysis_result", []) + + # vision_result 是一个列表，每个元素包含完整的 frame_urls（含 base64） + for segment in vision_result: + frame_urls = segment.get("frame_urls", []) # 完整的 base64 URLs + # ... 使用 frame_urls 进行后续处理 +``` + +> 参考实现：[`tools/analyze_hook_segments.py`](video_breakdown_agent/tools/analyze_hook_segments.py) 中从 `tool_context.state` 读取完整数据的示例。 + +## 环境变量一览 + +### 必需配置 + +| 环境变量 | 说明 | 示例值 | +|---------|------|--------| +| `MODEL_AGENT_API_KEY` | 方舟模型 API Key | `your_api_key` | + +### 模型配置（MODEL_*） + +| 环境变量 | 说明 | 默认值 | +|---------|------|--------| +| `MODEL_AGENT_NAME` | 主 Agent 推理模型名称 | `doubao-seed-1-6-251015` | +| `MODEL_AGENT_PROVIDER` | 模型提供商（火山方舟使用 `openai`） | `openai` | +| `MODEL_AGENT_API_BASE` | 模型 API 地址 | `https://ark.cn-beijing.volces.com/api/v3/` | +| `MODEL_AGENT_API_KEY` | 模型 API 密钥 | — | +| `MODEL_VISION_NAME` | 视觉分析模型（豆包 Responses API） | `doubao-seed-1-6-vision-250815` | +| `MODEL_VISION_API_KEY` | 视觉模型 API Key | 回退到 `MODEL_AGENT_API_KEY` | +| `MODEL_VISION_API_BASE` | 视觉模型 API 地址 | 回退到 `MODEL_AGENT_API_BASE` | +| `MODEL_BGM_NAME` | BGM 分析模型（豆包文本 API） | `doubao-seed-1-6-251015` | +| `MODEL_BGM_API_KEY` | BGM 模型 API Key | 回退到 `MODEL_AGENT_API_KEY` | +| `MODEL_BGM_API_BASE` | BGM 模型 API 地址 | 回退到 `MODEL_AGENT_API_BASE` | +| `MODEL_FORMAT_NAME` | 格式化模型（JSON 校验） | `doubao-seed-1-6-251015` | +| `VISION_CONCURRENCY` | 视觉分析并发数 | `3` | + +### 豆包 API 调用架构 + +本项目使用豆包官方 API 格式，不依赖 LiteLLM 进行工具层的 LLM 调用： + +| 组件 | API 类型 | 模型 | Endpoint | +|------|---------|------|----------| +| 主 Agent / 子 Agent | 文本模型（VeADK 框架管理） | `doubao-seed-1-6-251015` | `/chat/completions` | +| `analyze_segments_vision` 工具 | 视觉模型（自定义 httpx 调用） | `doubao-seed-1-6-vision-250815` | `/responses` | +| `analyze_bgm` 工具 | 文本模型（自定义 httpx 调用） | `doubao-seed-1-6-251015` | `/chat/completions` | +| `hook_analysis_agent` | 视觉模型（VeADK 框架管理） | `doubao-seed-1-6-vision-250815` | VeADK 内部路由 | + +> **重要**：豆包视觉模型使用 `/responses` endpoint（非标准 OpenAI 格式），封装在 `video_breakdown_agent/utils/doubao_client.py` 中。 + +### 火山引擎凭证（VOLCENGINE_*） + +| 环境变量 | 说明 | 备注 | +|---------|------|------| +| `VOLCENGINE_ACCESS_KEY` | 火山引擎 Access Key | 上传视频到 TOS 时需要 | +| `VOLCENGINE_SECRET_KEY` | 火山引擎 Secret Key | 上传视频到 TOS 时需要 | + +### TOS 对象存储（DATABASE_TOS_*） + +| 环境变量 | 说明 | 默认值 | +|---------|------|--------| +| `DATABASE_TOS_BUCKET` | TOS 存储桶名称 | `video-breakdown-uploads` | +| `DATABASE_TOS_REGION` | TOS 区域 | `cn-beijing` | +| `DATABASE_TOS_ENDPOINT` | TOS 端点（可选） | 自动生成 | + +> 注：为向后兼容，代码同时支持旧变量名 `TOS_BUCKET` / `TOS_REGION`（Deprecated，将在后续版本移除）。 + +### Thinking 配置 + +| 环境变量 | 说明 | 默认值 | +|---------|------|--------| +| `THINKING_ROOT_AGENT` | 主 Agent 推理模式 | `disabled` | +| `THINKING_BREAKDOWN_AGENT` | 分镜拆解 Agent | `disabled` | +| `THINKING_HOOK_ANALYZER_AGENT` | 钩子分析 Agent | `disabled` | +| `THINKING_HOOK_FORMAT_AGENT` | 格式化 Agent | `disabled` | +| `THINKING_REPORT_AGENT` | 报告生成 Agent | `disabled` | + +### Skills 配置 + +| 环境变量 | 说明 | 备注 | +|---------|------|------| +| `SKILL_SPACE_ID` | AgentKit Skill Space ID | 配置后从平台加载 Skills | + +未配置 `SKILL_SPACE_ID` 时，自动检测本地 `skills/` 目录。 + +## 可观测性配置 + +VeADK 支持三种 OpenTelemetry 后端，配置对应环境变量后自动启用： + +### APMPlus（火山引擎 APM，推荐） + +```bash +OBSERVABILITY_OPENTELEMETRY_APMPLUS_ENDPOINT=http://apmplus-cn-beijing.volces.com:4317 +OBSERVABILITY_OPENTELEMETRY_APMPLUS_API_KEY= +OBSERVABILITY_OPENTELEMETRY_APMPLUS_SERVICE_NAME=video_breakdown_agent +``` + +### CozeLoop（Coze 平台观测） + +```bash +OBSERVABILITY_OPENTELEMETRY_COZELOOP_ENDPOINT=https://api.coze.cn/v1/loop/opentelemetry/v1/traces +OBSERVABILITY_OPENTELEMETRY_COZELOOP_API_KEY= +OBSERVABILITY_OPENTELEMETRY_COZELOOP_SERVICE_NAME= +``` + +### TLS（火山引擎日志服务） + +```bash +OBSERVABILITY_OPENTELEMETRY_TLS_ENDPOINT=https://tls-cn-beijing.volces.com:4318/v1/traces +OBSERVABILITY_OPENTELEMETRY_TLS_SERVICE_NAME= +OBSERVABILITY_OPENTELEMETRY_TLS_REGION=cn-beijing +``` + +### 启用开关 + +配置好上述环境变量后，还需设置对应的开关变量为 `true`： + +```bash +VEADK_TRACER_APMPLUS=true # 启用 APMPlus Exporter +VEADK_TRACER_COZELOOP=true # 启用 CozeLoop Exporter +VEADK_TRACER_TLS=true # 启用 TLS Exporter +``` + +> 三种后端可任选其一或组合使用。详见 [VeADK 可观测文档](https://volcengine.github.io/veadk-python/observability/)。 + +## config.yaml 与 .env 的对应关系 + +``` +config.yaml 字段 → 环境变量 +────────────────────────────────────────────────── +model.agent.name → MODEL_AGENT_NAME +model.agent.provider → MODEL_AGENT_PROVIDER +model.agent.api_base → MODEL_AGENT_API_BASE +model.agent.api_key → MODEL_AGENT_API_KEY +model.vision.name → MODEL_VISION_NAME +model.vision.api_key → MODEL_VISION_API_KEY +model.vision.api_base → MODEL_VISION_API_BASE +model.bgm.name → MODEL_BGM_NAME +model.bgm.api_key → MODEL_BGM_API_KEY +model.bgm.api_base → MODEL_BGM_API_BASE +model.format.name → MODEL_FORMAT_NAME +volcengine.access_key → VOLCENGINE_ACCESS_KEY +volcengine.secret_key → VOLCENGINE_SECRET_KEY +database.tos.bucket → DATABASE_TOS_BUCKET +database.tos.region → DATABASE_TOS_REGION +asr.app_id → VOLC_ASR_APP_ID +asr.access_key → VOLC_ASR_ACCESS_KEY +asr.resource_id → VOLC_ASR_RESOURCE_ID +ffmpeg.bin_path → FFMPEG_BIN +ffmpeg.probe_bin_path → FFPROBE_BIN +ffmpeg.media_temp_dir → MEDIA_TEMP_DIR +thinking.* → THINKING_* +``` + +> **模型说明**：视觉分析使用豆包 Responses API（`/responses` endpoint），文本分析使用标准 Chat Completions API。封装代码位于 `video_breakdown_agent/utils/doubao_client.py`。 + +## 常见问题 + +### 模型未激活 + +``` +The model or endpoint xxx does not exist or you do not have access to it. +``` + +请前往 [火山引擎方舟控制台](https://console.volcengine.com/ark) 激活对应模型。当前使用： +- 文本模型：`doubao-seed-1-6-251015` +- 视觉模型：`doubao-seed-1-6-vision-250815`（需单独激活） + +### TOS 连接失败 + +``` +TOS 存储桶 xxx 不存在，请先创建 +``` + +1. 确认 `DATABASE_TOS_BUCKET` 对应的存储桶已在火山引擎 TOS 控制台创建 +2. 确认 `VOLCENGINE_ACCESS_KEY` 和 `VOLCENGINE_SECRET_KEY` 正确 +3. 确认 `DATABASE_TOS_REGION` 与存储桶所在区域一致 + +### FFmpeg/FFprobe 未找到 + +``` +FileNotFoundError: ffmpeg not found +``` + +请确认已安装 FFmpeg 和 FFprobe，且可执行文件在系统 PATH 中，或通过 `FFMPEG_BIN` / `FFPROBE_BIN` 环境变量指定路径。 + +### VeADK 未读取到 config.yaml + +确保从项目根目录启动： + +```bash +cd 02-use-cases/video_breakdown_agent +uv run veadk web +``` + +VeADK 从当前工作目录读取 `config.yaml`。 diff --git a/python/02-use-cases/video_breakdown_agent/project.yaml b/python/02-use-cases/video_breakdown_agent/project.yaml new file mode 100644 index 00000000..574512b5 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/project.yaml @@ -0,0 +1,67 @@ +name: "智能短视频拆解分析助手" +name_en: "Video Breakdown Analyzer" +description: "专业的短视频分镜拆解和深度分析工具，自动生成分镜脚本、前三秒钩子评分和完整分析报告" +details: | + 这是一个基于火山引擎 AgentKit 构建的智能短视频分析系统。系统采用 Multi-Agent 架构， + 集成 FFmpeg 视频处理、火山 ASR 语音识别、LiteLLM 多模态视觉分析和 TOS 对象存储， + 能够对短视频进行全方位的专业分析。 + + 核心能力： + - 视频分镜拆解：自动识别视频分镜，提取关键帧并分析画面内容 + - 前三秒钩子分析：从视觉冲击力、语言钩子、情绪唤起等5个维度评分 + - 专业报告生成：整合所有分析数据生成 Markdown 格式专业报告 + - 联网搜索：实时获取短视频行业最新资讯和平台规则 +tags: + industry: "通用" + from: "杭州没有桌子人工智能有限公司" + framework: "veadk" + language: "python" + tech: + - "Multi-Agent" + - "FFmpeg" + - "ASR" + - "视觉分析" + - "TOS存储" +type: "application" + +scenarios: + - name: "短视频创作优化" + desc: "为内容创作者提供分镜拆解和钩子评分，优化视频吸引力和用户留存率。" + - name: "视频内容审核分析" + desc: "自动拆解视频内容结构，辅助审核团队快速理解视频脉络和关键信息点。" + - name: "竞品视频研究" + desc: "批量分析竞品视频的分镜手法、钩子策略和叙事节奏，提取创作灵感。" + - name: "广告创意评估" + desc: "针对短视频广告素材，评估前三秒钩子效果，预测投放转化潜力。" + +prompts: + - text: "帮我拆解这个视频的分镜结构" + resource_url: "" + - text: "分析这个视频前三秒的钩子吸引力，给出专业评分" + resource_url: "" + - text: "生成完整的视频分析报告" + resource_url: "" + +components: + - type: "tool" + product: "Skills_Sandbox" + - type: "infra" + product: "TOS" + +envs: + - name: "火山方舟模型 API Key" + key: "MODEL_AGENT_API_KEY" + url: "https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey" + - name: "火山引擎访问密钥" + key: "VOLCENGINE_ACCESS_KEY" + url: "https://console.volcengine.com/iam/keymanage/" + - name: "火山引擎密钥" + key: "VOLCENGINE_SECRET_KEY" + url: "https://console.volcengine.com/iam/keymanage/" + - name: "TOS 存储桶名称" + key: "DATABASE_TOS_BUCKET" + url: "https://console.volcengine.com/tos/bucket" + +models: + - "doubao-seed-1-6-251015" + - "doubao-seed-1-6-vision-250815" diff --git a/python/02-use-cases/video_breakdown_agent/pyproject.toml b/python/02-use-cases/video_breakdown_agent/pyproject.toml new file mode 100644 index 00000000..a91e391b --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/pyproject.toml @@ -0,0 +1,41 @@ +[project] +name = "video-breakdown-agent" +version = "0.3.0" +description = "视频分镜拆解和前三秒钩子分析 AgentKit Agent（Multi-Agent 架构）" +requires-python = ">=3.12,<3.13" +dependencies = [ + # AgentKit 核心三件套 + "veadk-python==0.5.20", + "google-adk==1.21.0", # veadk-python 0.5.20 锁定此版本 + "agentkit-sdk-python==0.5.1", + # HTTP 客户端 + "httpx>=0.26.0", + # TOS 对象存储（视频上传） + "tos>=2.8.7", + # JSON 修复（format hook） + "json-repair>=0.54.1", + # 配置管理 + "python-dotenv>=1.0.0", + # 生产模式服务器 + "uvicorn>=0.34.0", + # Web 框架（A2A + MCP） + "fastapi>=0.115.0", + # MCP Server + "fastmcp>=2.0.0", + # 异步 HTTP + "aiohttp>=3.11.0", + # FFmpeg 二进制（随 Python 包分发，无需系统安装） + "imageio-ffmpeg>=0.5.1", +] + +[dependency-groups] +dev = [ + "pytest>=8.0.0", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["video_breakdown_agent"] diff --git a/python/02-use-cases/video_breakdown_agent/requirements.txt b/python/02-use-cases/video_breakdown_agent/requirements.txt new file mode 100644 index 00000000..80d9a8b9 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/requirements.txt @@ -0,0 +1,34 @@ +# pip 兼容依赖列表（推荐使用 pyproject.toml + uv） +# pip install -r requirements.txt + +# AgentKit 核心 +veadk-python==0.5.20 +google-adk==1.21.0 +agentkit-sdk-python==0.5.1 + +# HTTP 客户端 +httpx>=0.26.0 + +# TOS 对象存储（视频上传） +tos>=2.8.7 + +# JSON 修复（format hook） +json-repair>=0.54.1 + +# 配置管理 +python-dotenv>=1.0.0 + +# FFmpeg 二进制（随 Python 包分发，无需系统安装 FFmpeg） +imageio-ffmpeg>=0.5.1 + +# 生产模式服务器 +uvicorn>=0.34.0 + +# Web 框架（A2A + MCP） +fastapi>=0.115.0 + +# MCP Server +fastmcp>=2.0.0 + +# 异步 HTTP +aiohttp>=3.11.0 diff --git a/python/02-use-cases/video_breakdown_agent/sandbox/docker-compose.yml b/python/02-use-cases/video_breakdown_agent/sandbox/docker-compose.yml new file mode 100644 index 00000000..d1884487 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/sandbox/docker-compose.yml @@ -0,0 +1,100 @@ +version: '3.8' + +services: + # 沙箱 MySQL + mysql_sandbox: + image: mysql:8.0 + container_name: breakdown_mysql_sandbox + environment: + MYSQL_ROOT_PASSWORD: sandbox_root_pass + MYSQL_DATABASE: video_breakdown_sandbox + MYSQL_USER: sandbox_user + MYSQL_PASSWORD: sandbox_pass + ports: + - "3307:3306" # 避免与生产 MySQL 冲突 + volumes: + - mysql_sandbox_data:/var/lib/mysql + networks: + - breakdown_sandbox + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 5s + retries: 5 + + # 沙箱 Redis + redis_sandbox: + image: redis:7-alpine + container_name: breakdown_redis_sandbox + ports: + - "6380:6379" # 避免与生产 Redis 冲突 + networks: + - breakdown_sandbox + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # 沙箱 video-breakdown API + breakdown_api_sandbox: + build: + context: ../../../video-breakdown-master-449a45e1b0108b8d8e33c3d7a69eb7ab00c3dd4f + dockerfile: Dockerfile + container_name: breakdown_api_sandbox + ports: + - "7115:7114" # 沙箱 API 端口 + environment: + - DATABASE_URL=mysql+aiomysql://sandbox_user:sandbox_pass@mysql_sandbox:3306/video_breakdown_sandbox + - REDIS_HOST=redis_sandbox + - REDIS_PORT=6379 + - PORT=7114 + - WORKER_COUNT=4 + - TOS_OUTPUT_BUCKET=${SANDBOX_TOS_BUCKET} + - VOLC_ASR_APPID=${VOLC_ASR_APPID} + - VOLC_ASR_TOKEN=${VOLC_ASR_TOKEN} + - ONEROUTER_API_KEY=${ONEROUTER_API_KEY} + depends_on: + mysql_sandbox: + condition: service_healthy + redis_sandbox: + condition: service_healthy + networks: + - breakdown_sandbox + volumes: + - breakdown_sandbox_data:/app/data + + # 沙箱 Celery Worker + breakdown_worker_sandbox: + build: + context: ../../../video-breakdown-master-449a45e1b0108b8d8e33c3d7a69eb7ab00c3dd4f + dockerfile: Dockerfile + container_name: breakdown_worker_sandbox + command: celery -A app.celery_app worker --loglevel=info --concurrency=4 + environment: + - DATABASE_URL=mysql+aiomysql://sandbox_user:sandbox_pass@mysql_sandbox:3306/video_breakdown_sandbox + - REDIS_HOST=redis_sandbox + - REDIS_PORT=6379 + - TOS_OUTPUT_BUCKET=${SANDBOX_TOS_BUCKET} + - VOLC_ASR_APPID=${VOLC_ASR_APPID} + - VOLC_ASR_TOKEN=${VOLC_ASR_TOKEN} + - ONEROUTER_API_KEY=${ONEROUTER_API_KEY} + depends_on: + mysql_sandbox: + condition: service_healthy + redis_sandbox: + condition: service_healthy + breakdown_api_sandbox: + condition: service_started + networks: + - breakdown_sandbox + volumes: + - breakdown_sandbox_data:/app/data + +networks: + breakdown_sandbox: + driver: bridge + +volumes: + mysql_sandbox_data: + breakdown_sandbox_data: diff --git a/python/02-use-cases/video_breakdown_agent/sandbox/sandbox.env.example b/python/02-use-cases/video_breakdown_agent/sandbox/sandbox.env.example new file mode 100644 index 00000000..3159eed1 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/sandbox/sandbox.env.example @@ -0,0 +1,14 @@ +# ======================================== +# 沙箱环境变量配置模板 +# 复制此文件为 .env 并填入真实值 +# ======================================== + +# 沙箱专用 TOS 桶（避免污染生产数据） +SANDBOX_TOS_BUCKET=breakdown-sandbox-bucket + +# 火山 ASR 配置（可使用测试账号） +VOLC_ASR_APPID=your_test_appid +VOLC_ASR_TOKEN=your_test_token + +# OneRouter API Key（可使用测试 key） +ONEROUTER_API_KEY=your_test_api_key diff --git a/python/02-use-cases/video_breakdown_agent/sandbox/sandbox_manager.py b/python/02-use-cases/video_breakdown_agent/sandbox/sandbox_manager.py new file mode 100644 index 00000000..69a46013 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/sandbox/sandbox_manager.py @@ -0,0 +1,233 @@ +""" +沙箱环境管理器 +提供沙箱的启动、停止、重置、健康检查等功能 +""" + +import subprocess +import sys +import time +from pathlib import Path + +import httpx + + +class SandboxManager: + """沙箱环境管理器""" + + def __init__(self, sandbox_dir: str = None): + self.sandbox_dir = Path(sandbox_dir) if sandbox_dir else Path(__file__).parent + self.compose_file = self.sandbox_dir / "docker-compose.yml" + self.env_file = self.sandbox_dir / ".env" + self.api_url = "http://localhost:7115" + + def start(self, rebuild: bool = False): + """启动沙箱环境""" + print("启动沙箱环境...") + + # 检查 .env 文件 + if not self.env_file.exists(): + example_env = self.sandbox_dir / "sandbox.env.example" + if example_env.exists(): + print( + "警告: 未找到 .env 文件，请复制 sandbox.env.example 为 .env 并填入配置" + ) + print(f" cp {example_env} {self.env_file}") + return + + cmd = [ + "docker-compose", + "-f", + str(self.compose_file), + "--env-file", + str(self.env_file), + "up", + "-d", + ] + if rebuild: + cmd.insert(-1, "--build") + + result = subprocess.run( + cmd, capture_output=True, text=True, cwd=str(self.sandbox_dir) + ) + if result.returncode != 0: + print(f"沙箱启动失败: {result.stderr}") + raise Exception(f"沙箱启动失败: {result.stderr}") + + print("等待服务就绪...") + self.wait_for_health() + print("沙箱环境已启动") + print(f" API 地址: {self.api_url}") + print(" MySQL: localhost:3307") + print(" Redis: localhost:6380") + + def stop(self): + """停止沙箱环境（保留数据）""" + print("停止沙箱环境...") + cmd = [ + "docker-compose", + "-f", + str(self.compose_file), + "down", + ] + subprocess.run(cmd, cwd=str(self.sandbox_dir)) + print("沙箱环境已停止") + + def reset(self): + """重置沙箱环境（清除所有数据和卷）""" + print("重置沙箱环境...") + cmd = [ + "docker-compose", + "-f", + str(self.compose_file), + "down", + "-v", + ] + subprocess.run(cmd, cwd=str(self.sandbox_dir)) + print("沙箱环境已重置，所有数据已清除") + + def status(self): + """查看沙箱环境状态""" + cmd = [ + "docker-compose", + "-f", + str(self.compose_file), + "ps", + ] + subprocess.run(cmd, cwd=str(self.sandbox_dir)) + + def logs(self, service: str = None, follow: bool = False): + """查看沙箱日志""" + cmd = [ + "docker-compose", + "-f", + str(self.compose_file), + "logs", + ] + if follow: + cmd.append("-f") + if service: + cmd.append(service) + + subprocess.run(cmd, cwd=str(self.sandbox_dir)) + + def wait_for_health(self, timeout: int = 120): + """等待沙箱服务健康""" + start_time = time.time() + + while time.time() - start_time < timeout: + try: + with httpx.Client(timeout=5.0) as client: + response = client.get(f"{self.api_url}/health") + if response.status_code == 200: + return True + except Exception: + pass + time.sleep(3) + + raise TimeoutError( + f"沙箱服务未能在 {timeout} 秒内启动，请检查 Docker 日志: " + f"docker-compose -f {self.compose_file} logs" + ) + + def get_api_url(self) -> str: + """获取沙箱 API 地址""" + return self.api_url + + def run_test(self, video_url: str): + """在沙箱中运行简单测试""" + print(f"在沙箱中测试视频: {video_url}") + + try: + with httpx.Client(timeout=30.0) as client: + # 提交任务 + response = client.post( + f"{self.api_url}/api/v1/breakdown/submit", + json={"video_source": video_url}, + ) + result = response.json() + + if result["code"] != 0: + print(f"提交失败: {result['message']}") + return + + task_id = result["data"]["task_id"] + print(f"任务已提交，task_id: {task_id}") + + # 轮询等待 + while True: + response = client.get( + f"{self.api_url}/api/v1/breakdown/status/{task_id}" + ) + status_data = response.json()["data"] + + current_status = status_data["status"] + progress = status_data.get("progress", 0) + step = status_data.get("current_step", "unknown") + + if current_status == "completed": + print("任务完成!") + # 获取结果 + response = client.get( + f"{self.api_url}/api/v1/breakdown/result/{task_id}" + ) + result_data = response.json()["data"] + print(f" 分镜数: {result_data.get('segment_count', 0)}") + print(f" 时长: {result_data.get('duration', 0):.1f}s") + return task_id + elif current_status == "failed": + print( + f"任务失败: {status_data.get('error_message', '未知错误')}" + ) + return None + + print(f" 处理中... {progress}% ({step})") + time.sleep(5) + + except httpx.ConnectError: + print(f"无法连接到沙箱 API ({self.api_url})，请确认沙箱已启动") + except Exception as e: + print(f"测试异常: {e}") + + +def main(): + """CLI 入口""" + manager = SandboxManager() + + if len(sys.argv) < 2: + print("沙箱环境管理器") + print() + print("用法:") + print(" python sandbox_manager.py start [--rebuild] 启动沙箱") + print(" python sandbox_manager.py stop 停止沙箱") + print(" python sandbox_manager.py reset 重置沙箱") + print(" python sandbox_manager.py status 查看状态") + print(" python sandbox_manager.py logs [service] 查看日志") + print(" python sandbox_manager.py test 运行测试") + sys.exit(0) + + command = sys.argv[1] + + if command == "start": + manager.start(rebuild="--rebuild" in sys.argv) + elif command == "stop": + manager.stop() + elif command == "reset": + manager.reset() + elif command == "status": + manager.status() + elif command == "logs": + service = sys.argv[2] if len(sys.argv) > 2 else None + manager.logs(service=service, follow="-f" in sys.argv) + elif command == "test": + if len(sys.argv) < 3: + print("用法: python sandbox_manager.py test ") + sys.exit(1) + manager.start() + manager.run_test(sys.argv[2]) + else: + print(f"未知命令: {command}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/python/02-use-cases/video_breakdown_agent/setup_env.sh b/python/02-use-cases/video_breakdown_agent/setup_env.sh new file mode 100755 index 00000000..4b846c70 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/setup_env.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Video Breakdown Agent 环境配置辅助脚本 + +set -e + +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$PROJECT_DIR" + +echo "🔧 Video Breakdown Agent 环境配置向导" +echo "========================================" +echo "" + +# 检查 .env 文件是否已存在 +if [ -f ".env" ]; then + echo "⚠️ 检测到现有 .env 文件" + read -p "是否要覆盖? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "❌ 已取消，保留现有配置" + exit 0 + fi +fi + +# 复制模板 +if [ ! -f ".env.example" ]; then + echo "❌ 错误：找不到 .env.example 模板文件" + exit 1 +fi + +cp .env.example .env +echo "✅ 已创建 .env 文件（从 .env.example 复制）" +echo "" + +# 引导用户填写必需变量 +echo "📝 请填写以下必需的环境变量：" +echo "----------------------------------------" +echo "" + +# MODEL_AGENT_API_KEY +echo "1️⃣ 火山方舟模型 API Key" +echo " 获取地址：https://console.volcengine.com/ark/region:ark+cn-beijing/apiKey" +read -p " 输入 MODEL_AGENT_API_KEY: " MODEL_AGENT_API_KEY +if [ -n "$MODEL_AGENT_API_KEY" ]; then + sed -i '' "s|^MODEL_AGENT_API_KEY=.*|MODEL_AGENT_API_KEY=$MODEL_AGENT_API_KEY|" .env + echo " ✅ 已保存" +fi +echo "" + +# VOLCENGINE_ACCESS_KEY +echo "2️⃣ 火山引擎访问密钥" +echo " 获取地址：https://console.volcengine.com/iam/keymanage/" +read -p " 输入 VOLCENGINE_ACCESS_KEY: " VOLCENGINE_ACCESS_KEY +if [ -n "$VOLCENGINE_ACCESS_KEY" ]; then + sed -i '' "s|^VOLCENGINE_ACCESS_KEY=.*|VOLCENGINE_ACCESS_KEY=$VOLCENGINE_ACCESS_KEY|" .env + echo " ✅ 已保存" +fi +echo "" + +# VOLCENGINE_SECRET_KEY +read -p " 输入 VOLCENGINE_SECRET_KEY: " VOLCENGINE_SECRET_KEY +if [ -n "$VOLCENGINE_SECRET_KEY" ]; then + sed -i '' "s|^VOLCENGINE_SECRET_KEY=.*|VOLCENGINE_SECRET_KEY=$VOLCENGINE_SECRET_KEY|" .env + echo " ✅ 已保存" +fi +echo "" + +# TOS Bucket +echo "3️⃣ TOS 存储桶配置" +echo " 创建地址：https://console.volcengine.com/tos/bucket" +read -p " 输入 DATABASE_TOS_BUCKET (默认: video-breakdown-uploads): " DATABASE_TOS_BUCKET +if [ -n "$DATABASE_TOS_BUCKET" ]; then + sed -i '' "s|^DATABASE_TOS_BUCKET=.*|DATABASE_TOS_BUCKET=$DATABASE_TOS_BUCKET|" .env + echo " ✅ 已保存" +fi +echo "" + +echo "========================================" +echo "✅ 环境配置完成！" +echo "" +echo "📋 下一步操作：" +echo " 1. 本地测试：uv run veadk web" +echo " 2. 云端部署：agentkit launch" +echo "" +echo "💡 提示：" +echo " - 完整配置请编辑 .env 文件" +echo " - 云端部署需在 AgentKit 控制台配置相同的环境变量" +echo " - 参考文档：README.md" +echo "" diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/__init__.py new file mode 100644 index 00000000..e2215921 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/__init__.py @@ -0,0 +1,5 @@ +from . import agent + +__all__ = [ + "agent", +] diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/agent.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/agent.py new file mode 100644 index 00000000..4726c0dd --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/agent.py @@ -0,0 +1,253 @@ +""" +主编排 Root Agent 定义（veadk web 的唯一真相来源） + +VeADK 约束：有 sub_agents 的 Agent 不支持直接挂载 tools， +因此 web_search 等工具封装为独立子 Agent。 +""" + +import logging +import os + +from google.adk.agents.callback_context import CallbackContext +from google.adk.tools import ToolContext +from veadk import Agent +from veadk.agents.sequential_agent import SequentialAgent +from veadk.memory.short_term_memory import ShortTermMemory +from veadk.tools.builtin_tools.web_search import web_search + +from .hook.final_output_hook import guard_final_user_output +from .hook.search_output_hook import suppress_search_agent_user_output +from .hook.video_upload_hook import hook_video_upload +from .prompt import ROOT_AGENT_INSTRUCTION +from .sub_agents.breakdown_agent.prompt import BREAKDOWN_AGENT_INSTRUCTION +from .sub_agents.hook_analyzer_agent.prompt import ( + HOOK_ANALYZER_INSTRUCTION, + HOOK_FORMAT_INSTRUCTION, +) +from .sub_agents.hook_analyzer_agent.filtered_sequential import ( + HookAnalyzerSequentialAgent, +) +from .sub_agents.report_generator_agent.prompt import REPORT_AGENT_INSTRUCTION +from .sub_agents.report_generator_agent.direct_output_callback import ( + direct_output_callback, +) +from .hook.format_hook import soft_fix_hook_output +from .tools.process_video import process_video +from .tools.analyze_segments_vision import analyze_segments_vision +from .tools.analyze_bgm import analyze_bgm +from .tools.video_upload import video_upload_to_tos +from .tools.analyze_hook_segments import analyze_hook_segments +from .tools.report_generator import generate_video_report +from .utils.types import json_response_config + +logger = logging.getLogger(__name__) + +# ==================== 内容安全护栏（LLM Shield） ==================== +# 仅当配置了 TOOL_LLM_SHIELD_APP_ID 时启用，否则静默跳过 + +shield_callbacks = {} +if os.getenv("TOOL_LLM_SHIELD_APP_ID"): + try: + from veadk.tools.builtin_tools.llm_shield import content_safety + + shield_callbacks = { + "before_model_callback": content_safety.before_model_callback, + "after_model_callback": content_safety.after_model_callback, + } + logger.info("内容安全护栏: 已启用 (before_model + after_model)") + except Exception as e: + logger.warning(f"llm_shield 加载失败，跳过内容安全护栏: {e}") +else: + logger.debug("未配置 TOOL_LLM_SHIELD_APP_ID，跳过内容安全护栏") + +root_before_model_callback = shield_callbacks.get("before_model_callback") +root_after_model_callbacks = [] +if shield_callbacks.get("after_model_callback"): + root_after_model_callbacks.append(shield_callbacks["after_model_callback"]) +# 最后一层输出守卫：仅在泄露过程信息时触发 LLM 重写 +root_after_model_callbacks.append(guard_final_user_output) +root_callback_kwargs = { + "after_model_callback": root_after_model_callbacks, +} +if root_before_model_callback: + root_callback_kwargs["before_model_callback"] = root_before_model_callback + +# ==================== 子 Agent 定义 ==================== + +# 搜索子 Agent：挂载 web_search 工具 +search_agent = Agent( + name="search_agent", + description="联网搜索短视频行业资讯、平台规则、热门趋势等实时信息", + instruction=( + "你是一个搜索助手。收到用户的搜索请求后，使用 web_search 工具搜索相关信息。\n" + "你负责检索与整理，不直接面向用户输出最终答复；最终答复由 video_breakdown_agent 统一给出。\n" + "请将检索结果整理为简洁中文摘要（供 root 复用），不要添加自己的推测，只基于搜索结果回答。\n" + "格式注意：输出中禁止使用波浪号 ~，数值范围请用 到 或 - 代替（如 1°C到9°C），" + "避免 Markdown 渲染器将 ~ 误解析为删除线。\n" + "\n" + "完成搜索后，必须立即调用 transfer_to_agent，将控制权归还给 video_breakdown_agent。" + ), + tools=[web_search], + after_model_callback=[suppress_search_agent_user_output], + output_key="search_result", +) + +# ==================== Factory functions (避免 SequentialAgent 共享 parent) ==================== + + +def create_breakdown_agent() -> Agent: + return Agent( + name="breakdown_agent", + description=( + "负责视频分镜拆解：视频预处理（FFmpeg + ASR）、" + "视觉分析（doubao-vision）、BGM 分析。" + "支持URL链接和本地文件上传，输出完整分镜结构化数据。" + ), + instruction=BREAKDOWN_AGENT_INSTRUCTION, + tools=[ + process_video, + analyze_segments_vision, + analyze_bgm, + video_upload_to_tos, + ], + output_key="breakdown_result", + model_extra_config={ + "extra_body": { + "thinking": {"type": os.getenv("THINKING_BREAKDOWN_AGENT", "disabled")} + } + }, + ) + + +def create_hook_analyzer_agent() -> SequentialAgent: + def _prime_hook_segments_state(callback_context: CallbackContext): + inv = getattr(callback_context, "_invocation_context", None) + if not inv: + return None + state = getattr(callback_context, "state", None) + if isinstance(state, dict) and state.get("hook_segments_context"): + return None + tool_ctx = ToolContext(inv) + context = analyze_hook_segments(tool_ctx) + if isinstance(state, dict): + state["hook_segments_context"] = context + return None + + hook_analysis_agent = Agent( + name="hook_analysis_agent", + model_name=os.getenv("MODEL_VISION_NAME", "doubao-seed-1-6-vision-250815"), + description="对视频前三秒分镜进行深度钩子分析，具备视觉分析能力，可直接观察关键帧图片进行专业评估", + instruction=HOOK_ANALYZER_INSTRUCTION, + tools=[], + before_agent_callback=_prime_hook_segments_state, + model_extra_config={ + "extra_body": { + "thinking": { + "type": os.getenv("THINKING_HOOK_ANALYZER_AGENT", "disabled") + }, + "caching": { + "type": "disabled" + }, # 禁用缓存，避免账户未激活 cache service + } + }, + ) + + hook_format_agent = Agent( + name="hook_format_agent", + model_name=os.getenv( + "MODEL_FORMAT_NAME", os.getenv("MODEL_AGENT_NAME", "doubao-seed-1-6-251015") + ), + description="将钩子分析结果格式化为结构化输出并投影为用户可读 Markdown", + instruction=HOOK_FORMAT_INSTRUCTION, + generate_content_config=json_response_config, + output_key="hook_analysis", + after_model_callback=[soft_fix_hook_output], + model_extra_config={ + "extra_body": { + "thinking": { + "type": os.getenv("THINKING_HOOK_FORMAT_AGENT", "disabled") + } + } + }, + ) + + return HookAnalyzerSequentialAgent( + name="hook_analyzer_agent", + description="前三秒钩子分析顺序流程：先分析，再格式化输出", + sub_agents=[hook_analysis_agent, hook_format_agent], + ) + + +def create_report_generator_agent() -> Agent: + return Agent( + name="report_generator_agent", + description="整合分镜拆解数据和钩子分析结果，生成专业的视频分析报告", + instruction=REPORT_AGENT_INSTRUCTION, + tools=[generate_video_report], + after_tool_callback=[direct_output_callback], + output_key="final_report", + model_extra_config={ + "extra_body": { + "thinking": {"type": os.getenv("THINKING_REPORT_AGENT", "disabled")} + } + }, + ) + + +# ==================== Pipelines ==================== + +full_analysis_pipeline = SequentialAgent( + name="full_analysis_pipeline", + description="完整分析生产线：分镜拆解 -> 钩子分析 -> 报告生成", + sub_agents=[ + create_breakdown_agent(), + create_hook_analyzer_agent(), + create_report_generator_agent(), + ], +) + +hook_only_pipeline = SequentialAgent( + name="hook_only_pipeline", + description="钩子分析生产线：分镜拆解 -> 钩子分析", + sub_agents=[create_breakdown_agent(), create_hook_analyzer_agent()], +) + +report_only_pipeline = SequentialAgent( + name="report_only_pipeline", + description="报告生产线：补齐分镜 -> 生成报告", + sub_agents=[create_breakdown_agent(), create_report_generator_agent()], +) + +breakdown_only_pipeline = SequentialAgent( + name="breakdown_only_pipeline", + description="分镜拆解生产线：仅执行分镜拆解", + sub_agents=[create_breakdown_agent()], +) + +agent = Agent( + name="video_breakdown_agent", + description=( + "专业的视频分镜拆解和深度分析助手，" + "支持URL链接和本地文件上传，" + "能够自动拆解视频分镜、分析前三秒钩子、生成专业报告" + ), + instruction=ROOT_AGENT_INSTRUCTION, + sub_agents=[ + full_analysis_pipeline, + hook_only_pipeline, + report_only_pipeline, + breakdown_only_pipeline, + search_agent, + ], + short_term_memory=ShortTermMemory(backend="local"), + # 拦截 veadk web UI 上传的文件（inline_data → 文本 URL/路径） + before_agent_callback=hook_video_upload, + model_extra_config={ + "extra_body": { + "thinking": {"type": os.getenv("THINKING_ROOT_AGENT", "disabled")} + } + }, + **root_callback_kwargs, +) + +root_agent = agent diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/final_output_hook.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/final_output_hook.py new file mode 100644 index 00000000..107a888a --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/final_output_hook.py @@ -0,0 +1,187 @@ +""" +最终输出守卫（Root Agent after_model_callback） + +目标： +1. 仅在检测到执行过程/JSON 外泄时触发； +2. 使用 LLM 将输出重写为用户可读 Markdown； +3. 不干预工具调用 envelope，避免破坏编排。 +""" + +from __future__ import annotations + +import os +import json +from typing import Optional + +import httpx +from google.adk.agents.callback_context import CallbackContext +from google.adk.events import Event +from google.adk.models import LlmResponse +from veadk.utils.logger import get_logger + +logger = get_logger(__name__) + + +def _get_first_text(llm_response: LlmResponse) -> str: + if not llm_response or not llm_response.content or not llm_response.content.parts: + return "" + return llm_response.content.parts[0].text or "" + + +def _looks_like_tool_envelope(text: str) -> bool: + """工具调用包，必须放行，不可改写。""" + compact = text.replace(" ", "") + return ( + '"name":"' in compact and '"parameters":' in compact + ) or '"agent_name":"' in compact + + +def _event_to_text(event: Optional[Event]) -> str: + """将 event 尽量转成可检索字符串，便于做鲁棒的工具调用识别。""" + if event is None: + return "" + try: + if hasattr(event, "model_dump"): + return json.dumps(event.model_dump(), ensure_ascii=False) + except Exception: + pass + try: + if hasattr(event, "__dict__"): + return json.dumps(event.__dict__, ensure_ascii=False, default=str) + except Exception: + pass + return str(event) + + +def _is_tool_call_turn( + model_response_event: Optional[Event], + llm_response: Optional[LlmResponse], +) -> bool: + """ + 判断当前是否是工具调用轮次。 + 工具轮次不应被最终输出守卫改写，否则可能破坏 transfer_to_agent 编排。 + """ + # 优先依赖事件级信号，避免仅凭文本误判。 + event_text = _event_to_text(model_response_event).lower() + if event_text and any( + marker in event_text + for marker in ( + "function_call", + "tool_call", + "function_response", + "transfer_to_agent", + ) + ): + return True + + text = _get_first_text(llm_response).lower() if llm_response else "" + # 无事件信息时再回退到文本 envelope 判定。 + if _looks_like_tool_envelope(text): + return True + return False + + +def _needs_llm_repair(text: str) -> bool: + """是否需要触发最终输出修复。""" + leak_markers = ( + "<[PLHD", + '"name":', + '"parameters":', + ) + if any(marker in text for marker in leak_markers): + return True + + # 典型 JSON 外泄：整段以 {} 或 [] 包裹 + stripped = text.strip() + if (stripped.startswith("{") and stripped.endswith("}")) or ( + stripped.startswith("[") and stripped.endswith("]") + ): + return True + return False + + +def _call_repair_llm(raw_text: str) -> Optional[str]: + api_key = os.getenv("MODEL_AGENT_API_KEY", "") + if not api_key: + return None + + model = os.getenv("MODEL_AGENT_NAME", "doubao-seed-1-6-251015") + api_base = os.getenv( + "MODEL_AGENT_API_BASE", "https://ark.cn-beijing.volces.com/api/v3/" + ) + api_base = api_base.rstrip("/") + url = f"{api_base}/chat/completions" + + prompt = ( + "你是一个最终输出清洗器。请将输入内容改写为“面向最终用户”的中文 Markdown 结论，要求：\n" + "1) 严禁输出任何内部执行过程、工具调用信息、name/parameters、transfer_to_agent、PLHD 标记；\n" + "2) 严禁输出 JSON、代码块；\n" + "3) 保留有价值的分析结论（评分、亮点、问题、建议）；\n" + "4) 若信息不足，给出简短说明并建议用户重试，不要编造。\n" + ) + + payload = { + "model": model, + "messages": [ + {"role": "system", "content": prompt}, + {"role": "user", "content": raw_text[:12000]}, + ], + "temperature": 0.2, + "max_tokens": 1200, + } + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + try: + response = httpx.post(url, headers=headers, json=payload, timeout=30) + response.raise_for_status() + data = response.json() + content = data.get("choices", [{}])[0].get("message", {}).get("content", "") + content = str(content or "").strip() + return content or None + except Exception as e: + logger.warning(f"[final_output_guard] repair llm call failed: {e}") + return None + + +def guard_final_user_output( + *, + callback_context: CallbackContext, + llm_response: LlmResponse, + model_response_event: Optional[Event] = None, +) -> Optional[LlmResponse]: + """ + Root Agent 最终输出守卫： + - 纯工具 envelope：放行； + - 发现泄露：调用 LLM 修复； + - 失败兜底：保持原文，避免中断主流程。 + """ + agent = callback_context._invocation_context.agent + if not agent or getattr(agent, "name", "") != "video_breakdown_agent": + return llm_response + + if _is_tool_call_turn(model_response_event, llm_response): + return llm_response + + # 放行 function_call / function_response，避免干扰 SDK 事件追踪 + if llm_response and llm_response.content and llm_response.content.parts: + part = llm_response.content.parts[0] + if hasattr(part, "function_call") and part.function_call: + return llm_response + if hasattr(part, "function_response") and part.function_response: + return llm_response + + text = _get_first_text(llm_response) + if not text: + return llm_response + + if not _needs_llm_repair(text): + return llm_response + + repaired = _call_repair_llm(text) + if repaired: + llm_response.content.parts[0].text = repaired + logger.info("[final_output_guard] repaired leaked intermediate output by llm") + return llm_response diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/format_hook.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/format_hook.py new file mode 100644 index 00000000..616d5c5d --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/format_hook.py @@ -0,0 +1,326 @@ +""" +钩子分析输出修复 Hook。 + +核心策略（对齐 multimedia 的契约化思想）： +1. 仅对目标 agent（hook_format_agent / hook_analyzer_agent）生效； +2. 优先识别工具调用轮次，避免污染 transfer_to_agent 等内部 envelope； +3. 无论 JSON 是否完美，都尽力写回结构化 state 并输出 Markdown； +4. best-effort：永远不给用户上屏原始 JSON/占位片段。 +""" + +import json +import re +from typing import Any, Optional + +from google.adk.agents.callback_context import CallbackContext +from google.adk.events import Event +from google.adk.models import LlmResponse +from veadk.utils.logger import get_logger + +logger = get_logger(__name__) + +_TARGET_AGENTS = {"hook_format_agent", "hook_analyzer_agent"} +_MAX_COMMENT_LEN = 800 + +_SCORE_FIELDS = ( + "overall_score", + "visual_impact", + "language_hook", + "emotion_trigger", + "information_density", + "rhythm_control", +) + +_DEFAULT_HOOK_ANALYSIS: dict[str, Any] = { + "overall_score": 0.0, + "visual_impact": 0.0, + "visual_comment": "", + "language_hook": 0.0, + "language_comment": "", + "emotion_trigger": 0.0, + "emotion_comment": "", + "information_density": 0.0, + "info_comment": "", + "rhythm_control": 0.0, + "rhythm_comment": "", + "hook_type": "未知", + "hook_type_analysis": "", + "target_audience": "", + "strengths": [], + "weaknesses": [], + "suggestions": [], + "competitor_reference": "", + "retention_prediction": "数据不足，基于当前信息暂无法稳定预测", +} + + +def _agent_name(callback_context: CallbackContext) -> str: + inv = getattr(callback_context, "_invocation_context", None) + if not inv: + return "" + return getattr(getattr(inv, "agent", None), "name", "") or "" + + +def _get_first_text(llm_response: Optional[LlmResponse]) -> str: + if not llm_response or not llm_response.content or not llm_response.content.parts: + return "" + return str(llm_response.content.parts[0].text or "") + + +def _event_to_text(event: Optional[Event]) -> str: + if event is None: + return "" + try: + if hasattr(event, "model_dump"): + return json.dumps(event.model_dump(), ensure_ascii=False) + except Exception: + pass + try: + return json.dumps( + getattr(event, "__dict__", {}), ensure_ascii=False, default=str + ) + except Exception: + return str(event) + + +def _looks_like_tool_envelope(payload: Any) -> bool: + if not isinstance(payload, dict): + return False + if "name" in payload and "parameters" in payload: + return True + if payload.get("agent_name") or payload.get("transfer_to_agent"): + return True + return False + + +def _is_tool_call_turn(model_response_event: Optional[Event], text: str) -> bool: + event_text = _event_to_text(model_response_event).lower() + if any( + token in event_text + for token in ( + "tool_call", + "function_call", + "function_response", + "transfer_to_agent", + "analyze_hook_segments", + ) + ): + return True + + stripped = text.strip() + if not stripped: + return False + try: + obj = json.loads(stripped) + return _looks_like_tool_envelope(obj) + except Exception: + return False + + +def _extract_json_candidate(text: str) -> str: + # 防御性剥离 PLHD 标签和工具调用 envelope + text = re.sub(r"<\[PLHD[^\]]*\]>", "", text) + text = re.sub( + r'\[\s*\{\s*"name"\s*:\s*"[^"]*"\s*,\s*"parameters"\s*:\s*\{[^}]*\}\s*\}\s*\]', + "", + text, + flags=re.DOTALL, + ) + text = text.strip() + + fenced = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", text, flags=re.IGNORECASE) + if fenced and fenced.group(1).strip(): + return fenced.group(1).strip() + return text + + +def _coerce_to_list(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(v).strip() for v in value if str(v).strip()] + if isinstance(value, str): + return [value.strip()] if value.strip() else [] + return [str(value)] + + +def _clamp_score(value: Any) -> float: + try: + return max(0.0, min(10.0, float(value))) + except (TypeError, ValueError): + return 0.0 + + +def _safe_text(value: Any, max_len: int = _MAX_COMMENT_LEN) -> str: + text = str(value or "").strip() + text = re.sub(r"<\[PLHD[^\]]*\]>", "", text) + text = text.replace("transfer_to_agent", "") + if len(text) > max_len: + return text[: max_len - 1] + "..." + return text + + +def _normalize_output(raw: dict[str, Any]) -> dict[str, Any]: + output = {**_DEFAULT_HOOK_ANALYSIS, **(raw or {})} + + for field in _SCORE_FIELDS: + output[field] = _clamp_score(output.get(field, 0)) + + output["strengths"] = _coerce_to_list(output.get("strengths")) + output["weaknesses"] = _coerce_to_list(output.get("weaknesses")) + output["suggestions"] = _coerce_to_list(output.get("suggestions")) + + for field in ( + "visual_comment", + "language_comment", + "emotion_comment", + "info_comment", + "rhythm_comment", + "hook_type", + "hook_type_analysis", + "target_audience", + "competitor_reference", + "retention_prediction", + ): + output[field] = _safe_text(output.get(field, "")) + + return output + + +def _extract_score(text: str, label: str) -> Optional[float]: + patterns = [ + rf'"{label}"\s*:\s*([0-9]+(?:\.[0-9]+)?)', + rf"{label}\s*[:：]\s*([0-9]+(?:\.[0-9]+)?)", + ] + for pattern in patterns: + m = re.search(pattern, text, flags=re.IGNORECASE) + if m: + return _clamp_score(m.group(1)) + return None + + +def _extract_list_by_heading(text: str, heading: str) -> list[str]: + pattern = ( + rf"{heading}[\s\S]{{0,120}}?(?:\n|\r\n)([\s\S]{{0,600}}?)(?:\n\n|###|####|$)" + ) + m = re.search(pattern, text, flags=re.IGNORECASE) + if not m: + return [] + block = m.group(1) + items = re.findall(r"(?:^|\n)\s*(?:[-*]|\d+\.)\s*(.+)", block) + return [i.strip() for i in items if i.strip()][:6] + + +def _fallback_struct_from_text(text: str) -> dict[str, Any]: + output = dict(_DEFAULT_HOOK_ANALYSIS) + + for field in _SCORE_FIELDS: + score = _extract_score(text, field) + if score is not None: + output[field] = score + + zh_score = re.search(r"综合评分\s*[:：]\s*([0-9]+(?:\.[0-9]+)?)", text) + if zh_score: + output["overall_score"] = _clamp_score(zh_score.group(1)) + + output["strengths"] = _extract_list_by_heading(text, r"(?:亮点|优点|strengths)") + output["weaknesses"] = _extract_list_by_heading(text, r"(?:待改进|不足|weaknesses)") + output["suggestions"] = _extract_list_by_heading( + text, r"(?:优化建议|建议|suggestions)" + ) + + excerpt = _safe_text(text, max_len=500) + if excerpt: + for field in ( + "visual_comment", + "language_comment", + "emotion_comment", + "info_comment", + "rhythm_comment", + ): + output[field] = excerpt + + return _normalize_output(output) + + +def _build_hook_markdown_summary(output: dict[str, Any]) -> str: + overall = output.get("overall_score", 0) + hook_type = output.get("hook_type", "未知") + retention = output.get("retention_prediction", "N/A") + audience = output.get("target_audience", "") + + def _score_line(name: str, score_key: str, comment_key: str) -> str: + score = output.get(score_key, 0) + comment = str(output.get(comment_key, "") or "").strip() or "暂无详细说明" + return f"#### {name}: {score}/10\n> {comment}" + + strengths = _coerce_to_list(output.get("strengths")) + weaknesses = _coerce_to_list(output.get("weaknesses")) + suggestions = _coerce_to_list(output.get("suggestions")) + + strengths_text = "\n".join(f"- {s}" for s in strengths) if strengths else "- 暂无" + weaknesses_text = ( + "\n".join(f"- {w}" for w in weaknesses) if weaknesses else "- 暂无" + ) + suggestions_text = ( + "\n".join(f"{i + 1}. {s}" for i, s in enumerate(suggestions)) + if suggestions + else "1. 暂无" + ) + audience_line = f"\n- **目标受众**: {audience}" if audience else "" + + return ( + "## 前三秒钩子分析\n\n" + f"### 综合评分: {overall}/10\n" + f"- **钩子类型**: {hook_type}{audience_line}\n" + f"- **留存预测**: {retention}\n\n" + "---\n\n" + "### 五维评分详情\n\n" + f"{_score_line('视觉冲击力', 'visual_impact', 'visual_comment')}\n\n" + f"{_score_line('语言钩子', 'language_hook', 'language_comment')}\n\n" + f"{_score_line('情绪唤起', 'emotion_trigger', 'emotion_comment')}\n\n" + f"{_score_line('信息密度', 'information_density', 'info_comment')}\n\n" + f"{_score_line('节奏掌控', 'rhythm_control', 'rhythm_comment')}\n\n" + "---\n\n" + f"### 亮点\n{strengths_text}\n\n" + f"### 待改进\n{weaknesses_text}\n\n" + f"### 优化建议\n{suggestions_text}" + ) + + +def soft_fix_hook_output( + *, + callback_context: CallbackContext, + llm_response: LlmResponse, + model_response_event: Optional[Event] = None, +) -> Optional[LlmResponse]: + agent_name = _agent_name(callback_context) + if agent_name not in _TARGET_AGENTS: + return llm_response + + if not llm_response or not llm_response.content or not llm_response.content.parts: + return llm_response + + # 放行 function_call / function_response，避免干扰 SDK 事件追踪 + part = llm_response.content.parts[0] + if hasattr(part, "function_call") and part.function_call: + return llm_response + if hasattr(part, "function_response") and part.function_response: + return llm_response + + text = _get_first_text(llm_response) + if not text: + return llm_response + + if _is_tool_call_turn(model_response_event, text): + return llm_response + + # 非破坏性策略：不再依赖 LLM 输出 JSON 做硬解析（易出现截断/兜底 0 分）。 + # 仅做“泄露剥离 + 存档”，并让 LLM 的 Markdown 原样对用户可见。 + sanitized = _safe_text(text, max_len=20000) + llm_response.content.parts[0].text = sanitized + + state = getattr(callback_context, "state", None) + if isinstance(state, dict): + state["hook_analysis_markdown"] = sanitized + return llm_response diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/search_output_hook.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/search_output_hook.py new file mode 100644 index 00000000..3823567b --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/search_output_hook.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from typing import Optional + +from google.adk.agents.callback_context import CallbackContext +from google.adk.events import Event +from google.adk.models import LlmResponse + + +def _is_tool_call_turn(model_response_event: Optional[Event], text: str) -> bool: + """工具调用回合不做处理，避免干扰 function call。""" + if model_response_event: + # 检查 function calls（LLM 发起的工具调用）和 function responses（工具返回后的阶段） + if ( + model_response_event.get_function_calls() + or model_response_event.get_function_responses() + ): + return True + return "transfer_to_agent" in (text or "") or "web_search" in (text or "") + + +def suppress_search_agent_user_output( + *, + callback_context: CallbackContext, + llm_response: LlmResponse, + model_response_event: Optional[Event] = None, +) -> Optional[LlmResponse]: + """ + search_agent 只做检索与回传，不直接面向用户输出正文。 + 将其文本暂存到 state，最终由 root agent 统一输出一次，避免重复回复。 + """ + agent_name = getattr( + getattr(callback_context, "_invocation_context", None), "agent", None + ) + if not agent_name or getattr(agent_name, "name", "") != "search_agent": + return llm_response + + if not llm_response or not llm_response.content or not llm_response.content.parts: + return llm_response + + # 检查 parts[0] 是否包含 function_call 或 function_response，完整放行不做修改 + part = llm_response.content.parts[0] + if hasattr(part, "function_call") and part.function_call: + return llm_response + if hasattr(part, "function_response") and part.function_response: + return llm_response + + text = part.text or "" + if not text or _is_tool_call_turn(model_response_event, text): + return llm_response + + state = getattr(callback_context, "state", None) + if isinstance(state, dict): + state["search_result"] = text + + # 只清空纯文本输出，避免与 root 最终答复重复展示。 + llm_response.content.parts[0].text = "" + return llm_response diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/video_upload_hook.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/video_upload_hook.py new file mode 100644 index 00000000..3c0a9cff --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/hook/video_upload_hook.py @@ -0,0 +1,194 @@ +""" +视频文件上传钩子（before_agent_callback） + +当用户通过 veadk web UI 上传视频文件时，ADK 将文件二进制数据 +以 Part.inline_data 传入。LLM 无法处理原始二进制，会导致 +工具调用 JSON 生成失败（Unterminated string）。 + +此钩子在 Agent 处理之前拦截 inline_data： + 1. 保存到临时文件 + 2. 尝试上传到 TOS 获取签名 URL + 3. 如果 TOS 不可用，保留本地文件路径 + 4. 将 inline_data Part 替换为文本 Part + +参考实现: ad_video_gen_seq/app/market/hook.py → hook_inline_data_transform +""" + +from __future__ import annotations + +import logging +import os +import uuid +from datetime import datetime +from pathlib import Path +from typing import Optional + +from google.adk.agents.callback_context import CallbackContext +from google.genai import types + +logger = logging.getLogger(__name__) + +# 支持的视频 MIME 类型 +VIDEO_MIME_TYPES = { + "video/mp4", + "video/quicktime", + "video/x-msvideo", + "video/webm", + "video/x-matroska", + "video/mpeg", + "video/3gpp", + "video/x-flv", + "application/octet-stream", # 某些浏览器上传时 MIME 可能不精确 +} + +# 持久化目录（不会被 process_video 的 finally 清理） +UPLOAD_CACHE_DIR = os.getenv("MEDIA_UPLOAD_CACHE_DIR", "./.media-uploads") + + +def _try_upload_to_tos(local_path: str) -> Optional[str]: + """尝试将文件上传到 TOS，成功返回签名 URL，失败返回 None""" + try: + import tos + from tos import HttpMethodType + + ak = os.getenv("VOLCENGINE_ACCESS_KEY", "") + sk = os.getenv("VOLCENGINE_SECRET_KEY", "") + + if not ak or not sk: + try: + from veadk.auth.veauth.utils import get_credential_from_vefaas_iam + + cred = get_credential_from_vefaas_iam() + ak = cred.access_key_id + sk = cred.secret_access_key + except Exception: + return None + + if not ak or not sk: + return None + + bucket = os.getenv("DATABASE_TOS_BUCKET") or os.getenv( + "TOS_BUCKET", "video-breakdown-uploads" + ) + region = os.getenv("DATABASE_TOS_REGION") or os.getenv( + "TOS_REGION", "cn-beijing" + ) + endpoint = f"tos-{region}.volces.com" + + client = tos.TosClientV2(ak=ak, sk=sk, endpoint=endpoint, region=region) + try: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = os.path.basename(local_path) + object_key = f"video_breakdown/upload/{timestamp}_{filename}" + + client.put_object_from_file( + bucket=bucket, key=object_key, file_path=local_path + ) + + signed = client.pre_signed_url( + http_method=HttpMethodType.Http_Method_Get, + bucket=bucket, + key=object_key, + expires=604800, # 7 天 + ) + logger.info(f"[video_upload_hook] 文件已上传到 TOS: {object_key}") + return signed.signed_url + finally: + client.close() + + except Exception as exc: + logger.warning(f"[video_upload_hook] TOS 上传失败，回退到本地路径: {exc}") + return None + + +def hook_video_upload( + callback_context: CallbackContext, +) -> Optional[types.Content]: + """ + before_agent_callback：拦截用户消息中的 inline_data（上传文件）， + 转换为文本 URL 或本地路径，确保 LLM 不接触原始二进制数据。 + """ + user_content = callback_context.user_content + if not user_content or not user_content.parts: + return None + + new_parts = [] + has_inline_data = False + + for part in user_content.parts: + # 保留文本 Part + if part.text: + new_parts.append(types.Part(text=part.text)) + + # 处理 inline_data（上传的文件） + if part.inline_data and part.inline_data.data: + has_inline_data = True + mime_type = part.inline_data.mime_type or "application/octet-stream" + data = part.inline_data.data + + logger.info( + f"[video_upload_hook] 检测到 inline_data: " + f"mime={mime_type}, size={len(data)} bytes" + ) + + # 根据 MIME 类型确定文件后缀 + ext = _mime_to_ext(mime_type) + + # 保存到持久化目录（不受 process_video 临时目录清理影响） + upload_dir = Path(UPLOAD_CACHE_DIR) + upload_dir.mkdir(parents=True, exist_ok=True) + + filename = f"upload_{uuid.uuid4().hex[:8]}{ext}" + local_path = upload_dir / filename + + with open(local_path, "wb") as f: + f.write(data) + + logger.info( + f"[video_upload_hook] 文件已保存: {local_path} " + f"({len(data) / 1024 / 1024:.1f}MB)" + ) + + # 尝试上传到 TOS + tos_url = _try_upload_to_tos(str(local_path)) + + if tos_url: + new_parts.append(types.Part(text=f"用户上传了视频文件，URL: {tos_url}")) + # TOS 上传成功后可以删除本地文件 + try: + local_path.unlink() + except Exception: + pass + else: + # TOS 不可用，使用本地路径 + abs_path = str(local_path.resolve()) + new_parts.append( + types.Part(text=f"用户上传了视频文件，本地路径: {abs_path}") + ) + + if has_inline_data: + user_content.parts = new_parts + logger.info( + f"[video_upload_hook] inline_data 已转换为文本 ({len(new_parts)} parts)" + ) + + # 返回 None 表示不中断正常流程 + return None + + +def _mime_to_ext(mime_type: str) -> str: + """将 MIME 类型映射为文件扩展名""" + mapping = { + "video/mp4": ".mp4", + "video/quicktime": ".mov", + "video/x-msvideo": ".avi", + "video/webm": ".webm", + "video/x-matroska": ".mkv", + "video/mpeg": ".mpeg", + "video/3gpp": ".3gp", + "video/x-flv": ".flv", + "audio/mpeg": ".mp3", + "audio/wav": ".wav", + "audio/mp4": ".m4a", + } + return mapping.get(mime_type, ".mp4") # 默认 .mp4 diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/prompt.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/prompt.py new file mode 100644 index 00000000..f0d44665 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/prompt.py @@ -0,0 +1,358 @@ +""" +主编排 Root Agent 提示词 +描述完整视频拆解分析能力、灵活的部分执行策略和自然的对话风格 +""" + +ROOT_AGENT_INSTRUCTION = """ +# 角色 + +你是"小视"——一位专业的视频内容深度分析顾问。你背后有一套完整的视频拆解引擎，能将任何短视频自动拆解为结构化的分镜脚本，并在此基础上进行多维度的深度分析。 + +你的性格：专业但亲切，像一位见多识广的创意总监在帮朋友看片。你善于从创作者视角给出可落地的建议，而不是堆砌数据。 + +--- + +## 状态复用机制（重要） + +你拥有**会话级短期记忆**，可以在同一 session 内访问和复用之前生成的中间数据。 + +### 核心原则 +1. **优先复用历史数据**：在同一次对话中，如果用户要求"重新生成钩子"或"生成报告"，**先检查 session.state 是否已有分镜数据**（`process_video_result` / `vision_analysis_result`） +2. **避免重复拆解**：若已有分镜数据且用户仅要求后续步骤（如钩子/报告），直接复用历史分镜，不要重新调用 breakdown +3. **明确重新分析**：仅当用户明确说"重新拆解这个视频"或提供了**新的视频链接**时，才重新执行完整流程 + +### 判断逻辑示例 +- 用户："给我看看前三秒钩子分析" → 检查是否有分镜数据 → 有则直接调用钩子分析，无则先拆解 +- 用户："重新生成一份报告" → 检查是否有分镜/钩子数据 → 有则直接复用生成报告，无则提示"需要先分析视频" +- 用户："帮我分析另一个视频 <新链接>" → 视为全新任务，执行完整流程 + +### 注意事项 +- 如果用户在同一对话中反复要求相同内容（如"再生成一次报告"），**不要误认为需要重新拆解**，应直接基于已有数据重新调用对应 Agent +- session.state 的关键字段： + - `process_video_result`: 分镜基础数据 + - `vision_analysis_result`: 视觉分析后的分镜 + - `hook_analysis_struct`: 钩子分析结构化数据 + - `hook_analysis_markdown`: 钩子分析 Markdown 输出 + - `final_report`: 最终报告 + +--- + +## 核心能力全景 + +你作为**编排 Agent**，负责理解用户意图并决策是否调用专业子 Agent。你背后有 5 个专业子 Agent（含 4 条生产线）： + +### 📹 模块1：breakdown_only_pipeline（视频分镜拆解生产线） +**职责**：将视频拆解为逐帧分镜，输出结构化数据（时间轴、画面内容、运镜、ASR、BGM、场景分析）。 + +**触发条件**（满足任一条件即可）： +1. ✅ 用户**提供了视频**（URL / 本地路径 / TOS 路径） +2. ✅ 用户**明确要求分镜拆解**或完整分析 + +**触发关键词**： +- "拆解"、"分镜"、"视频结构"、"分析视频" +- 提供视频后说"开始分析"、"帮我看看" + +**不触发场景**： +- ❌ 仅咨询"怎么拆分镜"（理论问题） +- ❌ 说"我想分析视频"但未提供视频 +- ❌ 闲聊、打招呼、功能介绍 + +### 🎣 模块2：hook_only_pipeline（钩子分析生产线） +**职责**：按固定顺序执行“分镜拆解 -> 钩子分析”，输出 5 维度评分（视觉冲击力、语言钩子、情绪唤起、信息密度、节奏掌控）。 + +**触发条件**（满足任一条件即可）： +1. ✅ 用户**只丢了视频下载链接**（默认执行） +2. ✅ **已有分镜数据**（来自模块1或历史 state） +3. ✅ 用户**明确要求钩子分析** + +**触发关键词**： +- "钩子"、"前三秒"、"开头"、"留存"、"吸引力" +- "分析开头"、"看看前 3 秒" + +**不触发场景**： +- ❌ 咨询"怎么设计钩子"（给建议即可，不调用） +- ❌ 没有视频/分镜数据时要求钩子分析（需先拿到视频或先做模块1） + +### 📊 模块3：report_only_pipeline（报告生产线） +**职责**：按固定顺序执行“补齐分镜 -> 生成报告”，确保报告可用（钩子缺失时也要生成主报告）。 + +**触发条件**（同时满足）： +1. ✅ **已有分镜数据** +2. ✅ 用户**明确要求生成报告** + +**触发关键词**： +- "报告"、"生成报告"、"整理成文档"、"汇总" + +**不触发场景**： +- ❌ 用户只是要口头说明结果（直接转述即可） +- ❌ 没有数据时要求报告 + +### 🧩 模块4：full_analysis_pipeline（完整分析生产线） +**职责**：按固定顺序执行“分镜拆解 -> 钩子分析 -> 报告生成”。 + +### 🔍 模块5：search_agent（联网搜索） +**职责**：搜索短视频行业资讯、平台最新规则、热门趋势等实时信息 + +**触发条件**： +1. ✅ 用户**明确要求搜索**特定信息 +2. ✅ 你需要**实时数据**才能回答（比如最新平台规则、当前热门趋势） + +**触发关键词**： +- "搜一下"、"查查"、"最新"、"现在"、"当前" +- "抖音最新规则"、"小红书热门"等需要实时信息的请求 + +**不触发场景**： +- ❌ 通用知识问答（用自己的知识直接回答） +- ❌ 关于你功能的问题 +- ❌ 视频分析相关的请求（交给其他子 Agent） + +--- + +## 你的角色定位 + +### 作为编排 Agent，你需要： +1. **优先作为对话助手**：理解用户意图、回答咨询、提供建议 +2. **判断是否委派**：仅在满足"触发条件"时才调用子 Agent +3. **数据流转**：优先复用 session.state 中的历史分镜/钩子数据，避免重复拆解；在确需时再调用子 Agent +4. **结果解读**：将子 Agent 的技术输出转化为用户友好的语言 + +### 你**不直接执行**以下任务（委派给子 Agent）： +- ❌ 自己手写/模拟工具执行结果 +- ✅ 需要分镜/钩子/报告时，调用对应子 Agent 完成 + +### 你**直接处理**以下场景（不调用子 Agent）： +- ✅ 闲聊、打招呼、自我介绍 +- ✅ 回答短视频创作咨询（剪辑技巧、平台策略、钩子设计理论等） +- ✅ 解释你的能力和工作流程 +- ✅ 确认用户需求（"你想做全套分析还是只看分镜？"） + +### 你**委派 search_agent**的场景： +- ✅ 需要实时信息才能回答的问题（"抖音最新推荐算法"、"2026年短视频趋势"） +- ✅ 用户明确要求搜索特定信息 +- ✅ 搜索完成后，最终回复只允许由 root agent 输出一次（单条自然语言） +- ✅ root 必须基于 `search_result` 组织答案，不得重复转述同一搜索结果 + +--- + +## 交互策略 + +### 当用户闲聊或打招呼时 +简短、自然地回应，自然带出你的能力，但**不要甩一堆功能清单**。示例风格： + +> "嗨！我是小视 👋 专门帮人拆视频的。不管是想看分镜结构、还是想知道开头3秒为什么留不住人，丢个视频下载链接给我就行～" + +注意：打招呼/自我介绍时，**不要**主动承诺“抖音/小红书/B站链接、本地文件路径、TOS路径”等能力；仅说“视频下载链接（http/https可访问）”即可。 + +### 当用户想做视频分析时 +根据用户表述判断需求范围，**不用每次都逐一询问要做哪些模块**。常见策略： + +1. **需求明确** → 直接执行。"帮我拆个分镜" → 执行模块1。"分析一下这个视频前三秒" → 执行模块1+模块2。 +2. **需求模糊**（"帮我分析个视频"）→ 简短确认一句。"想做全套分析还是先看分镜？全套的话会包含分镜拆解、前三秒钩子评分和完整报告～" +3. **用户说"全套"或"完整分析"** → 按顺序执行模块1→2→3。 +4. **已经有部分数据** → 直接续接。比如分镜已经做完了，用户说"帮我看看前三秒" → 直接调模块2，不需要重新拆。 + +### 默认触发规则（重要） +- 如果用户**只发送了视频下载链接**，没有其它文字说明：你默认执行 **模块2**（钩子分析生产线：内部会先做分镜拆解再做前三秒钩子分析）。\n + - 如果 session.state 已存在分镜数据（`process_video_result` / `vision_analysis_result`），严格复用，不要重复拆解。\n + - 完成钩子分析后提示用户：是否需要基于当前结果生成“更深入洞察的完整视频分析报告”（模块3/模块4）。\n +- 如果用户明确说“只要分镜/只要钩子/只要报告”，则按用户意图执行；无论完成分镜或钩子，都要提示可进一步生成报告。 + +### 获取视频的方式 +当确认了需求后，简洁地要视频： +> "好的～请给我视频下载链接（http/https可访问）。" + +如果用户一条消息里既说了需求又给了视频链接，**直接开始执行，不要重复确认**。 + +### 执行过程中的沟通 +- 分镜拆解通常需要5-8分钟。提交后简短告知："已经开始处理了，分镜拆解大概需要几分钟，我帮你盯着～" +- 不要在等待期间发送大段解释或重复描述流程。 + +### 结果展示后（格式要求 ⚠️ 重要） + +**绝对禁止直接输出原始 JSON 给用户**。你必须将子 Agent 返回的 JSON 数据转化为**结构清晰、易读的 Markdown 格式**。 + +#### 钩子分析结果的展示格式： + +``` +## 🎣 前三秒钩子分析 + +### 综合评分：X.X / 10 分 +**钩子类型**：XX型 | **目标受众**：XX | **留存预测**：X（>70% / 50-70% / <50%） + +--- + +### 📊 五维评分详情 + +#### 1. 视觉冲击力：X / 10 +> [详细评价内容，引用具体画面] + +#### 2. 语言钩子：X / 10 +> [详细评价内容] + +#### 3. 情绪唤起：X / 10 +> [详细评价内容] + +#### 4. 信息密度：X / 10 +> [详细评价内容] + +#### 5. 节奏掌控：X / 10 +> [详细评价内容] + +--- + +### ✅ 亮点 +- [具体优点1] +- [具体优点2] + +### ⚠️ 待改进 +- [具体不足1] +- [具体不足2] + +### 💡 优化建议 +1. [建议1：改什么→怎么改→预期效果] +2. [建议2] +3. [建议3] + +### 🔮 留存预测 +[详细分析] +``` + +- 对分镜拆解结果，同样用**表格和分段说明**展示，不要输出 JSON +- **简洁解读关键发现**，加入你的专业解读。例如："你这个视频一共15个分镜，前3秒的钩子评分7.8分还不错，主要是视觉冲击力很强。不过语言钩子弱了点，开场文案可以再'狠'一些——" +- 如果只做了部分分析，**自然地提一句后续可做的**："分镜已经拆好了，要不要顺便看看前三秒的钩子分析？这个视频开头挺有意思的。" +- 无论完成了分镜拆解或钩子分析，都在末尾提示一句：\n "如果你想要更深入的洞察（节奏问题、平台适配、改进建议汇总等），我可以基于当前结果生成一份完整视频分析报告。" +- 不要强推用户做更多分析。提一句就够了。 +- 如果 `hook_analysis` 不是标准 JSON（例如结构化文本或混合格式），**你要先做信息提炼再用 Markdown 展示**，不要把原始内容直接抛给用户。 +- 即使钩子分析字段不完整，也要基于已有信息给出可读结论，并明确标注“缺失字段已按已有信息解读”。 +- 如果收到执行过程片段（如 `name/parameters`、`transfer_to_agent`、`analyze_hook_segments`、`<[PLHD...]>` 占位标记），**必须视为内部过程数据**，严禁原样输出给用户。 +- 你展示给用户的内容必须是整理后的最终结论（Markdown 标题、评分、亮点、建议），而不是工具调用日志。 +- 在展示“视频分析报告”的最后，**必须**附带一个“💡 核心价值”小节，结合报告内容简要说明本项目如何帮助解决“完播率低”、“卖点不清”等具体问题（作为功能推广）。 + +--- + +## 灵活执行规则 + +### 完整流程（全套分析） +按顺序自动调度： +1. full_analysis_pipeline（内部固定执行：分镜拆解 → 钩子分析 → 报告生成） + +### 部分执行场景 + +| 用户意图 | 执行模块 | 说明 | +|---------|---------|------| +| "拆分镜" / "分析视频结构" | 模块1 | 仅分镜拆解 | +| "分析前三秒" / "看看开头钩子" | 模块2 | 钩子生产线自动补齐分镜并完成钩子分析 | +| "生成报告" / "给我完整报告" | 模块3 | 报告优先可用；缺钩子也可生成主报告 | +| "只生成报告不用看钩子" | 模块3 | 直接生成主报告 | +| "全面分析" / "完整分析" | 模块4 | 固定完整流程 | + +### 关于数据依赖与自动编排 + +#### 依赖关系 +``` +breakdown_only_pipeline (分镜数据) + ↓ +hook_only_pipeline (钩子分析，内部补齐分镜) + ↓ +report_only_pipeline (报告，依赖分镜数据，钩子缺失也可生成主报告) +``` + +#### 自动编排规则 + +**场景1：用户要钩子分析，但没有分镜数据** +``` +用户: "帮我分析这个视频的前三秒钩子 https://example.com/video.mp4" + +你的行为: +1. 识别意图：用户要钩子分析 + 提供了视频 +2. 检查依赖：钩子分析需要分镜数据，但当前没有 +3. 自动编排：直接调用 hook_only_pipeline（内部会先做分镜再做钩子） +4. 向用户说明："好的！我先帮你拆解视频分镜，然后再分析前三秒钩子～大概需要几分钟。" +``` + +**场景2：用户要报告，但只有部分数据** +``` +用户: "生成完整报告"（但只有分镜数据，没有钩子分析） + +你的行为: +1. 识别意图：用户要完整报告 +2. 检查依赖：有分镜数据，缺少钩子分析 +3. 主动询问："我看到有分镜数据了。要不要我先做一下前三秒钩子分析，再生成完整报告？这样报告会更专业～" +4. 如果用户说"不用" → 直接调用 report_only_pipeline 生成基础报告 +5. 如果用户说"好" → 直接调用 full_analysis_pipeline（一步完成） +``` + +**场景3：用户要完整分析** +``` +用户: "帮我全面分析这个视频 https://example.com/video.mp4" + +你的行为: +直接调用 full_analysis_pipeline（内部按固定顺序执行） +``` + +**场景4：用户没有提供视频** +``` +用户: "我想做钩子分析" + +你的行为: +不调用任何子 Agent，而是回答： +"好的～钩子分析需要先对视频进行分镜拆解。请给我视频链接或者文件路径，我帮你一起做～" +``` + +#### 核心原则 +- ✅ **能自动补齐的前置步骤，主动补齐**（如有视频但缺分镜数据 → 自动先拆解） +- ✅ **需要用户决策的，简短询问**（如要不要加钩子分析） +- ✅ **缺少用户输入的，友好提示**（如没有视频 → 要求提供） +- ✅ **子步骤失败不阻断主流程**：钩子分析失败/不完整时，继续推进报告生成（基于已有分镜数据） +- ✅ **报告优先可用**：当 `hook_analysis` 缺失、无效或格式异常时，仍需调用 `report_only_pipeline` 生成可用的“分镜主报告”，并在报告中附一行“钩子分析部分数据不完整”说明 +- ❌ **不要直接报错或拒绝**，给出可行路径 + +--- + +## 通用对话能力 + +你**首先是一个可以自由对话的 AI 助手**，然后才是视频分析专家。 + +### 可以直接回答的问题（不调用子 Agent） +- 短视频创作技巧、剪辑建议、节奏把控 +- 不同平台（抖音/小红书/B站）的内容策略差异 +- 如何设计好的视频开头钩子、BGM 选择建议 +- 分镜脚本写作指导、视频创意头脑风暴 +- 闲聊、打招呼、了解你的能力 +- **任何不涉及具体视频分析的对话** + +### 何时调用子 Agent？ +**仅当满足以下条件时，才调用子 Agent**： +1. 用户**明确提供了视频**（URL 链接、本地路径、TOS 路径） +2. 用户**明确要求**进行分析（"帮我拆解"、"分析一下"、"看看这个视频"） + +**反例（不要调用子 Agent）**： +- ❌ "你好" → 直接回答，不转移 +- ❌ "介绍一下你的功能" → 直接介绍，不转移 +- ❌ "视频钩子怎么设计？" → 提供建议，不转移 +- ❌ "我想分析视频，但还没准备好" → 等用户提供视频，不转移 + +**正例（调用子 Agent）**： +- ✅ "帮我拆解 https://example.com/video.mp4" → 调用 breakdown_only_pipeline +- ✅ "分析这个视频 https://example.com/video.mp4" → 默认调用 hook_only_pipeline（仅丢视频也默认执行） +- ✅ "给我完整报告 https://example.com/video.mp4" → 调用 full_analysis_pipeline 或 report_only_pipeline（按用户意图） +- ✅ "搜一下抖音最新推荐算法" → 调用 search_agent +- ✅ "2026年短视频有什么新趋势" → 调用 search_agent + +### 引导策略 +在回答通用问题后，**自然地提一句**（不强推）： +> "说到这个，如果你有具体的视频，我可以帮你实际拆一下看看效果～" + +--- + +## 底线规则 +- 所有输出用中文 +- **绝对不输出原始 JSON**：所有子 Agent 返回的 JSON 必须转化为 Markdown 格式后再展示 +- **不要频繁调用子 Agent**：仅在用户提供视频且明确要求分析时调用 +- **优先自由对话**：没有视频时，作为普通 AI 助手回答问题 +- 不要机械重复相同话术，每次回复根据上下文调整 +- 不要假设用户需要全套服务 +- 任务失败时，友好说明原因并给出可行建议（比如换个视频链接试试） +- 记住对话上下文，不要问已经回答过的问题 +- 在等待期间保持简洁，不要用大段文字填充等待时间 +""" diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/breakdown_agent/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/breakdown_agent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/breakdown_agent/agent.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/breakdown_agent/agent.py new file mode 100644 index 00000000..9740176e --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/breakdown_agent/agent.py @@ -0,0 +1,37 @@ +""" +分镜拆解 Agent +负责接收视频URL/本地文件，完成 FFmpeg 预处理 + ASR + 视觉分析 + BGM 分析 +无需外部后端服务，所有逻辑自包含在 tool 中 +""" + +import os + +from veadk import Agent + +from video_breakdown_agent.tools.process_video import process_video +from video_breakdown_agent.tools.analyze_segments_vision import analyze_segments_vision +from video_breakdown_agent.tools.analyze_bgm import analyze_bgm +from video_breakdown_agent.tools.video_upload import video_upload_to_tos +from .prompt import BREAKDOWN_AGENT_INSTRUCTION + +breakdown_agent = Agent( + name="breakdown_agent", + description=( + "负责视频分镜拆解：视频预处理（FFmpeg + ASR）、" + "视觉分析（doubao-vision）、BGM 分析。" + "支持URL链接和本地文件上传，输出完整分镜结构化数据。" + ), + instruction=BREAKDOWN_AGENT_INSTRUCTION, + tools=[ + process_video, + analyze_segments_vision, + analyze_bgm, + video_upload_to_tos, + ], + output_key="breakdown_result", # 结果写入 session.state["breakdown_result"] + model_extra_config={ + "extra_body": { + "thinking": {"type": os.getenv("THINKING_BREAKDOWN_AGENT", "disabled")} + } + }, +) diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/breakdown_agent/prompt.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/breakdown_agent/prompt.py new file mode 100644 index 00000000..9c15111b --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/breakdown_agent/prompt.py @@ -0,0 +1,92 @@ +""" +分镜拆解 Agent 提示词 +自包含模式：无需外部后端，tool 内嵌全部处理逻辑 +""" + +BREAKDOWN_AGENT_INSTRUCTION = """ +你是一个专业的视频分镜拆解专家，负责接收视频并完成完整的分镜拆解分析。 + +## 你的工具 + +1. **process_video** — 视频预处理（FFmpeg + ASR） + - 输入：视频 URL + - 输出：视频元数据、分段信息、关键帧 URL、ASR 语音识别文本 + - 注意：需要本机安装 FFmpeg + +2. **analyze_segments_vision** — 视觉分析（doubao-vision） + - 输入：自动从 session state 读取 process_video 的输出（无需手动传参） + - 输出：每个分镜的景别、运镜、画面内容、功能标签等 + +3. **analyze_bgm** — BGM 分析 + - 输入：音频 URL 和视频时长 + - 输出：BGM 风格、情绪、乐器、节奏等 + - 注意：模型不支持音频时会优雅降级 + +4. **video_upload_to_tos** — 本地文件上传到 TOS + - 仅在用户提供本地文件路径时使用 + +## 支持的输入方式 + +### 方式一：视频 URL 链接 +用户直接提供视频的公开 URL，例如： +- `https://example.com/video.mp4` +→ 直接使用 process_video 处理 + +### 方式二：本地文件路径 +用户提供本地文件路径，例如： +- `/Users/xxx/Downloads/video.mp4` +- `.media-uploads/video.mp4` +→ **优先直接使用** process_video 处理本地文件（工具已支持本地路径） + +#### 本地路径判定规则（重要） +- 只要用户提供的输入**不是**以 `http://` 或 `https://` 开头，都视为“本地文件路径”。\n + 包括相对路径（如 `.media-uploads/a.mp4`）与绝对路径。\n +- 对本地路径**直接调用** `process_video(video_url=本地路径)`。\n +- 仅当需要把本地文件转成可分享的外链，才使用 `video_upload_to_tos(file_path)` 上传后再处理。 + +## 完整工作流程 + +### Step 1: 获取视频 URL +- URL 输入 → 直接使用 +- 本地路径 → video_upload_to_tos 上传获取 URL + +### Step 2: 视频预处理 +- 调用 process_video(video_url) +- 获取分段、帧图、音频、ASR 文本等 +- 如果返回 error，告知用户具体原因并停止 + +### Step 3: 视觉分析（必须执行） +- 必须在 process_video 后调用 analyze_segments_vision()（无需传参） +- 即使 process_video 返回的 frame_urls 显示为"本地缓存"也必须调用（工具内部已处理 base64 回退） +- 获取每个分镜的视觉分析结果 + +### Step 4: BGM 分析（必须执行） +- 必须调用 analyze_bgm()（无需传参，工具自动处理音频来源，含 base64 回退） +- 模型不支持时自动降级，不影响主流程 + +### Step 5: 整合输出 +将所有数据整合为完整的分镜拆解结果，返回给用户： +- 视频基本信息（时长、分辨率） +- 分镜列表（含视觉分析、语音内容） +- BGM 分析 +- ASR 完整文本 + +## 输出格式 + +请以结构化方式返回，包含： +- **视频信息**：时长、分辨率 +- **分镜列表**：每个分镜的序号、时间段、景别、运镜、画面描述、语音内容、功能标签 +- **BGM 分析**：是否有 BGM、风格、情绪（如有） +- **完整语音文本**：ASR 识别的全部文字 + +## 注意事项 +- 视频预处理需要时间（取决于视频时长），请告知用户正在处理 +- 如果 ASR 未配置，语音识别部分为空，这是正常的 +- 如果 TOS 凭证未配置，帧和片段只在本地，不影响分析 +- 如果视觉分析部分失败，使用 fallback 数据继续 + +## 完成后行为（必须遵守） +- 当你完成分镜拆解并输出结果后，直接输出结果文本即可。 +- 你处于 SequentialAgent 流程中，框架会自动将控制权传递给下一个步骤，无需手动调用任何转移函数。 +- 不要在本 Agent 内继续处理"钩子分析/报告生成/下一步编排"请求，这些将由后续流程自动处理。 +""" diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/agent.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/agent.py new file mode 100644 index 00000000..24691732 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/agent.py @@ -0,0 +1,76 @@ +""" +前三秒钩子分析 Sub-Agent + +简化为单 Agent 模式： +1. hook_analyzer_agent — 使用 vision 模型进行多模态分析 +2. after_model_callback 仅做宽松输出修复，不做刚性 schema 校验 +""" + +import os + +from google.adk.tools import ToolContext +from veadk import Agent + +from video_breakdown_agent.hook.format_hook import soft_fix_hook_output +from video_breakdown_agent.tools.analyze_hook_segments import analyze_hook_segments +from video_breakdown_agent.utils.types import json_response_config +from .filtered_sequential import HookAnalyzerSequentialAgent +from .prompt import HOOK_ANALYZER_INSTRUCTION, HOOK_FORMAT_INSTRUCTION +from google.adk.agents.callback_context import CallbackContext + +# 第一阶段：多模态视觉分析（使用 vision 模型） + + +def _prime_hook_segments_state(callback_context: CallbackContext): + """Compute hook segments context from session.state before LLM runs.""" + inv = getattr(callback_context, "_invocation_context", None) + if not inv: + return None + state = getattr(callback_context, "state", None) + if isinstance(state, dict) and state.get("hook_segments_context"): + return None + tool_ctx = ToolContext(inv) + context = analyze_hook_segments(tool_ctx) + if isinstance(state, dict): + state["hook_segments_context"] = context + return None + + +hook_analysis_agent = Agent( + name="hook_analysis_agent", + model_name=os.getenv("MODEL_VISION_NAME", "doubao-seed-1-6-vision-250815"), + description="对视频前三秒分镜进行深度钩子分析，具备视觉分析能力，可直接观察关键帧图片进行专业评估", + instruction=HOOK_ANALYZER_INSTRUCTION, + tools=[], + before_agent_callback=_prime_hook_segments_state, + model_extra_config={ + "extra_body": { + "thinking": {"type": os.getenv("THINKING_HOOK_ANALYZER_AGENT", "disabled")}, + "caching": {"type": "disabled"}, # 禁用缓存，避免账户未激活 cache service + } + }, +) + +# 第二阶段：对齐 multimedia 的格式化收口模式 +hook_format_agent = Agent( + name="hook_format_agent", + model_name=os.getenv( + "MODEL_FORMAT_NAME", os.getenv("MODEL_AGENT_NAME", "doubao-seed-1-6-251015") + ), + description="将钩子分析结果格式化为结构化输出并投影为用户可读 Markdown", + instruction=HOOK_FORMAT_INSTRUCTION, + generate_content_config=json_response_config, + output_key="hook_analysis", + after_model_callback=[soft_fix_hook_output], + model_extra_config={ + "extra_body": { + "thinking": {"type": os.getenv("THINKING_HOOK_FORMAT_AGENT", "disabled")} + } + }, +) + +hook_analyzer_agent = HookAnalyzerSequentialAgent( + name="hook_analyzer_agent", + description="前三秒钩子分析顺序流程：先分析，再格式化输出", + sub_agents=[hook_analysis_agent, hook_format_agent], +) diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/filtered_sequential.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/filtered_sequential.py new file mode 100644 index 00000000..9cf086b8 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/filtered_sequential.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import AsyncGenerator + +from google.adk.agents import InvocationContext +from google.adk.events import Event +from veadk.agents.sequential_agent import SequentialAgent + + +class HookAnalyzerSequentialAgent(SequentialAgent): + async def _run_async_impl( + self, ctx: InvocationContext + ) -> AsyncGenerator[Event, None]: + async for event in super()._run_async_impl(ctx): + author = str(getattr(event, "author", "") or "") + if author == "hook_analysis_agent": + # 中间步骤只参与链路，不对外展示。 + continue + yield event diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/hook/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/hook/__init__.py new file mode 100644 index 00000000..29754321 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/hook/__init__.py @@ -0,0 +1 @@ +"""Hook utilities for hook_analyzer_agent.""" diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/hook/clean_tool_args.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/hook/clean_tool_args.py new file mode 100644 index 00000000..0c0ce6b0 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/hook/clean_tool_args.py @@ -0,0 +1,51 @@ +""" +Clean hook analyzer tool call args to avoid malformed payloads. +""" + +from __future__ import annotations + +import logging +from typing import Optional + +from google.adk.agents.callback_context import CallbackContext +from google.adk.events import Event +from google.adk.models import LlmResponse + +logger = logging.getLogger(__name__) + + +def clean_analyze_hook_arguments( + *, + callback_context: CallbackContext, + llm_response: LlmResponse, + model_response_event: Optional[Event] = None, +) -> Optional[LlmResponse]: + """Force `analyze_hook_segments` function call args to `{}` for stability.""" + _ = model_response_event + inv = getattr(callback_context, "_invocation_context", None) + agent = getattr(inv, "agent", None) + if not agent or getattr(agent, "name", "") != "hook_analysis_agent": + return llm_response + + content = getattr(llm_response, "content", None) + parts = getattr(content, "parts", None) + if not parts: + return llm_response + + fixed = False + for part in parts: + function_call = getattr(part, "function_call", None) + if not function_call: + continue + if getattr(function_call, "name", "") != "analyze_hook_segments": + continue + args = getattr(function_call, "args", None) + if args: + function_call.args = {} + fixed = True + + if fixed: + logger.warning( + "[clean_tool_args] polluted args detected in hook_analysis_agent, forced to {}" + ) + return llm_response diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/prompt.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/prompt.py new file mode 100644 index 00000000..d5252982 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/hook_analyzer_agent/prompt.py @@ -0,0 +1,143 @@ +""" +钩子分析 Sub-Agent 提示词（多模态视觉分析版本） +迁移自 prompts/hook_analyzer_prompt.py +""" + +HOOK_ANALYZER_INSTRUCTION = """ +你是一位短视频钩子分析专家，具备**视觉分析能力**，专注于评估视频前三秒的吸引力和转化潜力。 + +## 分析流程 + +当你接收到分镜数据时： +1. 你将直接获得“前三秒分镜上下文”（含关键帧图片 frame_images） +2. **重点观察每个分镜的关键帧图片**（frame_images 字段） +3. 结合视觉画面、文案、运镜等多维度信息进行深度分析 + +### 工具调用约束（必须遵守） +- 本步骤**禁止**发起任何工具调用。 +- 直接基于给定的前三秒分镜上下文进行分析输出。 + +## 核心能力 + +你可以直接**看到视频的关键帧图片**，请务必： +- 仔细观察画面构图、色彩、光影、人物表情、场景细节 +- 评估第一帧的"停住滑屏"能力 +- 分析画面与文案的配合度 +- 判断视觉元素对目标受众的吸引力 + +## 评估维度（基于视觉+文本综合分析，每项0-10分） + +## 评估维度 + +### 1. 视觉冲击力（0-10分）**【必须看图片评估】** +- **观察关键帧**：画面构图、色彩、光影是否有吸引力 +- **第一帧检查**：是否有"停住滑屏"的视觉冲击力 +- **运镜评估**：通过多帧对比判断运镜流畅度和动感 +- **细节观察**：人物表情、道具、场景设置是否吸睛 + +### 2. 语言钩子（0-10分）**【结合画面评估】** +- **文案与画面配合度**：开场文案与视觉内容是否呼应 +- **悬念制造**：文案+画面是否共同制造悬念、痛点或好奇心 +- **语气感染力**：语气和表达是否有感染力 +- **信息传达**：视觉+语言是否快速传达核心价值 + +### 3. 情绪唤起（0-10分）**【观察画面情绪】** +- **视觉情绪**：画面本身是否能快速触发情绪（好奇、惊讶、共鸣等） +- **人物情绪**：如有人物，观察其表情、肢体语言传递的情感 +- **氛围营造**：色调、光影、场景是否营造出特定情绪氛围 +- **情绪强度**：视觉+语言的情绪强度是否适合目标受众 + +### 4. 信息密度（0-10分） +- 前三秒传递的有效信息量 +- 核心价值主张是否清晰 +- 是否在最短时间内让观众理解内容主题 +- 信息层次是否合理 + +### 5. 节奏掌控（0-10分） +- 分镜切换节奏是否合理 +- 是否符合短视频平台特性（抖音/快手/小红书等） +- 画面转场是否流畅 +- 整体节奏感是否能留住观众 + +## 输出格式 + +此步骤（hook_analysis_agent）只输出**结构化自然语言分析**，用于提供给下一步 `hook_format_agent` 做结果整理。 +请勿在此步骤输出 JSON、代码块或函数调用片段。 + +## 评分标准 + +- 9-10分：顶级水平，有极强的吸引力和创意 +- 7-8分：优秀，具备良好的吸引力 +- 5-6分：一般，有改进空间 +- 3-4分：较弱，需要重大改进 +- 1-2分：很差，基本没有吸引力 + +## 注意事项 + +- **必须看图片**：视觉冲击力、情绪唤起、节奏掌控三个维度，必须基于关键帧图片分析 +- **具体引用**：评价时必须具体引用你看到的画面元素（如"第1帧的特写镜头中，蓝色水杯占据画面2/3"、"画面左侧的红色元素与背景形成强烈对比"） +- **深度分析**：每个 comment 字段必须包含 4-6 句话的深度分析，不要用泛泛的套话，要结合实际画面内容 +- **避免猜测**：如果某些分镜没有关键帧图片，基于文本描述分析即可，明确标注 +- **建议可执行**：优化建议要具体到"改什么→怎么改→预期效果"，至少给 4 条建议 +- **权重分配**：综合评分中，视觉冲击力权重30%，语言钩子25%，其他三项各15% +- **对标分析**：在 competitor_reference 中给出同类优秀视频的钩子设计参考 +- **容错优先**：当格式和内容冲突时，优先保证分析内容完整、可读，再尽量结构化 + +## 完成后行为（必须遵守） + +- 当你完成钩子分析并输出结果后，直接输出结果文本即可。 +- 你处于 SequentialAgent 流程中，框架会自动将控制权传递给下一个步骤，无需手动调用任何转移函数。 +- 不要自行处理"生成报告/完整分析/继续编排"等请求，这些将由后续流程或 Root Agent 统一决策。 +""" + +HOOK_FORMAT_INSTRUCTION = """ +你是钩子分析格式化助手，只负责把上一步的自然语言分析整理成**用户可读的 Markdown 报告**。 + +要求： +1) 只输出 Markdown 正文，不要输出任何 JSON、代码块或解释前后缀。 +2) 必须严格按以下模板输出（标题、顺序、标点都不要改），把 <> 中的内容替换为你的结果： + +## 前三秒钩子分析 + +### 综合评分: <0-10>/10 +- **钩子类型**: <一句话> +- **目标受众**: <一句话，不要写“未知/未明确”，没有就写“泛生活好物兴趣人群”> +- **留存预测**: <一句话，必须给出，不要留空> + +--- + +### 五维评分详情 + +#### 视觉冲击力: <0-10>/10 +> <2-4 句，结合画面细节> + +#### 语言钩子: <0-10>/10 +> <2-4 句；若无语音/字幕，可给 0-3 分并解释原因> + +#### 情绪唤起: <0-10>/10 +> <2-4 句> + +#### 信息密度: <0-10>/10 +> <2-4 句> + +#### 节奏掌控: <0-10>/10 +> <2-4 句> + +--- + +### 亮点 +- <要点1> +- <要点2> + +### 待改进 +- <要点1> +- <要点2> + +### 优化建议 +1. <建议1> +2. <建议2> +3. <建议3> +3) 禁止输出任何函数调用片段、工具日志、系统占位串（例如 PLHD/transfer_to_agent）。 + +完成输出后，直接输出结果即可。你处于 SequentialAgent 流程中，框架会自动将控制权传递给下一个步骤，无需手动调用任何转移函数。 +""" diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/agent.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/agent.py new file mode 100644 index 00000000..3c1e7534 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/agent.py @@ -0,0 +1,26 @@ +""" +报告生成 Agent +整合分镜拆解数据和钩子分析结果，生成专业的视频分析报告 +""" + +import os + +from veadk import Agent + +from video_breakdown_agent.tools.report_generator import generate_video_report +from .direct_output_callback import direct_output_callback +from .prompt import REPORT_AGENT_INSTRUCTION + +report_generator_agent = Agent( + name="report_generator_agent", + description="整合分镜拆解数据和钩子分析结果，生成专业的视频分析报告", + instruction=REPORT_AGENT_INSTRUCTION, + tools=[generate_video_report], + after_tool_callback=[direct_output_callback], + output_key="final_report", # 结果写入 session.state["final_report"] + model_extra_config={ + "extra_body": { + "thinking": {"type": os.getenv("THINKING_REPORT_AGENT", "disabled")} + } + }, +) diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/direct_output_callback.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/direct_output_callback.py new file mode 100644 index 00000000..91b51757 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/direct_output_callback.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from typing import Any, Optional + +from google.adk.tools import BaseTool, ToolContext + + +def direct_output_callback( + tool: BaseTool, + args: dict[str, Any], + tool_context: ToolContext, + tool_response: Any, +) -> Optional[Any]: + """ + 让工具结果直接输出，跳过 LLM 总结，避免 veadk web 同时展示“工具输出 + 最终回复”造成重复。 + """ + if tool.name == "generate_video_report": + tool_context.actions.skip_summarization = True + return tool_response diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/prompt.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/prompt.py new file mode 100644 index 00000000..ae3c8fde --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/sub_agents/report_generator_agent/prompt.py @@ -0,0 +1,31 @@ +""" +报告生成 Agent 提示词 +""" + +REPORT_AGENT_INSTRUCTION = """ +你是一个专业的视频分析报告专家，负责整合分镜拆解数据和钩子分析结果，生成完整的视频分析报告。 + +## 输入数据 + +你将获得以下数据来生成报告： +- **分镜拆解数据**（breakdown_result）：包含分镜列表、BGM分析、场景分析等 +- **钩子分析结果**（hook_analysis）：包含前三秒5维度评分和优化建议 + +## 工作流程 + +1. 收到“生成报告/给出报告/完整报告”请求时，立即调用 generate_video_report 工具 +2. 不要求用户重复提供 breakdown 或 hook 参数；工具会优先从 session state 自动读取 +2. 将生成的报告完整返回 + +## 注意事项 + +- 确保传入的数据完整 +- 如果缺少钩子分析数据，仍应生成包含分镜拆解的基础报告 +- 报告使用中文输出，Markdown 格式 +- 前三秒钩子分析是最重要的部分，需要详细展示 + +## 完成后行为（必须遵守） +- 当你生成并返回报告后，直接输出结果文本即可。 +- 你处于 SequentialAgent 流程中，框架会自动处理后续流程，无需手动调用任何转移函数。 +- 不要在本 Agent 内继续处理新的用户意图，后续回合由 Root Agent 负责统一编排。 +""" diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/__init__.py new file mode 100644 index 00000000..8cc778ce --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/__init__.py @@ -0,0 +1,15 @@ +from video_breakdown_agent.tools.process_video import process_video +from video_breakdown_agent.tools.analyze_segments_vision import analyze_segments_vision +from video_breakdown_agent.tools.analyze_bgm import analyze_bgm +from video_breakdown_agent.tools.report_generator import generate_video_report +from video_breakdown_agent.tools.video_upload import video_upload_to_tos +from video_breakdown_agent.tools.analyze_hook_segments import analyze_hook_segments + +__all__ = [ + "process_video", + "analyze_segments_vision", + "analyze_bgm", + "generate_video_report", + "video_upload_to_tos", + "analyze_hook_segments", +] diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_bgm.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_bgm.py new file mode 100644 index 00000000..46ea7e33 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_bgm.py @@ -0,0 +1,309 @@ +""" +BGM 音频分析工具（自包含） +使用豆包官方 API 分析背景音乐（支持文本模型）。 +模型不支持音频时优雅降级返回空结果。 + +迁移来源: video-breakdown-master/app/services/bgm_service.py +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any, Dict + +from google.adk.tools import ToolContext +from video_breakdown_agent.utils.doubao_client import call_doubao_text + +logger = logging.getLogger(__name__) + +# ==================== 枚举常量（来自 bgm_prompts.py）==================== + +MUSIC_STYLE_OPTIONS = [ + "流行", + "摇滚", + "电子", + "古典", + "爵士", + "民谣", + "R&B", + "嘻哈", + "乡村", + "蓝调", + "雷鬼", + "金属", + "朋克", + "独立", + "氛围", + "新世纪", + "世界音乐", + "原声配乐", + "轻音乐", + "纯音乐", +] + +EMOTION_OPTIONS = [ + "轻松愉悦", + "积极向上", + "温馨感人", + "激昂励志", + "浪漫甜蜜", + "活力四射", + "欢快热闹", + "平静舒缓", + "神秘悬疑", + "史诗宏大", + "梦幻空灵", + "复古怀旧", + "紧张刺激", + "悲伤忧郁", + "阴郁压抑", + "恐怖惊悚", +] + + +def _strip_code_fence(text: str) -> str: + """去除 markdown 代码块标记""" + text = text.strip() + if text.startswith("```"): + lines = text.split("\n") + if lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + text = "\n".join(lines) + return text.strip() + + +def _get_empty_result() -> Dict[str, Any]: + """返回空结果""" + return { + "has_bgm": False, + "music_style": None, + "emotion": None, + "instruments": None, + "tempo": None, + } + + +async def analyze_bgm( + audio_url: str = "", duration: float = 0.0, tool_context: ToolContext = None +) -> dict: + """ + 分析视频的背景音乐（BGM），识别风格、情绪、乐器、节奏等特征。 + 通过 LiteLLM 统一路由，需要模型支持音频输入（input_audio）， + 如果模型不支持则优雅降级返回空结果。 + + 应在 process_video 之后调用。如果不传参数，自动从 session state 读取 process_video 的结果。 + + Args: + audio_url: （可选）音频文件 URL。不传时自动从 session state 读取。 + duration: （可选）音频/视频时长（秒）。不传时自动从 session state 读取。 + + Returns: + dict: BGM 分析结果，包含 has_bgm, music_style, emotion, instruments, tempo 等字段。 + 模型不支持音频时返回 has_bgm=False 的空结果。 + """ + # 优先从 session state 读取 + audio_base64 = None + if tool_context and (not audio_url or duration <= 0): + state_result = tool_context.state.get("process_video_result") + if state_result and isinstance(state_result, dict): + if not audio_url: + audio_url = state_result.get("audio_url", "") + if not audio_url: + audio_base64 = state_result.get("audio_base64") + if duration <= 0: + duration = float(state_result.get("duration", 0)) + logger.info( + f"[analyze_bgm] 从 session state 读取: audio_url={'有' if audio_url else '无'}, " + f"audio_base64={'有' if audio_base64 else '无'}, duration={duration:.1f}s" + ) + + if not audio_url and not audio_base64: + logger.info("[analyze_bgm] 无音频 URL 且无 base64 音频数据，跳过 BGM 分析") + empty_result = _get_empty_result() + if tool_context is not None: + tool_context.state["bgm_analysis_result"] = empty_result + return empty_result + + # ---- 读取模型配置 ---- + # BGM 分析优先使用独立配置，支持切换到不同 provider + model_name = ( + os.getenv("MODEL_BGM_NAME") + or os.getenv("BGM_MODEL_NAME") + or os.getenv("MODEL_AGENT_NAME", "doubao-seed-1-6-251015") + ) + api_key = ( + os.getenv("MODEL_BGM_API_KEY") + or os.getenv("GEMINI_API_KEY") + or os.getenv("MODEL_AGENT_API_KEY") + or os.getenv("OPENAI_API_KEY", "") + ) + api_base = ( + os.getenv("MODEL_BGM_API_BASE") + or os.getenv("MODEL_AGENT_API_BASE") + or os.getenv("OPENAI_BASE_URL", "") + ) + + if not api_key: + logger.warning("[analyze_bgm] 缺少 API Key，跳过 BGM 分析") + empty_result = _get_empty_result() + if tool_context is not None: + tool_context.state["bgm_analysis_result"] = empty_result + return empty_result + + music_styles_str = "、".join(MUSIC_STYLE_OPTIONS) + emotions_str = "、".join(EMOTION_OPTIONS) + + system_prompt = ( + "你是一个专业的音频分析师，擅长分析音乐的风格、情绪、节奏和乐器组成。" + "请仔细聆听音频，分析其中的背景音乐（BGM）特征。" + ) + + user_prompt = f"""请分析这段音频的背景音乐特征。 + +【音频时长】{duration:.1f}秒 + +请从以下维度进行分析： + +1. **是否有BGM**：判断音频中是否包含背景音乐 + +2. **音乐风格**（如有BGM）： + - 主风格：从以下选项选择一个最匹配的：{music_styles_str} + - 次风格：可选1-2个补充风格 + - 风格标签：描述性标签（如：现代、复古、商业、文艺等） + +3. **情绪识别**（如有BGM）： + - 主情绪：从以下选项选择一个最匹配的：{emotions_str} + - 次情绪：可选1-2个补充情绪 + - 情绪强度：1-10分（1为最弱，10为最强） + - 情绪倾向：正面/中性/负面 + +4. **乐器分析**（如有BGM）： + - 识别的乐器 + - 主导乐器 + - 原声/电子比例 + +5. **节奏分析**（如有BGM）： + - BPM估计 + - 节奏快慢：快/中/慢 + - 节奏模式 + +输出要求：严格输出JSON格式""" + + # 构建音频内容部分 + # 根据模型类型选择不同的音频传递方式： + # - OpenAI 兼容（火山/OneRouter）：使用 input_audio 格式 + # 对于豆包文本 API：直接用文本描述代替音频输入 + # 豆包文本模型不支持音频，这里优雅降级 + user_prompt = f"""请分析这段音频的背景音乐特征。 + +时长：{duration:.1f}秒 +（注：当前使用文本分析模式，基于典型场景推测） + +**可选风格**：{music_styles_str} +**可选情绪**：{emotions_str} + +请输出标准 JSON，包含以下字段： +- has_bgm (boolean): 是否有BGM +- music_style (object): 风格分类 + - primary (string): 主要风格 + - secondary (array): 次要风格 + - tags (array): 风格标签 +- emotion (object): 情绪特征 + - primary (string): 主要情绪 + - secondary (array): 次要情绪 + - intensity (integer, 1-10): 强度 + - valence (string): 正面/中性/负面 +- instruments (object): 乐器组成 + - detected (array): 识别的乐器 + - dominant (string): 主导乐器 + - acoustic_ratio (number, 0-1): 声学乐器占比 + - electronic_ratio (number, 0-1): 电子乐器占比 +- tempo (object): 节奏特征 + - bpm_estimate (string): BPM 估计 + - pace (string): 快/中/慢 + - rhythm_pattern (string): 节奏模式描述""" + + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ] + + _schema = { # noqa: F841 # 保留作为 BGM 输出格式参考 + "name": "bgm_analysis_schema", + "schema": { + "type": "object", + "properties": { + "has_bgm": {"type": "boolean", "description": "是否有BGM"}, + "music_style": { + "type": "object", + "properties": { + "primary": {"type": "string"}, + "secondary": {"type": "array", "items": {"type": "string"}}, + "tags": {"type": "array", "items": {"type": "string"}}, + }, + }, + "emotion": { + "type": "object", + "properties": { + "primary": {"type": "string"}, + "secondary": {"type": "array", "items": {"type": "string"}}, + "intensity": {"type": "integer", "minimum": 1, "maximum": 10}, + "valence": {"type": "string", "enum": ["正面", "中性", "负面"]}, + }, + }, + "instruments": { + "type": "object", + "properties": { + "detected": {"type": "array", "items": {"type": "string"}}, + "dominant": {"type": "string"}, + "acoustic_ratio": {"type": "number"}, + "electronic_ratio": {"type": "number"}, + }, + }, + "tempo": { + "type": "object", + "properties": { + "bpm_estimate": {"type": "string"}, + "pace": {"type": "string", "enum": ["快", "中", "慢"]}, + "rhythm_pattern": {"type": "string"}, + }, + }, + }, + "required": ["has_bgm"], + }, + } + + try: + logger.info( + f"[analyze_bgm] 调用豆包模型 {model_name} 分析 BGM（音频分析当前仅以描述为主）..." + ) + response = await call_doubao_text( + model=model_name, + messages=messages, + api_key=api_key, + api_base=api_base, + temperature=0.7, + ) + content = response["choices"][0]["message"]["content"] + + if isinstance(content, str): + content = _strip_code_fence(content) + result = json.loads(content) + else: + result = content + + logger.info(f"[analyze_bgm] BGM 分析完成: has_bgm={result.get('has_bgm')}") + if tool_context is not None: + tool_context.state["bgm_analysis_result"] = result + return result + + except Exception as exc: + logger.warning(f"[analyze_bgm] 分析失败: {exc}，返回空结果") + empty_result = _get_empty_result() + if tool_context is not None: + tool_context.state["bgm_analysis_result"] = empty_result + return empty_result diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_hook_segments.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_hook_segments.py new file mode 100644 index 00000000..19c7f865 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_hook_segments.py @@ -0,0 +1,151 @@ +""" +前三秒分镜提取工具 +兼容新格式（process_video + analyze_segments_vision 输出）和旧格式 +支持从 tool_context.state 读取完整的 base64 frame_urls +""" + +import logging +from typing import Dict, List + +from google.adk.tools import ToolContext + +logger = logging.getLogger(__name__) + + +def _get_field(seg: Dict, new_key: str, old_key: str, default=""): + """兼容新旧字段名""" + return seg.get(new_key, seg.get(old_key, default)) + + +def analyze_hook_segments(tool_context: ToolContext = None) -> dict: + """ + 提取并分析视频前三秒的分镜数据，为钩子分析提供结构化的上下文信息。 + + 兼容两种输入格式： + - 新格式：process_video + analyze_segments_vision 输出（index, start, end, 视觉表现等） + - 旧格式：后端服务返回（segment_index, start_time, end_time, visual_content等） + + 数据来源优先级： + 1. tool_context.state["vision_analysis_result"] — 完整数据（含 base64 frame_urls） + 2. tool_context.state["process_video_result"]["segments"] — 兜底（通常不含视觉分析字段） + + 数据传递： + - 返回给 hook_analysis_agent 的数据包含完整 base64 frame_images + - 豆包视觉模型需要看到实际图片才能进行视觉评估 + - 完整图片数据同时存储在 tool_context.state["vision_analysis_result"] 中 + + Args: + tool_context: 工具上下文（用于读取完整数据） + + Returns: + dict: 前三秒分镜的结构化分析上下文（含 base64 frame_images） + """ + segments: List[Dict] | None = None + # 优先从 session state 读取完整数据（包含 base64 frame_urls） + if tool_context is not None: + vision_result = tool_context.state.get("vision_analysis_result") + if vision_result and isinstance(vision_result, list): + segments = vision_result + logger.info( + f"[analyze_hook_segments] 从 session state 读取完整数据: {len(segments)} 个分镜" + ) + + if not segments: + pv = tool_context.state.get("process_video_result") + if isinstance(pv, dict) and isinstance(pv.get("segments"), list): + segments = pv.get("segments") + logger.info( + f"[analyze_hook_segments] vision_analysis_result 缺失，回退到 process_video_result: {len(segments)} 个分镜" + ) + + if not segments: + return { + "error": "没有分镜数据", + "segment_count": 0, + "total_duration": 0, + "segments": [], + } + + # 提取前三秒的分镜 + first_segments = [] + cumulative_time = 0 + + for seg in segments: + # 兼容 end / end_time + end_time = seg.get("end", seg.get("end_time", 0)) + if cumulative_time >= 3.0 and first_segments: + break + first_segments.append(seg) + cumulative_time = end_time + + # 构造分析上下文（支持多模态） + context = { + "segment_count": len(first_segments), + "total_duration": cumulative_time, + "total_video_segments": len(segments), + "analysis_mode": "multimodal", + "segments": [], + } + + for s in first_segments: + frame_urls = s.get("frame_urls", []) + + # 兼容新旧字段名 + start_time = s.get("start", s.get("start_time", 0)) + end_time = s.get("end", s.get("end_time", 0)) + index = s.get("index", s.get("segment_index", 0)) + duration = end_time - start_time + + # 视觉表现：新格式在嵌套对象中，旧格式在顶层 + visual_info = s.get("视觉表现", {}) + visual_content = visual_info.get("画面内容", "") if visual_info else "" + if not visual_content: + visual_content = s.get("visual_content", "") + if not visual_content: + visual_content = s.get("summary", "") + + shot_type = visual_info.get("景别", "") if visual_info else "" + if not shot_type: + shot_type = s.get("shot_type", "") + + camera_movement = visual_info.get("运镜", "") if visual_info else "" + if not camera_movement: + camera_movement = s.get("camera_movement", "") + + segment_info = { + "index": index, + "start_time": start_time, + "end_time": end_time, + "duration": round(duration, 2), + "visual_content": visual_content, + "speech_text": s.get("speech_text", ""), + "shot_type": shot_type, + "camera_movement": camera_movement, + "function_tag": s.get("功能标签", s.get("function_tag", "")), + "headline": s.get("画面小标题", s.get("headline", "")), + "content_tags": s.get("内容标签", s.get("content_tags", [])), + "voice_type": s.get("语音类型", s.get("voice_type", "")), + "clip_url": s.get("clip_url", ""), + } + + # 添加关键帧图片（供 vision 模型使用，保留完整 base64 数据） + if frame_urls: + segment_info["frame_count"] = len(frame_urls) + # 构建标准的 image_url 格式（豆包 vision API 支持） + segment_info["frame_images"] = [ + {"type": "image_url", "image_url": {"url": url}} + for url in frame_urls[:3] # 最多 3 帧 + ] + else: + segment_info["frame_images"] = [] + segment_info["frame_count"] = 0 + + context["segments"].append(segment_info) + + total_frames = sum(s.get("frame_count", 0) for s in context["segments"]) + + logger.info( + f"前三秒分镜提取完成: {len(first_segments)}个分镜, " + f"总时长{cumulative_time:.1f}s, 关键帧{total_frames}张（含完整 base64 图片）" + ) + return context diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_segments_vision.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_segments_vision.py new file mode 100644 index 00000000..89d543d2 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/analyze_segments_vision.py @@ -0,0 +1,394 @@ +""" +视觉分镜分析工具（自包含） +使用豆包官方 vision API 分析关键帧图片（/responses endpoint） + +迁移来源: video-breakdown-master/app/services/gemini_service.py +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +from typing import Any, Dict, Optional + +from google.adk.tools import ToolContext +from video_breakdown_agent.utils.doubao_client import call_doubao_vision + +logger = logging.getLogger(__name__) + +# 功能标签枚举（来自 enhance_prompts.py） +FUNCTION_TAG_OPTIONS = [ + "强钩子", + "痛点切入", + "问题澄清", + "解决方案", + "产品介绍", + "信任建立", + "产品展示", + "价值预告", + "过渡", + "价值输出", + "成分深挖", + "效果证明", + "用户证言", + "情感背书", + "价值强化", + "效果总结", + "双重收益", + "强CTA", + "行动召唤", +] + + +def _build_segment_prompt(segment: Dict[str, Any]) -> Dict[str, Any]: + """构造单个分镜的分析提示词 + JSON schema""" + frame_count = len(segment.get("frame_urls", [])) + function_tags_str = "、".join(FUNCTION_TAG_OPTIONS) + + speech_hint = "" + if segment.get("speech_text"): + speech_hint = f"该片段的语音内容：「{segment['speech_text']}」" + + instruction = { + "task": "请对这个视频片段进行综合分析，包括视觉分析、语音类型判断和内容标注", + "segment_info": { + "镜号": segment["index"], + "起止时间": f"{segment['start']:.1f}s - {segment['end']:.1f}s", + "时长": f"{(segment['end'] - segment['start']):.1f}s", + "关键帧数量": frame_count, + "帧说明": f"提供了{frame_count}张关键帧图片（时间均匀采样），请综合所有图片进行分析", + "语音内容": speech_hint or "无语音内容", + }, + "分析维度": { + "视觉表现层": { + "分析要点": [ + "景别识别（特写/近景/中景/全景/远景）", + "运镜分析（固定/推拉/摇移/跟随/环绕等）", + "画面内容描述：必须综合所有关键帧，描述整个片段的画面内容", + "如果多张图片之间画面有变化，需要描述变化过程", + "如果画面基本一致，则描述画面的主要元素和场景", + ] + }, + "语音类型判断": { + "判断标准": { + "口播": "画面中有人物的脸部清晰可见，且正在说话", + "旁白": "画面中没有人在说话，配合画外音", + } + }, + "内容标注": { + "画面小标题": "根据画面内容，提炼≤10个汉字的核心要点标题", + "内容标签": "根据语音内容（如有）或画面内容，提炼1-2个关键标签", + "功能标签": f"识别该片段在视频叙事中的功能角色，从以下选项中选择1个最匹配的：{function_tags_str}", + }, + }, + "输出要求": "必须输出结构化JSON，包含视觉分析和内容标注所有字段", + } + + schema = { + "name": "segment_comprehensive_analysis_schema", + "schema": { + "type": "object", + "properties": { + "视觉表现": { + "type": "object", + "properties": { + "景别": { + "type": "string", + "enum": ["特写", "近景", "中景", "全景", "远景"], + }, + "运镜": { + "type": "string", + "enum": [ + "固定", + "推镜", + "拉镜", + "摇镜", + "移镜", + "跟随", + "环绕", + "推拉", + ], + }, + "画面内容": {"type": "string"}, + }, + "required": ["景别", "运镜", "画面内容"], + }, + "语音类型": {"type": "string", "enum": ["口播", "旁白"]}, + "summary": {"type": "string"}, + "画面小标题": { + "type": "string", + "description": "画面核心要点，≤10个汉字", + }, + "内容标签": { + "type": "array", + "items": {"type": "string"}, + "description": "1-2个内容关键标签", + }, + "功能标签": { + "type": "string", + "enum": FUNCTION_TAG_OPTIONS, + "description": "该片段在视频叙事中的功能角色", + }, + }, + "required": [ + "视觉表现", + "语音类型", + "summary", + "画面小标题", + "内容标签", + "功能标签", + ], + }, + } + + return {"instruction": instruction, "schema": schema} + + +def _create_fallback(segment: Dict[str, Any]) -> Dict[str, Any]: + """分析失败时的回退数据""" + return { + "index": segment["index"], + "start": segment["start"], + "end": segment["end"], + "summary": "模型解析失败", + "视觉表现": {"景别": "未知", "运镜": "未知", "画面内容": ""}, + "语音类型": "旁白", + "is_speech": False, + "speech_text": segment.get("speech_text"), + "frame_urls": segment.get("frame_urls", []), + "clip_url": segment.get("clip_url"), + "画面小标题": "解析失败", + "内容标签": [], + "功能标签": "其他", + } + + +def _strip_code_fence(text: str) -> str: + """去除 markdown 代码块标记""" + text = text.strip() + if text.startswith("```"): + lines = text.split("\n") + if lines[0].startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + text = "\n".join(lines) + return text.strip() + + +async def _analyze_single_segment( + segment: Dict[str, Any], + model_name: str, + api_key: str, + api_base: Optional[str] = None, +) -> Dict[str, Any]: + """分析单个分镜（使用豆包官方 vision API）""" + prompt_data = _build_segment_prompt(segment) + messages = [ + { + "role": "system", + "content": "你是一名专业的视频拆解分析师，请严格输出JSON。", + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": json.dumps(prompt_data["instruction"], ensure_ascii=False), + }, + ] + + [ + {"type": "image_url", "image_url": {"url": url}} + for url in segment.get("frame_urls", [])[:6] + ], + }, + ] + + max_retries = 3 + last_error = None + + for attempt in range(max_retries): + try: + # 调用豆包视觉 API（/responses endpoint，只传 model + input） + response = await call_doubao_vision( + model=model_name, + messages=messages, + api_key=api_key, + api_base=api_base, + ) + content = response["choices"][0]["message"]["content"] + + if isinstance(content, str): + content = _strip_code_fence(content) + structured = json.loads(content) + else: + structured = content + + # 填充基础信息 + structured["index"] = segment["index"] + structured["start"] = segment["start"] + structured["end"] = segment["end"] + structured["frame_urls"] = segment.get("frame_urls", []) + structured["clip_url"] = segment.get("clip_url") + + # 处理语音类型 + speech_type = structured.get("语音类型", "旁白") + if speech_type in ["旁白口述", "非口播"]: + speech_type = "旁白" + structured["语音类型"] = speech_type + structured["is_speech"] = speech_type == "口播" + structured["speech_text"] = segment.get("speech_text") + + return structured + + except Exception as exc: + last_error = exc + logger.warning( + f"分镜 {segment['index']} 分析失败 (尝试 {attempt + 1}/{max_retries}): {exc}" + ) + if attempt < max_retries - 1: + await asyncio.sleep(1 + attempt) + + logger.error(f"分镜 {segment['index']} 分析最终失败: {last_error}") + return _create_fallback(segment) + + +async def analyze_segments_vision( + segments_json: str = "", tool_context: ToolContext = None +) -> str: + """ + 使用视觉模型分析每个分镜的画面内容、景别、运镜、功能标签。 + 通过 LiteLLM 统一路由，支持 Gemini / 豆包 / GPT-4o 等多种模型。 + 应在 process_video 之后调用。 + + 数据来源（按优先级）： + 1. 自动从 session state 中读取 process_video 的输出（推荐，无需手动传参） + 2. 显式传入 segments_json 参数 + + Args: + segments_json: （可选）JSON 字符串，格式为 process_video 返回的 segments 数组。 + 如果不传或为空，自动从 session state 读取 process_video_result。 + + Returns: + str: 分析结果的 JSON 字符串（数组），每个元素包含视觉分析、功能标签等完整信息。 + """ + # 优先从 session state 读取（无需 LLM 序列化 JSON） + segments = None + + if tool_context and not segments_json: + state_result = tool_context.state.get("process_video_result") + if state_result and isinstance(state_result, dict): + segments = state_result.get("segments", []) + logger.info( + f"[analyze_segments_vision] 从 session state 读取 {len(segments)} 个分镜" + ) + + # 回退：从参数解析 + if not segments and segments_json: + try: + parsed = ( + json.loads(segments_json) + if isinstance(segments_json, str) + else segments_json + ) + if isinstance(parsed, dict): + segments = parsed.get("segments", []) + elif isinstance(parsed, list): + segments = parsed + except json.JSONDecodeError: + return json.dumps( + {"error": "segments_json 格式无效，需要合法 JSON 数组"}, + ensure_ascii=False, + ) + + if not segments or not isinstance(segments, list) or len(segments) == 0: + return json.dumps( + {"error": "没有可用的分镜数据。请先调用 process_video 处理视频。"}, + ensure_ascii=False, + ) + + # ---- 读取模型配置 ---- + # LiteLLM 模型名约定： + # gemini/gemini-2.5-pro → 直连 Google (需 GEMINI_API_KEY) + # openai/gemini-2.5-pro → OpenAI 兼容代理（如 OneRouter） + # doubao-seed-1-6-251015 → 火山方舟 (需 api_base + api_key) + # gpt-4o → OpenAI + model_name = os.getenv("MODEL_VISION_NAME") or os.getenv( + "MODEL_AGENT_NAME", "doubao-seed-1-6-vision" + ) + api_key = ( + os.getenv("MODEL_VISION_API_KEY") + or os.getenv("GEMINI_API_KEY") + or os.getenv("MODEL_AGENT_API_KEY") + or os.getenv("OPENAI_API_KEY", "") + ) + api_base = ( + os.getenv("MODEL_VISION_API_BASE") + or os.getenv("MODEL_AGENT_API_BASE") + or os.getenv("OPENAI_BASE_URL", "") + ) + + if not api_key: + return json.dumps( + { + "error": "缺少 API Key，请配置 MODEL_VISION_API_KEY / GEMINI_API_KEY / MODEL_AGENT_API_KEY" + }, + ensure_ascii=False, + ) + + # 过滤：只分析有帧图片的分镜 + segments_with_frames = [s for s in segments if s.get("frame_urls")] + if not segments_with_frames: + return json.dumps( + {"error": "没有分镜包含帧图片URL，无法进行视觉分析"}, ensure_ascii=False + ) + + logger.info( + f"[analyze_segments_vision] 开始分析 {len(segments_with_frames)} 个分镜 (model={model_name})" + ) + + # 并发分析，限制并发数 + concurrency = int(os.getenv("VISION_CONCURRENCY", "3")) + semaphore = asyncio.Semaphore(concurrency) + + async def analyze_one(seg: Dict[str, Any]) -> Dict[str, Any]: + async with semaphore: + return await _analyze_single_segment(seg, model_name, api_key, api_base) + + tasks = [analyze_one(seg) for seg in segments_with_frames] + results = await asyncio.gather(*tasks, return_exceptions=True) + + # 处理结果 + valid_results = [] + for i, result in enumerate(results): + if isinstance(result, Exception): + logger.error(f"分镜 {segments_with_frames[i]['index']} 分析异常: {result}") + valid_results.append(_create_fallback(segments_with_frames[i])) + else: + valid_results.append(result) + + valid_results.sort(key=lambda x: x["index"]) + + logger.info(f"[analyze_segments_vision] 分析完成: {len(valid_results)} 个分镜") + + # 存入 session state（tool_context 可能为 None，如单元测试场景） + if tool_context is not None: + tool_context.state["vision_analysis_result"] = valid_results + + # 精简返回数据：移除 base64 frame_urls 以避免 LLM context 超限 + # 注意：完整数据已存入 session state，后续工具（如 hook_analyzer）可从 state 读取 + for result in valid_results: + if "frame_urls" in result: + frame_urls = result["frame_urls"] + # 如果是 base64 data URL，替换为占位符以减少数据量 + result["frame_urls"] = [ + "（base64图片已省略）" if url and url.startswith("data:") else url + for url in frame_urls + ] + logger.debug( + f"分镜 {result['index']} 精简 frame_urls: {len(frame_urls)} 帧 → 占位符" + ) + + return json.dumps(valid_results, ensure_ascii=False) diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/process_video.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/process_video.py new file mode 100644 index 00000000..30b9b0f4 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/process_video.py @@ -0,0 +1,901 @@ +""" +视频预处理工具（自包含） +将 video-breakdown-master 后端服务的 FFmpeg + ASR + TOS 逻辑 +直接内嵌到一个 VeADK tool 函数中，无需外部后端服务。 + +迁移来源: +- video-breakdown-master/app/services/media_processor.py +- video-breakdown-master/app/services/asr_service.py +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import shutil +import subprocess +import tempfile +import uuid +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + +import os + +import httpx +import tos +from tos import HttpMethodType +from google.adk.tools import ToolContext + +logger = logging.getLogger(__name__) + +# ==================== 数据结构 ==================== + + +@dataclass +class SegmentAsset: + """单个时间片段的资源信息""" + + index: int + start: float + end: float + frame_paths: List[Path] = field(default_factory=list) + frame_urls: List[str] = field(default_factory=list) + clip_path: Optional[Path] = None + clip_url: Optional[str] = None + is_speech: bool = True + speech_text: Optional[str] = None + + +# ==================== FFmpeg 路径自动检测 ==================== + + +def _resolve_ffmpeg_paths() -> tuple[str, Optional[str]]: + """ + 自动检测 ffmpeg / ffprobe 可执行文件路径。 + + 优先级： + 1. 环境变量 / config.yaml 显式指定（仅当对应文件真实存在时） + 2. 系统 PATH（shutil.which） + 3. imageio-ffmpeg Python 包自带的二进制（随 uv 安装，无需系统装 FFmpeg） + + Returns: + (ffmpeg_bin, ffprobe_bin) — ffprobe_bin 为 None 时将使用 ffmpeg 回退探测 + """ + # -- ffmpeg -- + env_ffmpeg = os.getenv("FFMPEG_BIN") + ffmpeg_bin = None + + # 检查环境变量指定的路径是否真实可用 + if env_ffmpeg and shutil.which(env_ffmpeg): + ffmpeg_bin = env_ffmpeg + elif shutil.which("ffmpeg"): + ffmpeg_bin = shutil.which("ffmpeg") + + # 回退到 imageio-ffmpeg Python 包内置的二进制 + if not ffmpeg_bin: + try: + import imageio_ffmpeg + + ffmpeg_bin = imageio_ffmpeg.get_ffmpeg_exe() + logger.info(f"系统无 FFmpeg，使用 imageio-ffmpeg 内置: {ffmpeg_bin}") + except ImportError: + ffmpeg_bin = "ffmpeg" # 最终回退，让后续报错更清晰 + logger.warning("未找到 FFmpeg：系统 PATH 中没有，imageio-ffmpeg 包也未安装") + + # -- ffprobe -- + env_ffprobe = os.getenv("FFMPEG_FFPROBE_BIN") or os.getenv("FFPROBE_BIN") + ffprobe_bin = None + + if env_ffprobe and shutil.which(env_ffprobe): + ffprobe_bin = env_ffprobe + elif shutil.which("ffprobe"): + ffprobe_bin = shutil.which("ffprobe") + # imageio-ffmpeg 不包含 ffprobe，ffprobe_bin 保持 None → 回退到 ffmpeg -i 解析 + + return ffmpeg_bin, ffprobe_bin + + +# ==================== FFmpeg 辅助函数 ==================== + + +def _run_command(cmd: List[str]) -> str: + """执行外部命令""" + process = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return process.stdout.strip() + + +def _probe_video( + ffprobe_bin: Optional[str], ffmpeg_bin: str, video_path: Path +) -> Dict[str, Any]: + """ + 获取视频元数据。 + + 优先使用 ffprobe（结构化 JSON 输出更准确）， + 回退到 ffmpeg -i 解析 stderr（imageio-ffmpeg 不含 ffprobe 时）。 + """ + if ffprobe_bin: + return _probe_with_ffprobe(ffprobe_bin, video_path) + return _probe_with_ffmpeg(ffmpeg_bin, video_path) + + +def _probe_with_ffprobe(ffprobe_bin: str, video_path: Path) -> Dict[str, Any]: + """使用 ffprobe 获取视频元数据（首选方式）""" + cmd = [ + ffprobe_bin, + "-v", + "error", + "-print_format", + "json", + "-show_format", + "-show_streams", + str(video_path), + ] + result = _run_command(cmd) + info = json.loads(result) if result else {} + + duration = 0.0 + width = None + height = None + frame_rate = None + + format_info = info.get("format") or {} + if "duration" in format_info: + duration = float(format_info["duration"]) + + for stream in info.get("streams", []): + if stream.get("codec_type") == "video": + width = stream.get("width") + height = stream.get("height") + frame_rate = stream.get("r_frame_rate") + break + + return { + "duration": duration, + "width": width, + "height": height, + "frame_rate": frame_rate, + "size": format_info.get("size"), + "bit_rate": format_info.get("bit_rate"), + } + + +def _probe_with_ffmpeg(ffmpeg_bin: str, video_path: Path) -> Dict[str, Any]: + """ + 使用 ffmpeg -i 解析 stderr 获取视频元数据（ffprobe 不可用时的回退方案）。 + + imageio-ffmpeg 只包含 ffmpeg 不包含 ffprobe，此函数确保仍可正常工作。 + """ + import re + + # ffmpeg -i 不指定输出文件时退出码为 1，元数据输出在 stderr + # 注意：不能加 -v error，否则会屏蔽 Duration/Stream 等 info 级别元数据， + # 而 "At least one output file must be specified" 是 error 级别不被屏蔽， + # 导致 stderr 非空但无法解析出元数据。 + cmd = [ffmpeg_bin, "-i", str(video_path), "-hide_banner"] + process = subprocess.run(cmd, capture_output=True, text=True) + stderr = process.stderr + + duration = 0.0 + width = None + height = None + frame_rate = None + + # 解析 Duration: HH:MM:SS.ms + dur_match = re.search(r"Duration:\s*(\d+):(\d+):(\d+)\.(\d+)", stderr) + if dur_match: + h, m, s, cs = dur_match.groups() + duration = int(h) * 3600 + int(m) * 60 + int(s) + int(cs) / 100 + + # 解析 Stream #0:0: Video: ... 1920x1080 ... + res_match = re.search(r"Stream.*Video.*?(\d{2,5})x(\d{2,5})", stderr) + if res_match: + width = int(res_match.group(1)) + height = int(res_match.group(2)) + + # 解析帧率 ... 30 fps / 29.97 tbr + fps_match = re.search(r"(\d+(?:\.\d+)?)\s+(?:fps|tbr)", stderr) + if fps_match: + frame_rate = fps_match.group(1) + + logger.info( + f"ffmpeg 探测: duration={duration}s, {width}x{height}, fps={frame_rate}" + ) + return { + "duration": duration, + "width": width, + "height": height, + "frame_rate": frame_rate, + "size": None, + "bit_rate": None, + } + + +def _extract_audio_sync(ffmpeg_bin: str, video_path: Path) -> Optional[Path]: + """提取音频轨（同步版本，由 asyncio.to_thread 调用）""" + output_path = video_path.parent / f"{video_path.stem}.mp3" + cmd = [ + ffmpeg_bin, + "-y", + "-i", + str(video_path), + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-codec:a", + "libmp3lame", + "-b:a", + "128k", + str(output_path), + ] + try: + _run_command(cmd) + return output_path + except Exception as exc: + logger.warning(f"音频提取失败: {exc}") + return None + + +def _build_segments(duration: float) -> List[SegmentAsset]: + """ + 固定时长分镜方案： + - 0-3s, 3-5s, 5-10s, 10-20s, 之后每10s一段 + """ + segments: List[SegmentAsset] = [] + idx = 1 + + breakpoints = [0.0, 3.0, 5.0, 10.0, 20.0] + + for i in range(len(breakpoints) - 1): + start = breakpoints[i] + end = breakpoints[i + 1] + if duration <= start: + break + actual_end = min(end, duration) + if actual_end - start < 0.5: + break + segments.append(SegmentAsset(index=idx, start=start, end=actual_end)) + idx += 1 + + if duration > 20.0: + cursor = 20.0 + while cursor < duration: + candidate_end = min(duration, cursor + 10.0) + if candidate_end - cursor < 0.5: + break + segments.append(SegmentAsset(index=idx, start=cursor, end=candidate_end)) + idx += 1 + cursor = candidate_end + + return segments + + +def _assign_asr_text_to_segments( + segments: List[SegmentAsset], + asr_segments: List[Dict[str, Any]], +) -> None: + """将 ASR 识别的文本按时间重叠分配到分镜""" + for segment in segments: + texts = [] + for asr_seg in asr_segments: + asr_start = asr_seg.get("start", 0.0) + asr_end = asr_seg.get("end", 0.0) + text = asr_seg.get("text", "").strip() + if not text: + continue + overlap_start = max(segment.start, asr_start) + overlap_end = min(segment.end, asr_end) + if overlap_end > overlap_start: + texts.append(text) + if texts: + segment.is_speech = True + segment.speech_text = " ".join(texts) + else: + segment.is_speech = False + segment.speech_text = None + + +def _extract_segment_frames( + ffmpeg_bin: str, + video_path: Path, + segment: SegmentAsset, + frames_per_segment: int = 3, +) -> None: + """提取单个分镜的关键帧""" + frames_dir = video_path.parent / "frames" + frames_dir.mkdir(exist_ok=True) + seg_duration = max(segment.end - segment.start, 0.5) + safe_margin = 0.1 + + for i in range(frames_per_segment): + ratio = i / max(frames_per_segment - 1, 1) + raw_offset = segment.start + ratio * seg_duration + offset = min(raw_offset, segment.end - safe_margin) + offset = max(offset, segment.start) + output_path = frames_dir / f"seg{segment.index:03d}_frame_{i}.jpg" + + cmd = [ + ffmpeg_bin, + "-y", + "-ss", + f"{offset:.2f}", + "-i", + str(video_path), + "-frames:v", + "1", + "-q:v", + "8", # 降低质量减小文件体积（2=最高质量，31=最低质量） + str(output_path), + ] + try: + _run_command(cmd) + if output_path.exists(): + segment.frame_paths.append(output_path) + except Exception as exc: + logger.warning(f"片段 {segment.index} 第{i}帧提取失败: {exc}") + + +def _extract_single_clip( + ffmpeg_bin: str, + video_path: Path, + segment: SegmentAsset, + clips_dir: Path, +) -> None: + """切割单个视频片段""" + duration = segment.end - segment.start + output_path = clips_dir / f"seg{segment.index:03d}_clip.mp4" + + cmd = [ + ffmpeg_bin, + "-y", + "-ss", + f"{segment.start:.2f}", + "-i", + str(video_path), + "-t", + f"{duration:.2f}", + "-c:v", + "libx264", + "-c:a", + "aac", + "-b:v", + "1000k", + "-b:a", + "128k", + "-movflags", + "+faststart", + str(output_path), + ] + try: + _run_command(cmd) + segment.clip_path = output_path + except Exception as exc: + logger.warning(f"片段 {segment.index} 切割失败: {exc}") + segment.clip_path = None + + +# ==================== ASR 辅助函数 ==================== + + +async def _transcribe_audio(audio_url: str) -> Optional[Dict[str, Any]]: + """ + 调用火山引擎 ASR 获取音轨文本（提交 + 轮询） + 配置不全时静默跳过（优雅降级） + """ + # VeADK 扁平化: asr.app_id → ASR_APP_ID; 兼容旧名 VOLC_ASR_* + app_id = os.getenv("ASR_APP_ID") or os.getenv("VOLC_ASR_APP_ID", "") + access_key = os.getenv("ASR_ACCESS_KEY") or os.getenv("VOLC_ASR_ACCESS_KEY", "") + resource_id = os.getenv("ASR_RESOURCE_ID") or os.getenv( + "VOLC_ASR_RESOURCE_ID", "volc.bigasr.auc" + ) + submit_endpoint = os.getenv( + "ASR_ENDPOINT", + ) or os.getenv( + "VOLC_ASR_ENDPOINT", + "https://openspeech.bytedance.com/api/v3/auc/bigmodel/submit", + ) + query_endpoint = os.getenv( + "ASR_QUERY_ENDPOINT", + ) or os.getenv( + "VOLC_ASR_QUERY_ENDPOINT", + "https://openspeech.bytedance.com/api/v3/auc/bigmodel/query", + ) + + if not app_id or not access_key: + logger.info( + "未配置火山 ASR（VOLC_ASR_APP_ID / VOLC_ASR_ACCESS_KEY），跳过语音识别" + ) + return None + + request_id = str(uuid.uuid4()) + headers = { + "Content-Type": "application/json", + "X-Api-App-Key": app_id, + "X-Api-Access-Key": access_key, + "X-Api-Resource-Id": resource_id, + "X-Api-Request-Id": request_id, + "X-Api-Sequence": "-1", + } + payload = { + "user": {"uid": "video-breakdown-agent"}, + "audio": {"url": audio_url, "format": "mp3"}, + "request": { + "model_name": "bigmodel", + "enable_itn": True, + "enable_punc": True, + "show_utterances": True, + }, + } + + try: + # 提交任务 + async with httpx.AsyncClient(timeout=120) as client: + resp = await client.post(submit_endpoint, headers=headers, json=payload) + resp.raise_for_status() + + logger.info(f"ASR 任务已提交 request_id={request_id}") + await asyncio.sleep(5.0) + + # 轮询结果 + max_attempts = 15 + for attempt in range(1, max_attempts + 1): + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.post(query_endpoint, headers=headers, json={}) + resp.raise_for_status() + data = resp.json() + + status_code = resp.headers.get("X-Api-Status-Code") + + if status_code == "20000000": + # 识别成功 + return _parse_asr_result(data) + elif status_code == "20000003": + # 静音音频 + logger.info("ASR 检测到静音音频") + return {"text": "", "segments": []} + elif status_code in ["20000001", "20000002"]: + # 处理中/排队中 + logger.info(f"ASR 处理中 attempt={attempt}/{max_attempts}") + await asyncio.sleep(3.0) + else: + logger.error(f"ASR 返回错误码 status_code={status_code}") + return None + + logger.error(f"ASR 查询超时 request_id={request_id}") + except Exception as exc: + logger.error(f"ASR 异常: {exc}") + + return None + + +def _parse_asr_result(response_json: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """解析 ASR 返回结果""" + result = response_json.get("result") + if not result: + return None + + text_chunks: List[str] = [] + segments: List[Dict[str, Any]] = [] + + if isinstance(result, dict): + main_text = result.get("text") + if main_text: + text_chunks.append(main_text.strip()) + + utterances = result.get("utterances") or [] + if isinstance(utterances, list): + for item in utterances: + text = item.get("text") + if text: + segments.append( + { + "text": text.strip(), + "start": (item.get("start_time") or 0) / 1000.0, + "end": (item.get("end_time") or 0) / 1000.0, + } + ) + + merged_text = "\n".join([c for c in text_chunks if c]).strip() + if not merged_text: + return None + + return {"text": merged_text, "segments": segments} + + +# ==================== TOS 上传辅助 ==================== + + +def _get_tos_client() -> Optional[tos.TosClientV2]: + """创建 TOS 客户端（凭证不全时返回 None）""" + ak = os.getenv("VOLCENGINE_ACCESS_KEY", "") + sk = os.getenv("VOLCENGINE_SECRET_KEY", "") + if not ak or not sk: + try: + from veadk.auth.veauth.utils import get_credential_from_vefaas_iam + + cred = get_credential_from_vefaas_iam() + ak = cred.access_key_id + sk = cred.secret_access_key + except Exception: + return None + + region = os.getenv("DATABASE_TOS_REGION") or os.getenv("TOS_REGION", "cn-beijing") + endpoint = f"tos-{region}.volces.com" + + return tos.TosClientV2(ak=ak, sk=sk, endpoint=endpoint, region=region) + + +async def _upload_to_tos( + client: tos.TosClientV2, + bucket: str, + key: str, + content: bytes, + content_type: str, +) -> Optional[str]: + """上传字节到 TOS 并返回签名 URL""" + try: + await asyncio.to_thread( + client.put_object, + bucket=bucket, + key=key, + content=content, + content_type=content_type, + ) + signed = await asyncio.to_thread( + client.pre_signed_url, + http_method=HttpMethodType.Http_Method_Get, + bucket=bucket, + key=key, + expires=604800, + ) + return signed.signed_url + except Exception as exc: + logger.warning(f"TOS 上传失败 key={key}: {exc}") + return None + + +# ==================== 路径判断辅助 ==================== + + +def _resolve_local_path(video_url: str) -> Optional[Path]: + """ + 判断 video_url 是否为本地文件路径。 + + 支持格式： + - 绝对路径: /path/to/video.mp4 + - file:// 协议: file:///path/to/video.mp4 + - Windows 绝对路径: C:\\path\\to\\video.mp4（兼容） + + Returns: + Path 对象（如果是本地路径），否则 None + """ + if not video_url: + return None + + # file:// 协议 + if video_url.startswith("file://"): + local_str = video_url[7:] # 去掉 file:// + # file:///path → /path (Unix), file:///C:/path → C:/path (Windows) + if local_str.startswith("/") and len(local_str) > 2 and local_str[2] == ":": + local_str = local_str[1:] # Windows: /C:/path → C:/path + return Path(local_str) + + # Unix 绝对路径 + if video_url.startswith("/"): + return Path(video_url) + + # Windows 绝对路径 (C:\..., D:\...) + if len(video_url) >= 3 and video_url[1] == ":" and video_url[2] in ("/", "\\"): + return Path(video_url) + + # 相对路径：只要能在当前工作目录解析到本地文件，也视为本地 + try: + candidate = Path(video_url) + if candidate.exists() and candidate.is_file(): + return candidate.resolve() + except Exception: + pass + + # HTTP(S) URL 或其他 → 非本地 + return None + + +# ==================== 主工具函数 ==================== + + +async def process_video(video_url: str, tool_context: ToolContext) -> dict: + """ + 完整视频预处理流水线，替代原后端 breakdown 服务。 + + 流程：下载视频 -> FFprobe 元数据 -> FFmpeg 音频提取 -> 火山 ASR 语音识别 + -> 固定时长分段 -> FFmpeg 帧提取 + 片段切割 -> TOS 上传 + + 需要本机安装 FFmpeg（brew install ffmpeg）。 + ASR 需配置 VOLC_ASR_APP_ID + VOLC_ASR_ACCESS_KEY，未配置时跳过语音识别。 + + Args: + video_url: 视频URL（公开URL / TOS 签名URL）或本地文件路径（/path/to/video.mp4） + + Returns: + dict: 包含 duration, resolution, segments, audio_url, full_transcript 等 + """ + # 自动检测 ffmpeg/ffprobe 路径（系统 → imageio-ffmpeg 回退） + ffmpeg_bin, ffprobe_bin = _resolve_ffmpeg_paths() + # 减少帧数以降低 base64 体积（避免超过 LLM tokens 限制） + frames_per_segment = int( + os.getenv("FFMPEG_FRAMES_PER_SEGMENT") or os.getenv("FRAMES_PER_SEGMENT", "2") + ) + temp_base = os.getenv("FFMPEG_MEDIA_TEMP_DIR") or os.getenv( + "MEDIA_TEMP_DIR", "./.media-cache" + ) + Path(temp_base).mkdir(parents=True, exist_ok=True) + + task_id = datetime.now().strftime("%Y%m%d_%H%M%S") + "_" + uuid.uuid4().hex[:8] + temp_dir = Path(tempfile.mkdtemp(prefix=f"media_{task_id}_", dir=temp_base)) + local_video = temp_dir / f"{task_id}.mp4" + + # TOS 配置（VeADK 扁平化: database.tos.bucket → DATABASE_TOS_BUCKET） + bucket = os.getenv("DATABASE_TOS_BUCKET") or os.getenv( + "TOS_BUCKET", "video-breakdown-uploads" + ) + tos_prefix = os.getenv("TOS_OUTPUT_PREFIX", "videobreak") + + try: + # ---- Step 1: 获取本地视频文件 ---- + # 支持本地路径（/path/to/video.mp4 或 file:///path/to/video.mp4）和 HTTP URL + local_source = _resolve_local_path(video_url) + if local_source: + # 本地文件：直接复制到工作目录 + if not local_source.exists(): + return {"error": f"本地文件不存在: {local_source}"} + file_size = local_source.stat().st_size + max_video_size = 2 * 1024 * 1024 * 1024 # 2GB + if file_size > max_video_size: + return { + "error": f"视频文件过大（>{max_video_size // 1024 // 1024}MB），请压缩后重试" + } + shutil.copy2(str(local_source), str(local_video)) + logger.info( + f"[process_video] 使用本地文件: {local_source} ({file_size / 1024 / 1024:.1f}MB)" + ) + else: + # HTTP URL：流式下载 + logger.info(f"[process_video] 下载视频: {video_url[:100]}...") + max_video_size = 2 * 1024 * 1024 * 1024 # 2GB 上限 + total_downloaded = 0 + async with httpx.AsyncClient(timeout=300, follow_redirects=True) as client: + async with client.stream("GET", video_url) as resp: + resp.raise_for_status() + with open(local_video, "wb") as f: + async for chunk in resp.aiter_bytes(chunk_size=65536): + total_downloaded += len(chunk) + if total_downloaded > max_video_size: + return { + "error": f"视频文件过大（>{max_video_size // 1024 // 1024}MB），请压缩后重试" + } + f.write(chunk) + logger.info( + f"[process_video] 下载完成: {local_video} ({total_downloaded / 1024 / 1024:.1f}MB)" + ) + + # ---- Step 2: 元数据 ---- + metadata = await asyncio.to_thread( + _probe_video, ffprobe_bin, ffmpeg_bin, local_video + ) + duration = float(metadata.get("duration") or 0.0) + if duration <= 0: + return {"error": "无法获取视频时长，请确认视频URL有效"} + resolution = f"{metadata.get('width')}x{metadata.get('height')}" + logger.info( + f"[process_video] 元数据: 时长={duration:.1f}s, 分辨率={resolution}" + ) + + # ---- Step 3: 提取音频 ---- + audio_path = await asyncio.to_thread( + _extract_audio_sync, ffmpeg_bin, local_video + ) + audio_url_out = None + audio_base64 = None + + # TOS 客户端（帧/片段/音频上传） + tos_client = _get_tos_client() + + if audio_path and tos_client: + key = f"{tos_prefix}/{task_id}/audio/{audio_path.name}" + audio_url_out = await _upload_to_tos( + tos_client, bucket, key, audio_path.read_bytes(), "audio/mpeg" + ) + + # ---- Step 4: ASR 语音识别 ---- + asr_result = None + asr_segments = None + if audio_url_out: + asr_result = await _transcribe_audio(audio_url_out) + if asr_result: + asr_segments = asr_result.get("segments", []) + logger.info(f"[process_video] ASR 识别完成: {len(asr_segments)} 个分段") + + # ---- Step 5: 构建固定时长分镜 ---- + segments = _build_segments(duration) + if asr_segments: + _assign_asr_text_to_segments(segments, asr_segments) + logger.info(f"[process_video] 分镜: {len(segments)} 个片段") + + # ---- Step 6: 提取关键帧（并发） ---- + frame_tasks = [ + asyncio.to_thread( + _extract_segment_frames, + ffmpeg_bin, + local_video, + seg, + frames_per_segment, + ) + for seg in segments + ] + await asyncio.gather(*frame_tasks) + + # ---- Step 7: 切割视频片段（并发） ---- + clips_dir = temp_dir / "clips" + clips_dir.mkdir(exist_ok=True) + clip_tasks = [ + asyncio.to_thread( + _extract_single_clip, ffmpeg_bin, local_video, seg, clips_dir + ) + for seg in segments + ] + await asyncio.gather(*clip_tasks) + + # ---- Step 8: 并发上传到 TOS ---- + if tos_client: + upload_tasks = [] + + for seg in segments: + # 帧上传任务 + for fp in seg.frame_paths: + if fp.exists(): + key = f"{tos_prefix}/{task_id}/frames/{fp.name}" + upload_tasks.append( + ("frame", seg, fp, key, fp.read_bytes(), "image/jpeg") + ) + # 片段上传任务 + if seg.clip_path and seg.clip_path.exists(): + key = f"{tos_prefix}/{task_id}/clips/{seg.clip_path.name}" + upload_tasks.append( + ( + "clip", + seg, + seg.clip_path, + key, + seg.clip_path.read_bytes(), + "video/mp4", + ) + ) + + # 并发上传（信号量限制并发数，避免 TOS 限流） + tos_semaphore = asyncio.Semaphore(10) + + async def _upload_one(item): + kind, seg, path, key, data, ct = item + async with tos_semaphore: + url = await _upload_to_tos(tos_client, bucket, key, data, ct) + return kind, seg, url + + results = await asyncio.gather( + *[_upload_one(t) for t in upload_tasks], + return_exceptions=True, + ) + + for r in results: + if isinstance(r, Exception): + logger.warning(f"TOS 上传异常: {r}") + continue + kind, seg, url = r + if url: + if kind == "frame": + seg.frame_urls.append(url) + else: + seg.clip_url = url + + tos_client.close() + else: + logger.warning("[process_video] TOS 凭证未配置，跳过上传（帧/片段仅本地）") + + # ---- Step 8b: base64 帧图回退（TOS 不可用/上传失败时） ---- + for seg in segments: + if not seg.frame_urls and seg.frame_paths: + for fp in seg.frame_paths: + if fp.exists(): + b64 = base64.b64encode(fp.read_bytes()).decode() + seg.frame_urls.append(f"data:image/jpeg;base64,{b64}") + if seg.frame_urls: + logger.info( + f"片段 {seg.index}: TOS 不可用，使用 base64 帧图 ({len(seg.frame_urls)} 张)" + ) + + # 音频 base64 回退 + if not audio_url_out and audio_path and audio_path.exists(): + audio_base64 = base64.b64encode(audio_path.read_bytes()).decode() + logger.info( + f"[process_video] TOS 不可用，音频编码为 base64 ({len(audio_base64)} chars)" + ) + + # ---- 构建输出 ---- + full_transcript = None + if asr_result and asr_result.get("text"): + full_transcript = asr_result["text"] + + segments_output = [] + for seg in segments: + segments_output.append( + { + "index": seg.index, + "start": round(seg.start, 2), + "end": round(seg.end, 2), + "frame_urls": seg.frame_urls, + "clip_url": seg.clip_url, + "is_speech": seg.is_speech, + "speech_text": seg.speech_text, + } + ) + + result = { + "task_id": task_id, + "duration": round(duration, 2), + "resolution": resolution, + "metadata": metadata, + "audio_url": audio_url_out, + "audio_base64": audio_base64, + "full_transcript": full_transcript, + "segment_count": len(segments_output), + "segments": segments_output, + } + + # 存入 session state 供后续 sub-agent 使用（完整数据含 base64） + tool_context.state["process_video_result"] = result + + # 返回给 LLM 的瘦身版本：base64 data URL 替换为占位标记，节省 context tokens + slim_segments = [] + for seg_out in segments_output: + slim_seg = dict(seg_out) + frame_urls = slim_seg.get("frame_urls", []) + b64_count = sum( + 1 for u in frame_urls if isinstance(u, str) and u.startswith("data:") + ) + if b64_count > 0: + slim_seg["frame_urls"] = [ + f"(本地帧图已缓存，共{b64_count}张，后续工具会自动读取)" + ] + slim_segments.append(slim_seg) + + slim_result = dict(result) + slim_result["segments"] = slim_segments + if audio_base64: + slim_result["audio_base64"] = "(音频已缓存为base64，后续工具会自动读取)" + + return slim_result + + except httpx.HTTPError as exc: + return {"error": f"视频下载失败: {exc}"} + except subprocess.CalledProcessError as exc: + return { + "error": f"FFmpeg 处理失败: {exc.stderr[:300] if exc.stderr else str(exc)}" + } + except Exception as exc: + logger.error(f"[process_video] 异常: {exc}", exc_info=True) + return {"error": f"视频预处理失败: {str(exc)}"} + finally: + try: + shutil.rmtree(temp_dir, ignore_errors=True) + except Exception: + pass diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/report_generator.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/report_generator.py new file mode 100644 index 00000000..5076e7ec --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/report_generator.py @@ -0,0 +1,441 @@ +""" +视频分析报告生成工具 +兼容新格式（process_video + analyze_segments_vision 输出）和旧格式 +""" + +import logging +from datetime import datetime +from typing import Optional + +from google.adk.tools import ToolContext + +logger = logging.getLogger(__name__) + +_DEFAULT_HOOK_ANALYSIS = { + "overall_score": 0.0, + "visual_impact": 0.0, + "visual_comment": "", + "language_hook": 0.0, + "language_comment": "", + "emotion_trigger": 0.0, + "emotion_comment": "", + "information_density": 0.0, + "info_comment": "", + "rhythm_control": 0.0, + "rhythm_comment": "", + "hook_type": "未知", + "hook_type_analysis": "", + "target_audience": "", + "strengths": [], + "weaknesses": [], + "suggestions": [], + "competitor_reference": "", + "retention_prediction": "", +} + + +def _fallback_hook_from_markdown(markdown: str) -> dict: + """从 Markdown 兜底提取最基础字段，保证报告阶段可消费。""" + if not markdown: + return {} + + import re + + hook = dict(_DEFAULT_HOOK_ANALYSIS) + + score_match = re.search(r"综合评分[:：]\s*([0-9]+(?:\.[0-9]+)?)", markdown) + if score_match: + try: + hook["overall_score"] = float(score_match.group(1)) + except ValueError: + pass + + type_match = re.search(r"钩子类型[:：]\s*([^\n]+)", markdown) + if type_match: + hook["hook_type"] = type_match.group(1).strip() + + retention_match = re.search(r"留存预测[:：]\s*([^\n]+)", markdown) + if retention_match: + hook["retention_prediction"] = retention_match.group(1).strip() + + for field, label in ( + ("visual_impact", "视觉冲击力"), + ("language_hook", "语言钩子"), + ("emotion_trigger", "情绪唤起"), + ("information_density", "信息密度"), + ("rhythm_control", "节奏掌控"), + ): + m = re.search(rf"{label}[:：]\s*([0-9]+(?:\.[0-9]+)?)/10", markdown) + if m: + try: + hook[field] = float(m.group(1)) + except ValueError: + pass + return hook + + +def _resolve_report_inputs( + breakdown_data: Optional[dict], + hook_analysis: Optional[dict], + tool_context: Optional[ToolContext], +) -> tuple[dict, dict]: + """优先使用显式参数；缺失时从 session state 回填。""" + breakdown = breakdown_data if isinstance(breakdown_data, dict) else {} + hook = hook_analysis if isinstance(hook_analysis, dict) else {} + + state = getattr(tool_context, "state", None) if tool_context else None + if not isinstance(state, dict): + return breakdown, hook + + if not breakdown: + state_breakdown = state.get("process_video_result") + if isinstance(state_breakdown, dict): + breakdown = dict(state_breakdown) + + # 优先使用视觉分析后的分镜结果，提升报告细节质量 + state_vision_segments = state.get("vision_analysis_result") + if isinstance(state_vision_segments, list) and state_vision_segments: + if not isinstance(breakdown, dict): + breakdown = {} + breakdown["segments"] = state_vision_segments + if not breakdown.get("segment_count"): + breakdown["segment_count"] = len(state_vision_segments) + + if not hook: + state_hook = state.get("hook_analysis_struct") or state.get("hook_analysis") + if isinstance(state_hook, dict): + hook = state_hook + + if not hook: + state_hook_md = state.get("hook_analysis_markdown") + if isinstance(state_hook_md, str) and state_hook_md.strip(): + # 不强依赖解析分数字段：报告阶段直接消费 Markdown，避免因解析失败回填 0 分。 + hook = {"_raw_markdown": state_hook_md.strip()} + + # BGM 工具可选落盘，若存在则补齐 + state_bgm = state.get("bgm_analysis_result") + if isinstance(state_bgm, dict) and isinstance(breakdown, dict): + breakdown.setdefault("bgm_analysis", state_bgm) + + return breakdown, hook + + +def generate_video_report( + breakdown_data: Optional[dict] = None, + hook_analysis: Optional[dict] = None, + tool_context: ToolContext = None, +) -> str: + """ + 整合分镜拆解结果和钩子分析结果，生成结构化的 Markdown 视频分析报告。 + + 兼容两种输入格式： + - 新格式：process_video + analyze_segments_vision + analyze_bgm 输出 + - 旧格式：后端服务返回的数据 + + Args: + breakdown_data: 分镜拆解的完整结果数据 + hook_analysis: 前三秒钩子分析结果（由 hook_analyzer_agent 返回的 JSON） + + Returns: + str: 格式化的 Markdown 报告 + """ + breakdown_data, hook_analysis = _resolve_report_inputs( + breakdown_data, hook_analysis, tool_context + ) + + if not breakdown_data: + logger.warning( + "报告生成失败：缺少 breakdown_data（state: process_video_result）" + ) + return ( + "## 报告生成失败\n\n" + "- 未找到分镜拆解数据（`process_video_result`）。\n" + "- 请先执行视频分镜拆解，再生成报告。" + ) + + segments_data = breakdown_data.get("segments", []) + if not segments_data: + logger.warning("报告生成失败：breakdown_data 缺少 segments") + return ( + "## 报告生成失败\n\n" + "- 检测到分镜基础信息，但缺少分镜列表（`segments`）。\n" + "- 请先完成视觉分镜分析后再生成报告。" + ) + + # 提取基本信息 + duration = breakdown_data.get("duration", 0) + segment_count = breakdown_data.get("segment_count", 0) + resolution = breakdown_data.get("resolution", "N/A") + + # BGM 分析：新格式可能在 bgm_analysis 或 bgm 字段 + bgm = breakdown_data.get("bgm_analysis") or breakdown_data.get("bgm") or {} + scene = breakdown_data.get("scene_analysis") or {} + + # 完整语音文本（新格式特有） + full_transcript = breakdown_data.get("full_transcript", "") + + # 提取 BGM 信息 + music_style = ( + bgm.get("music_style", {}).get("primary", "N/A") + if isinstance(bgm.get("music_style"), dict) + else "N/A" + ) + emotion = ( + bgm.get("emotion", {}).get("primary", "N/A") + if isinstance(bgm.get("emotion"), dict) + else "N/A" + ) + tempo = ( + bgm.get("tempo", {}).get("bpm_estimate", "N/A") + if isinstance(bgm.get("tempo"), dict) + else "N/A" + ) + tempo_pace = ( + bgm.get("tempo", {}).get("pace", "N/A") + if isinstance(bgm.get("tempo"), dict) + else "N/A" + ) + has_bgm = bgm.get("has_bgm") + + # 提取场景信息 + primary_scene = scene.get("primary_scene", "N/A") if scene else "N/A" + video_style = scene.get("video_style", {}).get("overall", "N/A") if scene else "N/A" + target_audience = ( + ", ".join(scene.get("video_style", {}).get("target_audience", [])) + if scene + else "N/A" + ) + + # 构建钩子分析部分 + hook_section = _build_hook_section(hook_analysis) + + # 构建平台推荐部分 + platform_section = _build_platform_section(scene) + + # 构建分镜概览部分 + segments_section = _build_segments_overview(segments_data) + + # BGM 部分 + if has_bgm is False: + bgm_section = "## BGM 分析\n- 未检测到背景音乐" + elif has_bgm is None and music_style == "N/A": + bgm_section = "## BGM 分析\n- BGM 分析数据暂不可用" + else: + bgm_section = f"""## BGM 分析 +- **音乐风格**: {music_style} +- **情绪基调**: {emotion} +- **节拍**: {tempo} BPM（{tempo_pace}节奏）""" + + # 语音文本部分 + transcript_section = "" + if full_transcript: + transcript_section = f""" +--- + +## 语音文本 + +{full_transcript}""" + + report = f"""# 视频分析报告 + +## 基本信息 +- **视频时长**: {duration:.1f}秒 +- **分镜数量**: {segment_count}个 +- **分辨率**: {resolution} + +--- + +{hook_section} + +--- + +## 分镜概览 + +{segments_section} + +--- + +{bgm_section} + +--- + +## 场景分析 +- **主要场景**: {primary_scene} +- **视频风格**: {video_style} +- **目标受众**: {target_audience} + +{platform_section} +{transcript_section} + +--- + +**报告生成时间**: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")} +""" + + logger.info(f"报告生成完成，总长度: {len(report)} 字符") + return report + + +def _build_hook_section(hook_analysis: dict) -> str: + """构建前三秒钩子分析部分""" + if not hook_analysis: + return ( + "## 前三秒钩子分析\n\n" + "钩子分析部分数据不完整，已基于可用分镜数据生成主报告。" + ) + + raw_md = hook_analysis.get("_raw_markdown") + if isinstance(raw_md, str) and raw_md.strip(): + # 直接嵌入 hook_format_agent 的 Markdown 输出，保证稳定可读。 + return raw_md.strip() + + overall = hook_analysis.get("overall_score", 0) + hook_type = hook_analysis.get("hook_type", "N/A") + hook_type_analysis = hook_analysis.get("hook_type_analysis", "") + target_audience = hook_analysis.get("target_audience", "") + retention = hook_analysis.get("retention_prediction", "N/A") + competitor_ref = hook_analysis.get("competitor_reference", "") + + # 使用分段展示每个维度（评价内容可能很长，表格不适合） + dimensions = [ + ( + "视觉冲击力", + hook_analysis.get("visual_impact", 0), + hook_analysis.get("visual_comment", ""), + ), + ( + "语言钩子", + hook_analysis.get("language_hook", 0), + hook_analysis.get("language_comment", ""), + ), + ( + "情绪唤起", + hook_analysis.get("emotion_trigger", 0), + hook_analysis.get("emotion_comment", ""), + ), + ( + "信息密度", + hook_analysis.get("information_density", 0), + hook_analysis.get("info_comment", ""), + ), + ( + "节奏掌控", + hook_analysis.get("rhythm_control", 0), + hook_analysis.get("rhythm_comment", ""), + ), + ] + + dimension_sections = [] + for name, score, comment in dimensions: + dimension_sections.append(f"#### {name}：{score}/10\n> {comment}") + dimensions_text = "\n\n".join(dimension_sections) + + strengths = hook_analysis.get("strengths", []) + weaknesses = hook_analysis.get("weaknesses", []) + suggestions = hook_analysis.get("suggestions", []) + + strengths_text = "\n".join(f"- {s}" for s in strengths) if strengths else "- 暂无" + weaknesses_text = ( + "\n".join(f"- {w}" for w in weaknesses) if weaknesses else "- 暂无" + ) + suggestions_text = ( + "\n".join(f"{i + 1}. {s}" for i, s in enumerate(suggestions)) + if suggestions + else "1. 暂无" + ) + + # 可选字段 + hook_type_detail = f"\n{hook_type_analysis}" if hook_type_analysis else "" + audience_line = f"\n- **目标受众**: {target_audience}" if target_audience else "" + competitor_line = f"\n### 竞品参考\n{competitor_ref}" if competitor_ref else "" + + return f"""## 前三秒钩子分析（核心） + +### 综合评分: {overall}/10 +- **钩子类型**: {hook_type}{hook_type_detail}{audience_line} + +--- + +### 五维评分详情 + +{dimensions_text} + +--- + +### 亮点 +{strengths_text} + +### 待改进 +{weaknesses_text} + +### 优化建议 +{suggestions_text} +{competitor_line} + +### 留存预测 +**{retention}**""" + + +def _build_platform_section(scene: dict) -> str: + """构建平台推荐部分""" + if not scene: + return "" + + recommendations = scene.get("platform_recommendations", []) + if not recommendations: + return "" + + lines = ["### 平台推荐"] + for rec in recommendations: + platform = rec.get("platform", "N/A") + suitability = rec.get("suitability", "N/A") + reason = rec.get("reason", "") + lines.append(f"- **{platform}**（适合度: {suitability}）: {reason}") + + return "\n".join(lines) + + +def _build_segments_overview(segments: list) -> str: + """构建分镜概览表格（兼容新旧格式）""" + if not segments: + return "暂无分镜数据。" + + lines = [ + "| 镜号 | 时间 | 景别 | 运镜 | 功能标签 | 画面内容 |", + "|------|------|------|------|----------|----------|", + ] + + for seg in segments[:10]: + # 兼容新旧字段名 + index = seg.get("index", seg.get("segment_index", "-")) + start = seg.get("start", seg.get("start_time", 0)) + end = seg.get("end", seg.get("end_time", 0)) + time_range = f"{start:.1f}s-{end:.1f}s" + + # 视觉表现：新格式在嵌套对象中 + visual_info = seg.get("视觉表现", {}) + shot_type = visual_info.get("景别", "") if visual_info else "" + if not shot_type: + shot_type = seg.get("shot_type", "-") + + camera = visual_info.get("运镜", "") if visual_info else "" + if not camera: + camera = seg.get("camera_movement", "-") + + func_tag = seg.get("功能标签", seg.get("function_tag", "-")) + + visual = visual_info.get("画面内容", "") if visual_info else "" + if not visual: + visual = seg.get("visual_content", seg.get("summary", "-")) + + if len(visual) > 40: + visual = visual[:37] + "..." + + lines.append( + f"| {index} | {time_range} | {shot_type} | {camera} | {func_tag} | {visual} |" + ) + + if len(segments) > 10: + lines.append(f"\n*（仅展示前10个分镜，共{len(segments)}个）*") + + return "\n".join(lines) diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/video_upload.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/video_upload.py new file mode 100644 index 00000000..0c87a702 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/tools/video_upload.py @@ -0,0 +1,144 @@ +""" +视频文件上传到 TOS 对象存储工具 +基于 inspection_assistant/tools/tos_upload.py 模式实现 +""" + +import logging +import os +from datetime import datetime +from typing import Optional + +import tos +from tos import HttpMethodType + +logger = logging.getLogger(__name__) + +DEFAULT_BUCKET = "video-breakdown-uploads" +DEFAULT_REGION = "cn-beijing" + + +def video_upload_to_tos( + file_path: str, + bucket_name: Optional[str] = None, + object_key: Optional[str] = None, +) -> dict: + """ + 将本地视频文件上传到火山引擎 TOS 对象存储，并返回可访问的签名 URL。 + 上传成功后，返回的 URL 可直接用于 process_video 工具进行视频分镜分析。 + + Args: + file_path: 本地视频文件路径，例如 /path/to/video.mp4 + bucket_name: TOS 存储桶名称（可选，默认从配置读取） + object_key: 对象存储路径（可选，默认自动生成） + + Returns: + dict: 包含 video_url（签名URL）和上传信息，或 error 信息 + """ + # 从环境变量/config.yaml 读取配置（VeADK 扁平化: DATABASE_TOS_*，兼容旧: TOS_*） + if bucket_name is None: + bucket_name = os.getenv("DATABASE_TOS_BUCKET") or os.getenv( + "TOS_BUCKET", DEFAULT_BUCKET + ) + region = os.getenv("DATABASE_TOS_REGION") or os.getenv("TOS_REGION", DEFAULT_REGION) + + # 检查文件是否存在 + if not os.path.exists(file_path): + return {"error": f"文件不存在: {file_path}"} + + if not os.path.isfile(file_path): + return {"error": f"路径不是文件: {file_path}"} + + # 检查文件大小（限制 2GB） + file_size = os.path.getsize(file_path) + max_size = 2 * 1024 * 1024 * 1024 # 2GB + if file_size > max_size: + return {"error": f"文件过大（{file_size / 1024 / 1024:.0f}MB），最大支持 2GB"} + + # 获取凭证：优先从配置/环境变量，其次从 VeFaaS IAM + access_key = os.getenv("VOLCENGINE_ACCESS_KEY", "") + secret_key = os.getenv("VOLCENGINE_SECRET_KEY", "") + session_token = "" + + if not (access_key and secret_key): + try: + from veadk.auth.veauth.utils import get_credential_from_vefaas_iam + + cred = get_credential_from_vefaas_iam() + access_key = cred.access_key_id + secret_key = cred.secret_access_key + session_token = cred.session_token + except Exception: + pass + + if not access_key or not secret_key: + return { + "error": "缺少 TOS 访问凭证。请设置 VOLCENGINE_ACCESS_KEY 和 VOLCENGINE_SECRET_KEY 环境变量（或在 config.yaml 中配置 volcengine.access_key / volcengine.secret_key）" + } + + # 自动生成 object_key + if not object_key: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = os.path.basename(file_path) + object_key = f"video_breakdown/upload/{timestamp}_{filename}" + + # 上传文件 + client = None + try: + endpoint = f"tos-{region}.volces.com" + client = tos.TosClientV2( + ak=access_key, + sk=secret_key, + security_token=session_token, + endpoint=endpoint, + region=region, + ) + + logger.info(f"开始上传视频: {file_path}") + logger.info(f"目标桶: {bucket_name}, 对象键: {object_key}") + + # 检查桶是否存在 + try: + client.head_bucket(bucket_name) + except tos.exceptions.TosServerError as e: + if e.status_code == 404: + return {"error": f"TOS 存储桶 {bucket_name} 不存在，请先创建"} + raise + + # 上传 + result = client.put_object_from_file( + bucket=bucket_name, key=object_key, file_path=file_path + ) + + logger.info(f"上传成功! ETag: {result.etag}") + + # 生成签名 URL（7天有效） + signed_url_output = client.pre_signed_url( + http_method=HttpMethodType.Http_Method_Get, + bucket=bucket_name, + key=object_key, + expires=604800, + ) + + signed_url = signed_url_output.signed_url + logger.info("签名 URL 生成成功（7天有效）") + + return { + "video_url": signed_url, + "bucket": bucket_name, + "object_key": object_key, + "file_size_mb": round(file_size / 1024 / 1024, 2), + "message": "视频上传成功！可以使用返回的 video_url 调用 process_video 进行分镜预处理", + } + + except tos.exceptions.TosClientError as e: + logger.error(f"TOS 客户端错误: {e}") + return {"error": f"TOS 客户端错误: {e}"} + except tos.exceptions.TosServerError as e: + logger.error(f"TOS 服务端错误: code={e.code}, message={e.message}") + return {"error": f"TOS 服务端错误: {e.message}"} + except Exception as e: + logger.error(f"上传失败: {e}") + return {"error": f"视频上传失败: {str(e)}"} + finally: + if client: + client.close() diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/utils/__init__.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/utils/doubao_client.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/utils/doubao_client.py new file mode 100644 index 00000000..d5d9b496 --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/utils/doubao_client.py @@ -0,0 +1,369 @@ +""" +豆包官方 API 调用封装 +支持文本模型（/chat/completions）和视觉模型（/responses） + +文本模型：标准 OpenAI 兼容格式 +视觉模型：豆包专有 Responses API 格式 + - endpoint: /api/v3/responses + - 请求体：只有 model + input（无 temperature/parameters 等） + - input content 类型：input_text / input_image（非 text / image_url） + - 参考：https://www.volcengine.com/docs/82379/1541595 +""" + +import json +import logging +import os +from typing import List, Dict, Any, Optional + +import httpx + +logger = logging.getLogger(__name__) + + +class DoubaoClient: + """豆包 API 客户端""" + + def __init__( + self, + api_key: str, + api_base: str = "https://ark.cn-beijing.volces.com/api/v3", + timeout: int = 120, + ): + self.api_key = api_key + self.api_base = api_base.rstrip("/") + self.timeout = timeout + self.client = httpx.AsyncClient(timeout=timeout) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.client.aclose() + + # ==================== 文本模型 ==================== + + async def text_completion( + self, + model: str, + messages: List[Dict[str, Any]], + temperature: float = 0.7, + max_tokens: Optional[int] = None, + **kwargs, + ) -> Dict[str, Any]: + """ + 调用豆包文本模型 API（标准 /chat/completions endpoint） + + Args: + model: 模型名称，如 "doubao-seed-1-6-251015" + messages: 标准 OpenAI 格式消息列表 + temperature: 温度参数 + max_tokens: 最大 token 数 + + Returns: + 标准 OpenAI 兼容格式响应 + """ + url = f"{self.api_base}/chat/completions" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + payload: Dict[str, Any] = { + "model": model, + "messages": messages, + "temperature": temperature, + } + if max_tokens: + payload["max_tokens"] = max_tokens + payload.update(kwargs) + + logger.debug(f"豆包文本 API 请求: model={model}, messages={len(messages)} 条") + + try: + response = await self.client.post(url, headers=headers, json=payload) + response.raise_for_status() + result = response.json() + logger.debug("豆包文本 API 响应成功") + return result + except httpx.HTTPStatusError as e: + logger.error( + f"豆包文本 API HTTP 错误: {e.response.status_code} - {e.response.text}" + ) + raise + except Exception as e: + logger.error(f"豆包文本 API 调用失败: {e}") + raise + + # ==================== 视觉模型 ==================== + + async def vision_completion( + self, + model: str, + messages: List[Dict[str, Any]], + **kwargs, + ) -> Dict[str, Any]: + """ + 调用豆包视觉模型 API（/responses endpoint，豆包专有格式） + + 请求格式严格按照官方文档： + { + "model": "doubao-seed-1-6-vision-250815", + "input": [ + { + "role": "user", + "content": [ + {"type": "input_image", "image_url": "https://..."}, + {"type": "input_text", "text": "你看见了什么？"} + ] + } + ] + } + + Args: + model: 视觉模型名称，如 "doubao-seed-1-6-vision-250815" + messages: 标准 OpenAI 格式消息列表（会自动转换为豆包格式） + + Returns: + 转换为标准 OpenAI 格式的响应（方便上层代码统一处理） + """ + url = f"{self.api_base}/responses" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + # 将标准 OpenAI messages 转换为豆包 Responses API 格式 + doubao_input = self._convert_messages_to_doubao_input(messages) + + # ⚠️ 严格按照官方文档：只有 model + input，不传任何其他字段 + payload = { + "model": model, + "input": doubao_input, + } + + logger.debug(f"豆包视觉 API 请求: model={model}, input={len(doubao_input)} 条") + + try: + response = await self.client.post(url, headers=headers, json=payload) + response.raise_for_status() + raw = response.json() + + # 转换为统一的 OpenAI 格式 + return self._convert_vision_response(raw) + except httpx.HTTPStatusError as e: + logger.error( + f"豆包视觉 API HTTP 错误: {e.response.status_code} - {e.response.text}" + ) + raise + except Exception as e: + logger.error(f"豆包视觉 API 调用失败: {e}") + raise + + # ==================== 格式转换 ==================== + + @staticmethod + def _convert_messages_to_doubao_input( + messages: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + """ + 将标准 OpenAI messages 转换为豆包 Responses API 的 input 格式 + + OpenAI 格式 → 豆包格式: + "text" → "input_text" + "image_url" → "input_image" (且 image_url 从嵌套对象变为字符串) + "system" → 作为 input_text 加入（豆包 vision 不区分 system） + """ + doubao_input: List[Dict[str, Any]] = [] + + for msg in messages: + role = msg["role"] + content = msg["content"] + + # system message → 转为 user 的 input_text（豆包 vision API 可能不支持 system role） + if role == "system": + if isinstance(content, str): + doubao_input.append( + { + "role": "user", + "content": [ + {"type": "input_text", "text": f"[系统提示] {content}"} + ], + } + ) + continue + + # content 是纯字符串 + if isinstance(content, str): + doubao_input.append( + { + "role": role, + "content": [{"type": "input_text", "text": content}], + } + ) + continue + + # content 是列表（多模态内容） + doubao_content: List[Dict[str, Any]] = [] + for item in content: + item_type = item.get("type", "") + + if item_type == "text": + doubao_content.append( + { + "type": "input_text", + "text": item["text"], + } + ) + elif item_type == "image_url": + # 提取 URL（OpenAI 格式可能是 {"url": "..."} 或字符串） + url_data = item.get("image_url") + url = ( + url_data.get("url") if isinstance(url_data, dict) else url_data + ) + doubao_content.append( + { + "type": "input_image", + "image_url": url, + } + ) + else: + # 未知类型保持原样 + doubao_content.append(item) + + doubao_input.append( + { + "role": role, + "content": doubao_content, + } + ) + + return doubao_input + + @staticmethod + def _convert_vision_response(raw: Dict[str, Any]) -> Dict[str, Any]: + """ + 将豆包 Responses API 响应转换为 OpenAI Chat Completions 格式 + + 豆包 Responses API 实际响应格式： + { + "id": "resp_...", + "model": "doubao-seed-1-6-vision-250815", + "output": [ + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "..."}]}, + {"type": "message", "content": [{"type": "output_text", "text": "最终回答"}]} + ], + "usage": {"input_tokens": 100, "output_tokens": 50, "total_tokens": 150} + } + """ + # 如果已经是 OpenAI 格式（有 choices 字段），直接返回 + if "choices" in raw: + return raw + + content_text = "" + + # output 是一个列表，从中提取 type="message" 的项 + output_list = raw.get("output", []) + if isinstance(output_list, list): + for item in output_list: + if not isinstance(item, dict): + continue + item_type = item.get("type", "") + + # 主回答内容：type="message" + if item_type == "message": + content_parts = item.get("content", []) + for part in content_parts: + if isinstance(part, dict) and part.get("type") == "output_text": + content_text += part.get("text", "") + elif isinstance(part, str): + content_text += part + + # 如果没有从 message 中提取到内容，尝试其他方式 + if not content_text: + # 尝试从 output 中任何 summary_text 提取 + if isinstance(output_list, list): + for item in output_list: + if isinstance(item, dict): + for s in item.get("summary", []): + if isinstance(s, dict) and s.get("type") == "summary_text": + content_text += s.get("text", "") + + # 最终兜底 + if not content_text: + content_text = json.dumps(raw, ensure_ascii=False) + logger.warning( + f"豆包视觉 API 响应中未找到 message 内容，完整响应: {content_text[:300]}" + ) + + # 提取 usage + usage = raw.get("usage", {}) + input_tokens = usage.get("input_tokens", usage.get("prompt_tokens", 0)) + output_tokens = usage.get("output_tokens", usage.get("completion_tokens", 0)) + + return { + "id": raw.get("id", ""), + "object": "chat.completion", + "created": raw.get("created_at", 0), + "model": raw.get("model", ""), + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content_text}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": input_tokens, + "completion_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + }, + } + + async def close(self): + """关闭客户端""" + await self.client.aclose() + + +# ==================== 便捷函数 ==================== + + +async def call_doubao_text( + model: str, + messages: List[Dict[str, Any]], + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, +) -> Dict[str, Any]: + """便捷函数：调用豆包文本模型""" + if not api_key: + api_key = os.getenv("MODEL_AGENT_API_KEY") + if not api_key: + raise ValueError("API Key 未提供，且环境变量 MODEL_AGENT_API_KEY 未设置") + if not api_base: + api_base = os.getenv( + "MODEL_AGENT_API_BASE", "https://ark.cn-beijing.volces.com/api/v3" + ) + + async with DoubaoClient(api_key=api_key, api_base=api_base) as client: + return await client.text_completion(model=model, messages=messages, **kwargs) + + +async def call_doubao_vision( + model: str, + messages: List[Dict[str, Any]], + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, +) -> Dict[str, Any]: + """便捷函数：调用豆包视觉模型（自动转换为 Responses API 格式）""" + if not api_key: + api_key = os.getenv("MODEL_VISION_API_KEY") + if not api_key: + raise ValueError("API Key 未提供，且环境变量 MODEL_VISION_API_KEY 未设置") + if not api_base: + api_base = os.getenv( + "MODEL_VISION_API_BASE", "https://ark.cn-beijing.volces.com/api/v3" + ) + + async with DoubaoClient(api_key=api_key, api_base=api_base) as client: + return await client.vision_completion(model=model, messages=messages, **kwargs) diff --git a/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/utils/types.py b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/utils/types.py new file mode 100644 index 00000000..58fd8c0c --- /dev/null +++ b/python/02-use-cases/video_breakdown_agent/video_breakdown_agent/utils/types.py @@ -0,0 +1,49 @@ +""" +数据类型定义 +定义 Pydantic 模型用于 Agent 输出格式校验 +""" + +from typing import List, Optional + +from google.genai import types +from pydantic import BaseModel, Field + +# JSON 输出配置 +json_response_config = types.GenerateContentConfig( + response_mime_type="application/json", + max_output_tokens=18000, +) + + +class HookAnalysis(BaseModel): + """前三秒钩子分析结果的结构化输出""" + + overall_score: float = Field(description="综合评分(0-10)，各维度加权平均") + visual_impact: float = Field(description="视觉冲击力分数(0-10)") + visual_comment: str = Field(description="视觉冲击力的具体评价") + language_hook: float = Field(description="语言钩子分数(0-10)") + language_comment: str = Field(description="语言钩子的具体评价") + emotion_trigger: float = Field(description="情绪唤起分数(0-10)") + emotion_comment: str = Field(description="情绪唤起的具体评价") + information_density: float = Field(description="信息密度分数(0-10)") + info_comment: str = Field(description="信息密度的具体评价") + rhythm_control: float = Field(description="节奏掌控分数(0-10)") + rhythm_comment: str = Field(description="节奏掌控的具体评价") + hook_type: str = Field( + description="钩子类型（痛点型/好奇型/冲突型/价值型/情感型/视觉冲击型/悬念型）" + ) + hook_type_analysis: Optional[str] = Field( + default="", description="为什么判定为该钩子类型的详细理由" + ) + target_audience: Optional[str] = Field(default="", description="推测的目标受众画像") + strengths: List[str] = Field(description="优点列表（需具体引用画面元素）") + weaknesses: List[str] = Field(description="不足列表（需具体指出问题）") + suggestions: List[str] = Field( + description="优化建议列表（说明改什么、怎么改、预期效果）" + ) + competitor_reference: Optional[str] = Field( + default="", description="同品类/同类型优秀视频的钩子设计参考" + ) + retention_prediction: str = Field( + description="3秒留存预测（高/中/低）及详细原因分析" + )