feat: init tiktok platform extension + CLAUDE.md

api/main.py

@@ -154,6 +154,7 @@ async def get_platforms():
             {"value": "wb", "label": "Weibo", "icon": "message-circle"},
             {"value": "tieba", "label": "Baidu Tieba", "icon": "messages-square"},
             {"value": "zhihu", "label": "Zhihu", "icon": "help-circle"},
+            {"value": "tiktok", "label": "TikTok", "icon": "video"},
         ]
     }
 

api/routers/data.py

@@ -220,10 +220,11 @@ async def get_data_stats():
 
                 # Statistics by platform (inferred from path)
                 rel_path = str(file_path.relative_to(DATA_DIR))
-                for platform in ["xhs", "dy", "ks", "bili", "wb", "tieba", "zhihu"]:
+                for platform in ["xhs", "dy", "ks", "bili", "wb", "tieba", "zhihu", "tiktok"]:
                     if platform in rel_path.lower():
                         stats["by_platform"][platform] = stats["by_platform"].get(platform, 0) + 1
                         break
+
             except Exception:
                 continue
 

api/schemas/crawler.py

@@ -33,6 +33,8 @@ class PlatformEnum(str, Enum):
     WEIBO = "wb"
     TIEBA = "tieba"
     ZHIHU = "zhihu"
+    TIKTOK = "tiktok"
+
 
 
 class LoginTypeEnum(str, Enum):

cmd_arg/arg.py

@@ -47,6 +47,7 @@ class PlatformEnum(str, Enum):
     WEIBO = "wb"
     TIEBA = "tieba"
     ZHIHU = "zhihu"
+    TIKTOK = "tiktok"
 
 
 class LoginTypeEnum(str, Enum):
@@ -162,7 +163,7 @@ def main(
             PlatformEnum,
             typer.Option(
                 "--platform",
-                help="Media platform selection (xhs=XiaoHongShu | dy=Douyin | ks=Kuaishou | bili=Bilibili | wb=Weibo | tieba=Baidu Tieba | zhihu=Zhihu)",
+                help="Media platform selection (xhs=XiaoHongShu | dy=Douyin | ks=Kuaishou | bili=Bilibili | wb=Weibo | tieba=Baidu Tieba | zhihu=Zhihu | tiktok=TikTok)",
                 rich_help_panel="Basic Configuration",
             ),
         ] = _coerce_enum(PlatformEnum, config.PLATFORM, PlatformEnum.XHS),
@@ -384,6 +385,8 @@ def main(
                 ]
             elif platform == PlatformEnum.ZHIHU:
                 config.ZHIHU_SPECIFIED_ID_LIST = specified_id_list
+            elif platform == PlatformEnum.TIKTOK:
+                config.TIKTOK_VIDEO_ID_LIST = specified_id_list
 
         if creator_id_list:
             if platform == PlatformEnum.XHS:
@@ -400,6 +403,8 @@ def main(
                 config.TIEBA_CREATOR_URL_LIST = [
                     _normalize_tieba_creator_url(item) for item in creator_id_list
                 ]
+            elif platform == PlatformEnum.TIKTOK:
+                config.TIKTOK_CREATOR_ID_LIST = creator_id_list
 
         return SimpleNamespace(
             platform=config.PLATFORM,

config/base_config.py

@@ -56,7 +56,7 @@
 # 是否启用 CDP 模式 - 使用用户本地的 Chrome/Edge 浏览器进行爬取，具有更好的反检测能力
 # 开启后，会自动检测并启动用户的 Chrome/Edge 浏览器，通过 CDP 协议进行控制
 # 该方式使用真实浏览器环境，包括用户的扩展、Cookie 和设置，大幅降低被风控检测的风险
-ENABLE_CDP_MODE = True
+ENABLE_CDP_MODE = False
 
 # CDP 调试端口，用于与浏览器通信
 # 如果端口被占用，系统会自动尝试下一个可用端口
@@ -105,7 +105,7 @@
 MAX_CONCURRENCY_NUM = 1
 
 # Whether to enable crawling media mode (including image or video resources), crawling media is not enabled by default
-ENABLE_GET_MEIDAS = False
+ENABLE_GET_MEIDAS = True
 
 # Whether to enable comment crawling mode. Comment crawling is enabled by default.
 ENABLE_GET_COMMENTS = True
@@ -115,11 +115,11 @@
 
 # Whether to enable the mode of crawling second-level comments. By default, crawling of second-level comments is not enabled.
 # If the old version of the project uses db, you need to refer to schema/tables.sql line 287 to add table fields.
-ENABLE_GET_SUB_COMMENTS = False
+ENABLE_GET_SUB_COMMENTS = True
 
 # word cloud related
 # Whether to enable generating comment word clouds
-ENABLE_GET_WORDCLOUD = False
+ENABLE_GET_WORDCLOUD = True
 # Custom words and their groups
 # Add rule: xx:yy where xx is a custom-added phrase, and yy is the group name to which the phrase xx is assigned.
 CUSTOM_WORDS = {
@@ -147,3 +147,4 @@
 from .weibo_config import *
 from .tieba_config import *
 from .zhihu_config import *
+from .tiktok_config import *

config/tiktok_config.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2026 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/config/tiktok_config.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于 any 商业用途。
+# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于 any 非法或不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
+
+# TikTok platform configuration
+
+# Keyword list to search and crawl
+TIKTOK_KEYWORD_LIST: list = ["AI tools", "review sản phẩm"]
+
+# Specify TikTok video URL/ID list
+TIKTOK_VIDEO_ID_LIST: list = []
+
+# Specify TikTok creator URL/ID list
+TIKTOK_CREATOR_ID_LIST: list = []
+
+# Specify TikTok hashtag list
+TIKTOK_HASHTAG_LIST: list = []
+
+# Max comments count to crawl per video
+TIKTOK_MAX_COMMENTS_PER_VIDEO: int = 100
+
+# Max concurrency for task requests
+TIKTOK_MAX_CONCURRENCY: int = 3

constant/tiktok_constant.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2026 relakkes@gmail.com
+#
+# This file is part of MediaCrawler project.
+# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/constant/tiktok_constant.py
+# GitHub: https://github.com/NanmiCoder
+# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
+#
+
+# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
+# 1. 不得用于 any 商业用途。
+# 2. 使用时应遵守目标平台的使用条款 and robots.txt规则。
+# 3. 不得进行大规模爬取或对平台造成运营干扰。
+# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
+# 5. 不得用于 any 非法 or 不当的用途。
+#
+# 详细许可条款请参阅项目根目录下的LICENSE文件。
+# 使用本代码即表示您同意遵守上述原则 and LICENSE中的所有条款。
+
+TIKTOK_BASE_URL = "https://www.tiktok.com"
+
+# API endpoints
+SEARCH_VIDEO_URL = "/api/search/general/full/"
+VIDEO_DETAIL_URL = "/api/item/detail/"
+COMMENT_LIST_URL = "/api/comment/list/"
+COMMENT_REPLY_URL = "/api/comment/reply/list/"
+CREATOR_PROFILE_URL = "/api/user/detail/"
+CREATOR_VIDEO_LIST_URL = "/api/post/item_list/"
+HASHTAG_DETAIL_URL = "/api/challenge/detail/"
+HASHTAG_VIDEO_URL = "/api/challenge/item_list/"
+
+COMMON_PARAMS = {
+    "aid": "1988",
+    "app_language": "en",
+    "app_name": "tiktok_web",
+    "browser_language": "en-US",
+    "browser_platform": "Win32",
+    "channel": "tiktok_web",
+    "device_platform": "web_pc",
+    "os": "windows",
+    "region": "VN",
+    "tz_name": "Asia/Ho_Chi_Minh",
+}

database/models.py

@@ -450,3 +450,68 @@ class ZhihuCreator(Base):
     get_voteup_count = Column(Integer, default=0, comment='获赞数')
     add_ts = Column(BigInteger, comment='添加时间戳')
     last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
+
+
+class TiktokAweme(Base):
+    __tablename__ = 'tiktok_aweme'
+    id = Column(Integer, primary_key=True, comment='主键ID')
+    user_id = Column(String(255), comment='用户ID')
+    nickname = Column(Text, comment='用户昵称')
+    avatar = Column(Text, comment='用户头像')
+    user_signature = Column(Text, comment='用户签名')
+    ip_location = Column(Text, comment='IP地址位置')
+    add_ts = Column(BigInteger, comment='添加时间戳')
+    last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
+    aweme_id = Column(String(255), index=True, comment='作品ID')
+    aweme_type = Column(Text, comment='作品类型')
+    title = Column(Text, comment='作品标题')
+    desc = Column(Text, comment='作品描述')
+    create_time = Column(BigInteger, index=True, comment='创建时间戳')
+    liked_count = Column(Text, comment='点赞数')
+    comment_count = Column(Text, comment='评论数')
+    share_count = Column(Text, comment='分享数')
+    collected_count = Column(Text, comment='收藏数')
+    aweme_url = Column(Text, comment='作品URL')
+    cover_url = Column(Text, comment='封面URL')
+    video_download_url = Column(Text, comment='视频下载URL')
+    music_download_url = Column(Text, comment='音乐下载URL')
+    note_download_url = Column(Text, comment='笔记下载URL')
+    source_keyword = Column(Text, default='', comment='来源关键词')
+
+
+class TiktokAwemeComment(Base):
+    __tablename__ = 'tiktok_aweme_comment'
+    id = Column(Integer, primary_key=True, comment='主键ID')
+    user_id = Column(String(255), comment='用户ID')
+    nickname = Column(Text, comment='用户昵称')
+    avatar = Column(Text, comment='用户头像')
+    user_signature = Column(Text, comment='用户签名')
+    ip_location = Column(Text, comment='IP地址位置')
+    add_ts = Column(BigInteger, comment='添加时间戳')
+    last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
+    comment_id = Column(String(255), index=True, comment='评论ID')
+    aweme_id = Column(String(255), index=True, comment='作品ID')
+    content = Column(Text, comment='评论内容')
+    create_time = Column(BigInteger, comment='创建时间戳')
+    sub_comment_count = Column(Text, comment='子评论数')
+    parent_comment_id = Column(String(255), comment='父评论ID')
+    like_count = Column(Text, default='0', comment='点赞数')
+    pictures = Column(Text, default='', comment='图片')
+
+
+class TiktokCreator(Base):
+    __tablename__ = 'tiktok_creator'
+    id = Column(Integer, primary_key=True, comment='主键ID')
+    user_id = Column(String(255), comment='用户ID')
+    nickname = Column(Text, comment='用户昵称')
+    avatar = Column(Text, comment='用户头像')
+    ip_location = Column(Text, comment='IP地址位置')
+    add_ts = Column(BigInteger, comment='添加时间戳')
+    last_modify_ts = Column(BigInteger, comment='最后修改时间戳')
+    desc = Column(Text, comment='描述')
+    gender = Column(Text, comment='性别')
+    follows = Column(Text, comment='关注数')
+    fans = Column(Text, comment='粉丝数')
+    interaction = Column(Text, comment='互动数')
+    videos_count = Column(String(255), comment='视频数量')
+

main.py

@@ -43,6 +43,7 @@
 from media_platform.weibo import WeiboCrawler
 from media_platform.xhs import XiaoHongShuCrawler
 from media_platform.zhihu import ZhihuCrawler
+from media_platform.tiktok import TikTokCrawler
 from tools.async_file_writer import AsyncFileWriter
 from var import crawler_type_var
 
@@ -56,6 +57,7 @@ class CrawlerFactory:
         "wb": WeiboCrawler,
         "tieba": TieBaCrawler,
         "zhihu": ZhihuCrawler,
+        "tiktok": TikTokCrawler,
     }
 
     @staticmethod

media_platform/bilibili/core.py

@@ -93,7 +93,7 @@ async def start(self):
                 await self.browser_context.add_init_script(path="libs/stealth.min.js")
 
             self.context_page = await self.browser_context.new_page()
-            await self.context_page.goto(self.index_url)
+            await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
 
             # Create a client to interact with the xiaohongshu website.
             self.bili_client = await self.create_bilibili_client(httpx_proxy_format)

media_platform/douyin/core.py

@@ -95,7 +95,7 @@ async def start(self) -> None:
                 await self.browser_context.add_init_script(path="libs/stealth.min.js")
 
             self.context_page = await self.browser_context.new_page()
-            await self.context_page.goto(self.index_url)
+            await self.context_page.goto(self.index_url, wait_until="domcontentloaded")
 
             self.dy_client = await self.create_douyin_client(httpx_proxy_format)
             if not await self.dy_client.pong(browser_context=self.browser_context):

media_platform/kuaishou/core.py

@@ -94,7 +94,7 @@ async def start(self):
 
 
             self.context_page = await self.browser_context.new_page()
-            await self.context_page.goto(f"{self.index_url}?isHome=1")
+            await self.context_page.goto(f"{self.index_url}?isHome=1", wait_until="domcontentloaded")
 
             # Create a client to interact with the kuaishou website.
             self.ks_client = await self.create_ks_client(httpx_proxy_format)
@@ -296,7 +296,7 @@ async def get_comments(self, video_id: str, semaphore: asyncio.Semaphore):
                 for task in current_running_tasks:
                     task.cancel()
                 time.sleep(20)
-                await self.context_page.goto(f"{self.index_url}?isHome=1")
+                await self.context_page.goto(f"{self.index_url}?isHome=1", wait_until="domcontentloaded")
                 await self.ks_client.update_cookies(
                     browser_context=self.browser_context,
                     urls=self.cookie_urls,

media_platform/tiktok/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+from .core import TikTokCrawler
+