feat: 抖音指定创作者done

README.md

@@ -18,9 +18,9 @@
 > 下面不支持的项目，相关的代码架构已经搭建好，只需要实现对应的方法即可，欢迎大家提交PR
 
 | 平台  | 关键词搜索 | 指定帖子ID爬取 | 二级评论 | 指定创作者主页 | 登录态缓存 | IP代理池 |
-|-----|-------|----------|------|---------|-------|-------|
+|-----|-------|----------|------|--------|-------|-------|
 | 小红书 | ✅     | ✅        | ✅    | ✅      | ✅     | ✅     |
-| 抖音  | ✅     | ✅        | ❌    | ❌       | ✅     | ✅     |
+| 抖音  | ✅     | ✅        | ❌    | ✅       | ✅     | ✅     |
 | 快手  | ✅     | ✅        | ❌    | ❌      | ✅     | ✅     |
 | B 站 | ✅     | ✅        | ✅    | ❌      | ✅     | ✅     |
 | 微博  | ✅     | ✅        | ❌    | ❌      | ✅     | ✅     |
@@ -84,7 +84,7 @@
 
 ## 开发者服务
 - 付费咨询：提供 200 元/小时的咨询服务，帮你快速解决项目中遇到的问题。
-- 知识星球：沉淀高质量常见问题、最佳实践文档，提供付费知识星球服务，主动提问，作者会定期回答问题
+- 知识星球：沉淀高质量常见问题、最佳实践文档、多年编程+爬虫经验分享，提供付费知识星球服务，主动提问，作者会定期回答问题
   <p><img alt="知识星球" src="static/images/xingqiu.jpeg" style="width: 100%;" ></p>
 - 课程服务：
   > 如果你想很快入门这个项目，或者想了具体实现原理，我推荐你看看这个课程，从设计出发一步步带你如何使用，门槛大大降低，同时也是对我开源的支持，如果你能支持我的课程，我将会非常开心～<br>

config/base_config.py

@@ -89,3 +89,9 @@ XHS_CREATOR_ID_LIST = [
     "63e36c9a000000002703502b",
     # ........................
 ]
+
+# 指定Dy创作者ID列表(sec_id)
+DY_CREATOR_ID_LIST = [
+    "MS4wLjABAAAATJPY7LAlaa5X-c8uNdWkvz0jUGgpw4eeXIwu_8BhvqE",
+    # ........................
+]

media_platform/douyin/client.py

@@ -199,3 +199,39 @@ class DOUYINClient(AbstractApiClient):
                 continue
             # todo fetch sub comments
         return result
+
+    async def get_user_info(self, sec_user_id: str):
+        uri = "/aweme/v1/web/user/profile/other/"
+        params = {
+            "sec_user_id": sec_user_id,
+            "publish_video_strategy_type": 2,
+            "personal_center_strategy": 1,
+        }
+        return await self.get(uri, params)
+
+    async def get_user_aweme_posts(self, sec_user_id: str, max_cursor: str = "") -> Dict:
+        uri = "/aweme/v1/web/aweme/post/"
+        params = {
+            "sec_user_id": sec_user_id,
+            "count": 18,
+            "max_cursor": max_cursor,
+            "locate_query": "false",
+            "publish_video_strategy_type": 2
+        }
+        return await self.get(uri, params)
+
+    async def get_all_user_aweme_posts(self, sec_user_id: str, callback: Optional[Callable] = None):
+        posts_has_more = 1
+        max_cursor = ""
+        result = []
+        while posts_has_more == 1:
+            aweme_post_res = await self.get_user_aweme_posts(sec_user_id, max_cursor)
+            posts_has_more = aweme_post_res.get("has_more", 0)
+            max_cursor = aweme_post_res.get("max_cursor")
+            aweme_list = aweme_post_res.get("aweme_list") if aweme_post_res.get("aweme_list") else []
+            utils.logger.info(
+                f"[DOUYINClient.get_all_user_aweme_posts] got sec_user_id:{sec_user_id} video len : {len(aweme_list)}")
+            if callback:
+                await callback(aweme_list)
+            result.extend(aweme_list)
+        return result

media_platform/douyin/core.py

@@ -27,8 +27,11 @@ class DouYinCrawler(AbstractCrawler):
     context_page: Page
     dy_client: DOUYINClient
     browser_context: BrowserContext
+    start_page: int
+    keyword: str
 
     def __init__(self) -> None:
+        self.start_page = None
         self.user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36"  # fixed
         self.index_url = "https://www.douyin.com"
 
@@ -78,6 +81,9 @@ class DouYinCrawler(AbstractCrawler):
             elif self.crawler_type == "detail":
                 # Get the information and comments of the specified post
                 await self.get_specified_awemes()
+            elif self.crawler_type == "creator":
+                # Get the information and comments of the specified creator
+                await self.get_creators_and_videos()
 
             utils.logger.info("[DouYinCrawler.start] Douyin Crawler finished ...")
 
@@ -107,7 +113,8 @@ class DouYinCrawler(AbstractCrawler):
 
                 page += 1
                 if "data" not in posts_res:
-                    utils.logger.error(f"[DouYinCrawler.search] search douyin keyword: {keyword} failed，账号也许被风控了。")
+                    utils.logger.error(
+                        f"[DouYinCrawler.search] search douyin keyword: {keyword} failed，账号也许被风控了。")
                     break
 
                 for post_item in posts_res.get("data"):
@@ -142,10 +149,14 @@ class DouYinCrawler(AbstractCrawler):
                 utils.logger.error(f"[DouYinCrawler.get_aweme_detail] Get aweme detail error: {ex}")
                 return None
             except KeyError as ex:
-                utils.logger.error(f"[DouYinCrawler.get_aweme_detail] have not fund note detail aweme_id:{aweme_id}, err: {ex}")
+                utils.logger.error(
+                    f"[DouYinCrawler.get_aweme_detail] have not fund note detail aweme_id:{aweme_id}, err: {ex}")
                 return None
 
     async def batch_get_note_comments(self, aweme_list: List[str]) -> None:
+        """
+        Batch get note comments
+        """
         if not config.ENABLE_GET_COMMENTS:
             utils.logger.info(f"[DouYinCrawler.batch_get_note_comments] Crawling comment mode is not enabled")
             return
@@ -156,7 +167,7 @@ class DouYinCrawler(AbstractCrawler):
             task = asyncio.create_task(
                 self.get_comments(aweme_id, semaphore), name=aweme_id)
             task_list.append(task)
-        if len(task_list) > 0 :
+        if len(task_list) > 0:
             await asyncio.wait(task_list)
 
     async def get_comments(self, aweme_id: str, semaphore: asyncio.Semaphore) -> None:
@@ -169,10 +180,44 @@ class DouYinCrawler(AbstractCrawler):
                     callback=douyin_store.batch_update_dy_aweme_comments
 
                 )
-                utils.logger.info(f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} comments have all been obtained and filtered ...")
+                utils.logger.info(
+                    f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} comments have all been obtained and filtered ...")
             except DataFetchError as e:
                 utils.logger.error(f"[DouYinCrawler.get_comments] aweme_id: {aweme_id} get comments failed, error: {e}")
 
+    async def get_creators_and_videos(self) -> None:
+        """
+        Get the information and videos of the specified creator
+        """
+        utils.logger.info("[DouYinCrawler.get_creators_and_videos] Begin get douyin creators")
+        for user_id in config.DY_CREATOR_ID_LIST:
+            creator_info: Dict = await self.dy_client.get_user_info(user_id)
+            if creator_info:
+                await douyin_store.save_creator(user_id, creator=creator_info)
+
+            # Get all video information of the creator
+            all_video_list = await self.dy_client.get_all_user_aweme_posts(
+                sec_user_id=user_id,
+                callback=self.fetch_creator_video_detail
+            )
+
+            video_ids = [video_item.get("aweme_id") for video_item in all_video_list]
+            await self.batch_get_note_comments(video_ids)
+
+    async def fetch_creator_video_detail(self, video_list: List[Dict]):
+        """
+        Concurrently obtain the specified post list and save the data
+        """
+        semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM)
+        task_list = [
+            self.get_aweme_detail(post_item.get("aweme_id"), semaphore) for post_item in video_list
+        ]
+
+        note_details = await asyncio.gather(*task_list)
+        for aweme_item in note_details:
+            if aweme_item is not None:
+                await douyin_store.update_douyin_aweme(aweme_item)
+
     @staticmethod
     def format_proxy_info(ip_proxy_info: IpInfoModel) -> Tuple[Optional[Dict], Optional[Dict]]:
         """format proxy info for playwright and httpx"""

schema/tables.sql

@@ -103,6 +103,27 @@ CREATE TABLE `douyin_aweme_comment` (
   KEY `idx_douyin_awem_aweme_i_c50049` (`aweme_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='抖音视频评论';
 
+-- ----------------------------
+-- Table structure for dy_creator
+-- ----------------------------
+DROP TABLE IF EXISTS `dy_creator`;
+CREATE TABLE `dy_creator` (
+  `id` int NOT NULL AUTO_INCREMENT COMMENT '自增ID',
+  `user_id` varchar(128) NOT NULL COMMENT '用户ID',
+  `nickname` varchar(64) DEFAULT NULL COMMENT '用户昵称',
+  `avatar` varchar(255) DEFAULT NULL COMMENT '用户头像地址',
+  `ip_location` varchar(255) DEFAULT NULL COMMENT '评论时的IP地址',
+  `add_ts` bigint NOT NULL COMMENT '记录添加时间戳',
+  `last_modify_ts` bigint NOT NULL COMMENT '记录最后修改时间戳',
+  `desc` longtext COMMENT '用户描述',
+  `gender` varchar(1) DEFAULT NULL COMMENT '性别',
+  `follows` varchar(16) DEFAULT NULL COMMENT '关注数',
+  `fans` varchar(16) DEFAULT NULL COMMENT '粉丝数',
+  `interaction` varchar(16) DEFAULT NULL COMMENT '获赞数',
+  `videos_count` varchar(16) DEFAULT NULL COMMENT '作品数',
+  PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='抖音博主信息';
+
 -- ----------------------------
 -- Table structure for kuaishou_video
 -- ----------------------------

store/douyin/__init__.py

@@ -92,3 +92,31 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
         f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}")
 
     await DouyinStoreFactory.create_store().store_comment(comment_item=save_comment_item)
+
+
+
+
+async def save_creator(user_id: str, creator: Dict):
+    user_info = creator.get('user', {})
+    gender_map = {
+        0: '未知',
+        1: '男',
+        2: '女'
+    }
+    avatar_uri = user_info.get('avatar_300x300', {}).get('uri')
+    local_db_item = {
+        'user_id': user_id,
+        'nickname': user_info.get('nickname'),
+        'gender': gender_map.get(user_info.get('gender'), '未知'),
+        'avatar': f"https://p3-pc.douyinpic.com/img/{avatar_uri}" + r"~c5_300x300.jpeg?from=2956013662",
+        'desc': user_info.get('signature'),
+        'ip_location': user_info.get('ip_location'),
+        'follows': user_info.get("following_count", 0),
+        'fans': user_info.get("max_follower_count", 0),
+        'interaction': user_info.get("total_favorited", 0),
+        'videos_count': user_info.get("aweme_count", 0),
+        "last_modify_ts": utils.get_current_timestamp(),
+
+    }
+    utils.logger.info(f"[store.douyin.save_creator] creator:{local_db_item}")
+    await DouyinStoreFactory.create_store().store_creator(local_db_item)

store/douyin/douyin_store_impl.py

@@ -26,13 +26,14 @@ def calculate_number_of_files(file_store_path: str) -> int:
     if not os.path.exists(file_store_path):
         return 1
     try:
-        return max([int(file_name.split("_")[0])for file_name in os.listdir(file_store_path)])+1
+        return max([int(file_name.split("_")[0]) for file_name in os.listdir(file_store_path)]) + 1
     except ValueError:
         return 1
 
+
 class DouyinCsvStoreImplement(AbstractStore):
     csv_store_path: str = "data/douyin"
-    file_count:int=calculate_number_of_files(csv_store_path)
+    file_count: int = calculate_number_of_files(csv_store_path)
 
     def make_save_file_name(self, store_type: str) -> str:
         """
@@ -65,7 +66,7 @@ class DouyinCsvStoreImplement(AbstractStore):
 
     async def store_content(self, content_item: Dict):
         """
-        Xiaohongshu content CSV storage implementation
+        Douyin content CSV storage implementation
         Args:
             content_item: note item dict
 
@@ -76,7 +77,7 @@ class DouyinCsvStoreImplement(AbstractStore):
 
     async def store_comment(self, comment_item: Dict):
         """
-        Xiaohongshu comment CSV storage implementation
+        Douyin comment CSV storage implementation
         Args:
             comment_item: comment item dict
 
@@ -85,6 +86,17 @@ class DouyinCsvStoreImplement(AbstractStore):
         """
         await self.save_data_to_csv(save_item=comment_item, store_type="comments")
 
+    async def store_creator(self, creator: Dict):
+        """
+        Douyin creator CSV storage implementation
+        Args:
+            creator: creator item dict
+
+        Returns:
+
+        """
+        await self.save_data_to_csv(save_item=creator, store_type="creator")
+
 
 class DouyinDbStoreImplement(AbstractStore):
     async def store_content(self, content_item: Dict):
@@ -109,7 +121,6 @@ class DouyinDbStoreImplement(AbstractStore):
         else:
             await update_content_by_content_id(aweme_id, content_item=content_item)
 
-
     async def store_comment(self, comment_item: Dict):
         """
         Douyin content DB storage implementation
@@ -130,11 +141,29 @@ class DouyinDbStoreImplement(AbstractStore):
         else:
             await update_comment_by_comment_id(comment_id, comment_item=comment_item)
 
+    async def store_creator(self, creator: Dict):
+        """
+        Douyin content DB storage implementation
+        Args:
+            creator: creator dict
+
+        Returns:
+
+        """
+        from .douyin_store_sql import (add_new_creator, query_creator_by_user_id,
+                                    update_creator_by_user_id)
+        user_id = creator.get("user_id")
+        user_detail: Dict = await query_creator_by_user_id(user_id)
+        if not user_detail:
+            creator["add_ts"] = utils.get_current_timestamp()
+            await add_new_creator(creator)
+        else:
+            await update_creator_by_user_id(user_id, creator)
 
 class DouyinJsonStoreImplement(AbstractStore):
     json_store_path: str = "data/douyin"
     lock = asyncio.Lock()
-    file_count:int=calculate_number_of_files(json_store_path)
+    file_count: int = calculate_number_of_files(json_store_path)
 
     def make_save_file_name(self, store_type: str) -> str:
         """
@@ -146,7 +175,6 @@ class DouyinJsonStoreImplement(AbstractStore):
 
         """
 
-
         return f"{self.json_store_path}/{self.file_count}_{crawler_type_var.get()}_{store_type}_{utils.get_current_date()}.json"
 
     async def save_data_to_json(self, save_item: Dict, store_type: str):
@@ -193,3 +221,15 @@ class DouyinJsonStoreImplement(AbstractStore):
 
         """
         await self.save_data_to_json(comment_item, "comments")
+
+
+    async def store_creator(self, creator: Dict):
+        """
+        Douyin creator CSV storage implementation
+        Args:
+            creator: creator item dict
+
+        Returns:
+
+        """
+        await self.save_data_to_json(save_item=creator, store_type="creator")

store/douyin/douyin_store_sql.py

@@ -100,3 +100,49 @@ async def update_comment_by_comment_id(comment_id: str, comment_item: Dict) -> i
     async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
     effect_row: int = await async_db_conn.update_table("douyin_aweme_comment", comment_item, "comment_id", comment_id)
     return effect_row
+
+
+async def query_creator_by_user_id(user_id: str) -> Dict:
+    """
+    查询一条创作者记录
+    Args:
+        user_id:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    sql: str = f"select * from dy_creator where user_id = '{user_id}'"
+    rows: List[Dict] = await async_db_conn.query(sql)
+    if len(rows) > 0:
+        return rows[0]
+    return dict()
+
+
+async def add_new_creator(creator_item: Dict) -> int:
+    """
+    新增一条创作者信息
+    Args:
+        creator_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    last_row_id: int = await async_db_conn.item_to_table("dy_creator", creator_item)
+    return last_row_id
+
+
+async def update_creator_by_user_id(user_id: str, creator_item: Dict) -> int:
+    """
+    更新一条创作者信息
+    Args:
+        user_id:
+        creator_item:
+
+    Returns:
+
+    """
+    async_db_conn: AsyncMysqlDB = media_crawler_db_var.get()
+    effect_row: int = await async_db_conn.update_table("dy_creator", creator_item, "user_id", user_id)
+    return effect_row