fatsand/xhs_crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 抖音db存储增加parent_comment_id

Browse Source
This commit is contained in:
Relakkes 2024-06-07 13:32:58 +08:00
parent c8dbc0bf3d
commit 3c7c678d7a
3 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/base_config.py
View File

@ -47,7 +47,7 @@ ENABLE_GET_COMMENTS = False
# 是否开启爬二级评论模式, 默认不开启爬二级评论, 目前仅支持 xhs # 是否开启爬二级评论模式, 默认不开启爬二级评论, 目前仅支持 xhs
# 老版本项目使用了 db, 则需参考 schema/tables.sql line 287 增加表字段 # 老版本项目使用了 db, 则需参考 schema/tables.sql line 287 增加表字段
ENABLE_GET_SUB_COMMENTS = False ENABLE_GET_SUB_COMMENTS = True
# 指定小红书需要爬虫的笔记ID列表 # 指定小红书需要爬虫的笔记ID列表
XHS_SPECIFIED_ID_LIST = [ XHS_SPECIFIED_ID_LIST = [

2
schema/tables.sql
View File

@ -308,5 +308,7 @@ CREATE TABLE `xhs_note_comment` (
ALTER TABLE `xhs_note_comment` ALTER TABLE `xhs_note_comment`
ADD COLUMN `parent_comment_id` VARCHAR(64) DEFAULT NULL COMMENT '父评论ID'; ADD COLUMN `parent_comment_id` VARCHAR(64) DEFAULT NULL COMMENT '父评论ID';
ALTER TABLE `douyin_aweme_comment`
ADD COLUMN `parent_comment_id` VARCHAR(64) DEFAULT NULL COMMENT '父评论ID';
SET FOREIGN_KEY_CHECKS = 1; SET FOREIGN_KEY_CHECKS = 1;

2
store/douyin/__init__.py
View File

@ -70,6 +70,7 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
return return
user_info = comment_item.get("user", {}) user_info = comment_item.get("user", {})
comment_id = comment_item.get("cid") comment_id = comment_item.get("cid")
parent_comment_id = comment_item.get("reply_id", "0")
avatar_info = user_info.get("avatar_medium", {}) or user_info.get("avatar_300x300", {}) or user_info.get( avatar_info = user_info.get("avatar_medium", {}) or user_info.get("avatar_300x300", {}) or user_info.get(
"avatar_168x168", {}) or user_info.get("avatar_thumb", {}) or {} "avatar_168x168", {}) or user_info.get("avatar_thumb", {}) or {}
save_comment_item = { save_comment_item = {
@ -87,6 +88,7 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict):
"avatar": avatar_info.get("url_list", [""])[0], "avatar": avatar_info.get("url_list", [""])[0],
"sub_comment_count": str(comment_item.get("reply_comment_total", 0)), "sub_comment_count": str(comment_item.get("reply_comment_total", 0)),
"last_modify_ts": utils.get_current_timestamp(), "last_modify_ts": utils.get_current_timestamp(),
"parent_comment_id": parent_comment_id
} }
utils.logger.info( utils.logger.info(
f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}") f"[store.douyin.update_dy_aweme_comment] douyin aweme comment: {comment_id}, content: {save_comment_item.get('content')}")