fatsand/xhs_crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: #107

Browse Source
This commit is contained in:
Relakkes 2024-01-03 23:08:30 +08:00
parent 38d6f10bf0
commit 80ecc53cd3
2 changed files with 3 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
media_platform/bilibili/core.py
View File

@ -147,32 +147,12 @@ class BilibiliCrawler(AbstractCrawler):
async with semaphore: async with semaphore:
try: try:
utils.logger.info(f"[BilibiliCrawler.get_comments] begin get video_id: {video_id} comments ...") utils.logger.info(f"[BilibiliCrawler.get_comments] begin get video_id: {video_id} comments ...")
# Read keyword and quantity from config await self.bili_client.get_video_all_comments(
keywords = config.COMMENT_KEYWORDS
max_comments = config.MAX_COMMENTS_PER_POST
# Download comments
all_comments = await self.bili_client.get_video_all_comments(
video_id=video_id, video_id=video_id,
crawl_interval=random.random(), crawl_interval=random.random(),
callback=bilibili.batch_update_bilibili_video_comments
) )
# Filter comments by keyword
if keywords:
filtered_comments = [
comment for comment in all_comments if
any(keyword in comment["content"]["message"] for keyword in keywords)
]
else:
filtered_comments = all_comments
# Limit the number of comments
if max_comments > 0:
filtered_comments = filtered_comments[:max_comments]
# Update bilibili video comments
await bilibili.batch_update_bilibili_video_comments(video_id, filtered_comments)
except DataFetchError as ex: except DataFetchError as ex:
utils.logger.error(f"[BilibiliCrawler.get_comments] get video_id: {video_id} comment error: {ex}") utils.logger.error(f"[BilibiliCrawler.get_comments] get video_id: {video_id} comment error: {ex}")
except Exception as e: except Exception as e:

2
models/bilibili.py
View File

@ -123,7 +123,7 @@ async def update_bilibili_video_comment(video_id: str, comment_item: Dict):
local_db_item = { local_db_item = {
"comment_id": comment_id, "comment_id": comment_id,
"create_time": comment_item.get("ctime"), "create_time": comment_item.get("ctime"),
"video_id": video_id, "video_id": str(video_id),
"content": content.get("message"), "content": content.get("message"),
"user_id": user_info.get("mid"), "user_id": user_info.get("mid"),
"nickname": user_info.get("uname"), "nickname": user_info.get("uname"),