fatsand/xhs_crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复部分变量命名语义不明确

Browse Source
This commit is contained in:
Relakkes 2023-07-30 21:30:26 +08:00
parent bf659455bb
commit 1d3c2359e0
5 changed files with 10 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config/base_config.py
View File

@ -1,8 +1,8 @@
# Desc: base config # Desc: base config
PLATFORM = "xhs" PLATFORM = "xhs"
KEYWORDS = "健身,旅游" KEYWORDS = "健身,旅游"
LOGIN_TYPE = "qrcode" # qrcode or phone or cookies LOGIN_TYPE = "qrcode" # qrcode or phone or cookie
COOKIES = "web_session=xxxxcfed1566xxxxxxxxxxxxxxxxxxx;" # if platform is xhs, pleas set only web_session cookie attr COOKIES = "" # login by cookie, if login_type is cookie, you must set this value
# enable ip proxy # enable ip proxy
ENABLE_IP_PROXY = False ENABLE_IP_PROXY = False

4
media_platform/douyin/login.py
View File

@ -44,10 +44,10 @@ class DouYinLogin(AbstractLogin):
await self.login_by_qrcode() await self.login_by_qrcode()
elif self.login_type == "phone": elif self.login_type == "phone":
await self.login_by_mobile() await self.login_by_mobile()
elif self.login_type == "cookies": elif self.login_type == "cookie":
await self.login_by_cookies() await self.login_by_cookies()
else: else:
raise ValueError("Invalid Login Type Currently only supported qrcode or phone ...") raise ValueError("Invalid Login Type Currently only supported qrcode or phone or cookie ...")
# 如果页面重定向到滑动验证码页面，需要再次滑动滑块 # 如果页面重定向到滑动验证码页面，需要再次滑动滑块
await asyncio.sleep(6) await asyncio.sleep(6)

4
media_platform/xhs/client.py
View File

@ -82,14 +82,14 @@ class XHSClient:
async def ping(self) -> bool: async def ping(self) -> bool:
"""get a note to check if login state is ok""" """get a note to check if login state is ok"""
utils.logger.info("begin to ping xhs...") utils.logger.info("Begin to ping xhs...")
ping_flag = False ping_flag = False
try: try:
note_card: Dict = await self.get_note_by_keyword(keyword="小红书") note_card: Dict = await self.get_note_by_keyword(keyword="小红书")
if note_card.get("items"): if note_card.get("items"):
ping_flag = True ping_flag = True
except Exception as e: except Exception as e:
utils.logger.error(f"ping xhs failed: {e}") utils.logger.error(f"Ping xhs failed: {e}, and try to login again...")
ping_flag = False ping_flag = False
return ping_flag return ping_flag

2
media_platform/xhs/core.py
View File

@ -81,7 +81,7 @@ class XiaoHongShuCrawler(AbstractCrawler):
utils.logger.info("Begin search xiaohongshu keywords") utils.logger.info("Begin search xiaohongshu keywords")
xhs_limit_count = 20 # xhs limit page fixed value xhs_limit_count = 20 # xhs limit page fixed value
for keyword in config.KEYWORDS.split(","): for keyword in config.KEYWORDS.split(","):
utils.logger.info(f"Current keyword: {keyword}") utils.logger.info(f"Current search keyword: {keyword}")
page = 1 page = 1
while page * xhs_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: while page * xhs_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
note_id_list: List[str] = [] note_id_list: List[str] = []

4
media_platform/xhs/login.py
View File

@ -49,7 +49,7 @@ class XHSLogin(AbstractLogin):
await self.login_by_qrcode() await self.login_by_qrcode()
elif self.login_type == "phone": elif self.login_type == "phone":
await self.login_by_mobile() await self.login_by_mobile()
elif self.login_type == "cookies": elif self.login_type == "cookie":
await self.login_by_cookies() await self.login_by_cookies()
else: else:
raise ValueError("Invalid Login Type Currently only supported qrcode or phone or cookies ...") raise ValueError("Invalid Login Type Currently only supported qrcode or phone or cookies ...")
@ -172,6 +172,8 @@ class XHSLogin(AbstractLogin):
"""login xiaohongshu website by cookies""" """login xiaohongshu website by cookies"""
utils.logger.info("Begin login xiaohongshu by cookie ...") utils.logger.info("Begin login xiaohongshu by cookie ...")
for key, value in utils.convert_str_cookie_to_dict(self.cookie_str).items(): for key, value in utils.convert_str_cookie_to_dict(self.cookie_str).items():
if key != "web_session": # only set web_session cookie attr
continue
await self.browser_context.add_cookies([{ await self.browser_context.add_cookies([{
'name': key, 'name': key,
'value': value, 'value': value,