diff --git a/media_platform/douyin/client.py b/media_platform/douyin/client.py index 8e3230d..23f4a06 100644 --- a/media_platform/douyin/client.py +++ b/media_platform/douyin/client.py @@ -125,6 +125,7 @@ class DOUYINClient: "count": 10 # must be set to 10 } referer_url = "https://www.douyin.com/search/" + keyword + referer_url += f"?publish_time={publish_time.value}&sort_type={sort_type.value}&type=general" headers = copy.copy(self.headers) headers["Referer"] = urllib.parse.quote(referer_url, safe=':/') return await self.get("/aweme/v1/web/general/search/single/", params, headers=headers) diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index 68a5e72..0787520 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -17,6 +17,7 @@ from var import crawler_type_var from .client import DOUYINClient from .exception import DataFetchError from .login import DouYinLogin +from .field import PublishTimeType class DouYinCrawler(AbstractCrawler): @@ -88,7 +89,9 @@ class DouYinCrawler(AbstractCrawler): while (page + 1) * dy_limit_count <= config.CRAWLER_MAX_NOTES_COUNT: try: posts_res = await self.dy_client.search_info_by_keyword(keyword=keyword, - offset=page * dy_limit_count) + offset=page * dy_limit_count, + publish_time=PublishTimeType.UNLIMITED + ) except DataFetchError: utils.logger.error(f"[DouYinCrawler.search] search douyin keyword: {keyword} failed") break