import asyncio import json import re from typing import Any, Callable, Dict, List, Optional, Union from urllib.parse import urlencode import httpx from playwright.async_api import BrowserContext, Page import config from base.base_crawler import AbstractApiClient from tools import utils from .field import SearchNoteType, SearchSortType class BaiduTieBaClient(AbstractApiClient): def __init__( self, timeout=10, proxies=None, *, headers: Dict[str, str], playwright_page: Page, cookie_dict: Dict[str, str], ): self.proxies = proxies self.timeout = timeout self.headers = headers self.playwright_page = playwright_page self.cookie_dict = cookie_dict self._host = "https://tieba.baidu.com" async def request(self, method, url, **kwargs) -> Union[str, Any]: """ 封装httpx的公共请求方法,对请求响应做一些处理 Args: method: 请求方法 url: 请求的URL **kwargs: 其他请求参数,例如请求头、请求体等 Returns: """ # return response.text return_response = kwargs.pop('return_response', False) async with httpx.AsyncClient(proxies=self.proxies) as client: response = await client.request( method, url, timeout=self.timeout, **kwargs ) if return_response: return response.text return response.json() async def get(self, uri: str, params=None) -> Dict: """ GET请求,对请求头签名 Args: uri: 请求路由 params: 请求参数 Returns: """ final_uri = uri if isinstance(params, dict): final_uri = (f"{uri}?" f"{urlencode(params)}") return await self.request(method="GET", url=f"{self._host}{final_uri}", headers=self.headers) async def post(self, uri: str, data: dict) -> Dict: """ POST请求,对请求头签名 Args: uri: 请求路由 data: 请求体参数 Returns: """ json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False) return await self.request(method="POST", url=f"{self._host}{uri}", data=json_str, headers=self.headers) async def pong(self) -> bool: """ 用于检查登录态是否失效了 Returns: """ utils.logger.info("[BaiduTieBaClient.pong] Begin to pong tieba...") try: uri = "/mo/q/sync" res: Dict = await self.get(uri) if res and res.get("no") == 0: ping_flag = True else: utils.logger.info(f"[BaiduTieBaClient.pong] user not login, will try to login again...") ping_flag = False except Exception as e: utils.logger.error(f"[BaiduTieBaClient.pong] Ping tieba failed: {e}, and try to login again...") ping_flag = False return ping_flag async def update_cookies(self, browser_context: BrowserContext): """ API客户端提供的更新cookies方法,一般情况下登录成功后会调用此方法 Args: browser_context: 浏览器上下文对象 Returns: """ cookie_str, cookie_dict = utils.convert_cookies(await browser_context.cookies()) self.headers["Cookie"] = cookie_str self.cookie_dict = cookie_dict async def get_note_by_keyword( self, keyword: str, page: int = 1, page_size: int = 10, sort: SearchSortType = SearchSortType.TIME_DESC, note_type: SearchNoteType = SearchNoteType.FIXED_THREAD ) -> Dict: """ 根据关键词搜索贴吧帖子 Args: keyword: 关键词 page: 分页第几页 page_size: 每页肠病毒 sort: 结果排序方式 note_type: 帖子类型(主题贴|主题+回复混合模式) Returns: """ # todo impl it return {} async def get_note_by_id(self, note_id: str) -> Dict: """ 根据帖子ID获取帖子详情 Args: note_id: Returns: """ # todo impl it return {} async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0, callback: Optional[Callable] = None) -> List[Dict]: """ 获取指定帖子下的所有一级评论,该方法会一直查找一个帖子下的所有评论信息 Args: note_id: 帖子ID crawl_interval: 爬取一次笔记的延迟单位(秒) callback: 一次笔记爬取结束后 Returns: """ # todo impl it return []