169 lines
4.9 KiB
Python
169 lines
4.9 KiB
Python
import asyncio
|
||
import json
|
||
import re
|
||
from typing import Any, Callable, Dict, List, Optional, Union
|
||
from urllib.parse import urlencode
|
||
|
||
import httpx
|
||
from playwright.async_api import BrowserContext, Page
|
||
|
||
import config
|
||
from base.base_crawler import AbstractApiClient
|
||
from tools import utils
|
||
|
||
from .field import SearchNoteType, SearchSortType
|
||
|
||
|
||
class BaiduTieBaClient(AbstractApiClient):
|
||
def __init__(
|
||
self,
|
||
timeout=10,
|
||
proxies=None,
|
||
*,
|
||
headers: Dict[str, str],
|
||
playwright_page: Page,
|
||
cookie_dict: Dict[str, str],
|
||
):
|
||
self.proxies = proxies
|
||
self.timeout = timeout
|
||
self.headers = headers
|
||
self.playwright_page = playwright_page
|
||
self.cookie_dict = cookie_dict
|
||
self._host = "https://tieba.baidu.com"
|
||
|
||
async def request(self, method, url, **kwargs) -> Union[str, Any]:
|
||
"""
|
||
封装httpx的公共请求方法，对请求响应做一些处理
|
||
Args:
|
||
method: 请求方法
|
||
url: 请求的URL
|
||
**kwargs: 其他请求参数，例如请求头、请求体等
|
||
|
||
Returns:
|
||
|
||
"""
|
||
# return response.text
|
||
return_response = kwargs.pop('return_response', False)
|
||
|
||
async with httpx.AsyncClient(proxies=self.proxies) as client:
|
||
response = await client.request(
|
||
method, url, timeout=self.timeout,
|
||
**kwargs
|
||
)
|
||
|
||
if return_response:
|
||
return response.text
|
||
|
||
return response.json()
|
||
|
||
async def get(self, uri: str, params=None) -> Dict:
|
||
"""
|
||
GET请求，对请求头签名
|
||
Args:
|
||
uri: 请求路由
|
||
params: 请求参数
|
||
|
||
Returns:
|
||
|
||
"""
|
||
final_uri = uri
|
||
if isinstance(params, dict):
|
||
final_uri = (f"{uri}?"
|
||
f"{urlencode(params)}")
|
||
return await self.request(method="GET", url=f"{self._host}{final_uri}", headers=self.headers)
|
||
|
||
async def post(self, uri: str, data: dict) -> Dict:
|
||
"""
|
||
POST请求，对请求头签名
|
||
Args:
|
||
uri: 请求路由
|
||
data: 请求体参数
|
||
|
||
Returns:
|
||
|
||
"""
|
||
json_str = json.dumps(data, separators=(',', ':'), ensure_ascii=False)
|
||
return await self.request(method="POST", url=f"{self._host}{uri}",
|
||
data=json_str, headers=self.headers)
|
||
|
||
async def pong(self) -> bool:
|
||
"""
|
||
用于检查登录态是否失效了
|
||
Returns:
|
||
|
||
"""
|
||
utils.logger.info("[BaiduTieBaClient.pong] Begin to pong tieba...")
|
||
try:
|
||
uri = "/mo/q/sync"
|
||
res: Dict = await self.get(uri)
|
||
if res and res.get("no") == 0:
|
||
ping_flag = True
|
||
else:
|
||
utils.logger.info(f"[BaiduTieBaClient.pong] user not login, will try to login again...")
|
||
ping_flag = False
|
||
except Exception as e:
|
||
utils.logger.error(f"[BaiduTieBaClient.pong] Ping tieba failed: {e}, and try to login again...")
|
||
ping_flag = False
|
||
return ping_flag
|
||
|
||
async def update_cookies(self, browser_context: BrowserContext):
|
||
"""
|
||
API客户端提供的更新cookies方法，一般情况下登录成功后会调用此方法
|
||
Args:
|
||
browser_context: 浏览器上下文对象
|
||
|
||
Returns:
|
||
|
||
"""
|
||
cookie_str, cookie_dict = utils.convert_cookies(await browser_context.cookies())
|
||
self.headers["Cookie"] = cookie_str
|
||
self.cookie_dict = cookie_dict
|
||
|
||
async def get_note_by_keyword(
|
||
self, keyword: str,
|
||
page: int = 1,
|
||
page_size: int = 10,
|
||
sort: SearchSortType = SearchSortType.TIME_DESC,
|
||
note_type: SearchNoteType = SearchNoteType.FIXED_THREAD
|
||
) -> Dict:
|
||
"""
|
||
根据关键词搜索贴吧帖子
|
||
Args:
|
||
keyword: 关键词
|
||
page: 分页第几页
|
||
page_size: 每页肠病毒
|
||
sort: 结果排序方式
|
||
note_type: 帖子类型（主题贴｜主题+回复混合模式）
|
||
|
||
Returns:
|
||
|
||
"""
|
||
# todo impl it
|
||
return {}
|
||
|
||
async def get_note_by_id(self, note_id: str) -> Dict:
|
||
"""
|
||
根据帖子ID获取帖子详情
|
||
Args:
|
||
note_id:
|
||
|
||
Returns:
|
||
|
||
"""
|
||
# todo impl it
|
||
return {}
|
||
|
||
async def get_note_all_comments(self, note_id: str, crawl_interval: float = 1.0,
|
||
callback: Optional[Callable] = None) -> List[Dict]:
|
||
"""
|
||
获取指定帖子下的所有一级评论，该方法会一直查找一个帖子下的所有评论信息
|
||
Args:
|
||
note_id: 帖子ID
|
||
crawl_interval: 爬取一次笔记的延迟单位（秒）
|
||
callback: 一次笔记爬取结束后
|
||
|
||
Returns:
|
||
|
||
"""
|
||
# todo impl it
|
||
return [] |