Archived
snowy-yang
65adfc9089
core/aiohttp_compat 是「同步转异步」迁移期的 requests 同形异步垫片：它存在的 唯一理由是让既有 `requests.post/get + raise_for_status()` 风格的代码以最小 改动迁到异步。其两个调用方（music_api/api.py、music_api/client.py）**本就已是 async def**——垫片只剩「保留 requests 语义」这一层包装，属于迁移残留。 改动： - api.py：23 处 requests.post/get → await core.http_client.request()。 该文件每处都在请求后立刻 raise_for_status() 才取 .json()，从不把 4xx 当数据读， 故「_once 对非 2xx 直接抛 UpstreamHTTPError」与旧语义等价（原 raise_for_status 抛的 HTTPError 也一律落入 RequestException 分支）； except requests.RequestException → except UpstreamError。 - client.py：两个自管重试的方法改用新公开的 core.http_client.request_once() （单次、不重试），保留原有「5xx 退避重试一次 / 4xx 立即失败」的分流—— 把 status_code>=500 的判断换成 UpstreamHTTPError.status，行为逐条对齐 （4xx=风控不重试，网络/5xx 重试一次）。响应体上限交由客户端 max_bytes 统一把关。 post_request_full 返回类型改为 HTTPResult（qr.py 用到的 .text/.cookies/.headers 三者在 HTTPResult 上齐备）。 - core/http_client：_once 更名并公开为 request_once（自管重试的调用方需要它）。 注：这是为「删垫片」而补的唯一一个小口子，不是新增抽象。 注意一处**有意的行为变化**：api.py 原经垫片请求零重试，现在 request() 自带 「网络异常/5xx 退避重试一次、4xx 不重试」（core.http_client 既定口径）。 429/460 属 4xx 不重试，不加剧风控。 - 删除 core/aiohttp_compat.py；web/app.py 去掉其 close()；pyproject 去掉其 per-file-ignore；core/http_errors.py 注释更正为「仅供 core.sync_http」。 - 测试：test_aiohttp_compat.py 拆解为 test_async_bridge.py（只留桥的两条契约， 垫片自身的 4 条随垫片消亡）；test_search_music / test_playlist_detail 的替身 边界从 api.requests.post 换成 api._http.request，异常类型换成 UpstreamNetworkError。 门禁：ruff 全过 / pyright 0-0 / pytest 383 项全绿（0 失败；383=387-6+2，账面吻合）。
294 lines
11 KiB
Python
294 lines
11 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""异步 HTTP 客户端（aiohttp）：连接池 + 有界重试 + 响应体上限。
|
||
|
||
替代旧的长连接 `requests.Session`/裸 `requests.get/post`：
|
||
- 单一 `ClientSession` 复用 TCP 连接（keep-alive），连接池限制防上游打爆；
|
||
- 重试口径与旧实现一致：网络异常/5xx 退避重试一次，4xx（429/460 风控）**不**重试；
|
||
- 响应体超过 `max_bytes` 直接拒绝（防止被劫持/异常上游的超大 body 进内存）。
|
||
|
||
异常层次（调用方按需映射为业务异常，例如 music_api.APIException）：
|
||
UpstreamError
|
||
├── UpstreamNetworkError 网络/超时（可重试）
|
||
├── UpstreamHTTPError 非 2xx（5xx 可重试，4xx 不可）
|
||
└── UpstreamTooLarge 响应体超限
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
from collections.abc import AsyncGenerator, Mapping
|
||
from contextlib import asynccontextmanager
|
||
from dataclasses import dataclass, field
|
||
from typing import Any
|
||
|
||
import aiohttp
|
||
|
||
DEFAULT_TIMEOUT = 30
|
||
DEFAULT_MAX_BYTES = 8 * 1024 * 1024
|
||
|
||
# UA 与 Referer 由调用方传入（music_api.APIConstants）
|
||
_EMPTY: Mapping[str, str] = {}
|
||
|
||
|
||
class UpstreamError(Exception):
|
||
"""上游请求错误基类。"""
|
||
|
||
|
||
class UpstreamNetworkError(UpstreamError):
|
||
"""网络异常/超时（可重试）。"""
|
||
|
||
|
||
class UpstreamHTTPError(UpstreamError):
|
||
"""上游返回非 2xx 状态码。"""
|
||
|
||
def __init__(self, status: int, message: str = ''):
|
||
super().__init__(f"HTTP {status} {message}".strip())
|
||
self.status = status
|
||
|
||
|
||
class UpstreamTooLarge(UpstreamError):
|
||
"""响应体超过大小上限。"""
|
||
|
||
|
||
@dataclass
|
||
class HTTPResult:
|
||
"""一次请求的完整结果（供需要状态码/头/cookie 的调用点使用）。"""
|
||
|
||
status: int
|
||
url: str
|
||
text: str
|
||
headers: Mapping[str, str] = field(default_factory=dict)
|
||
cookies: Mapping[str, str] = field(default_factory=dict)
|
||
|
||
def json(self) -> Any:
|
||
return json.loads(self.text)
|
||
|
||
|
||
class AsyncHTTPClient:
|
||
"""带连接池的异步 HTTP 客户端。"""
|
||
|
||
def __init__(
|
||
self,
|
||
*,
|
||
total_timeout: float = DEFAULT_TIMEOUT,
|
||
max_bytes: int = DEFAULT_MAX_BYTES,
|
||
pool_limit: int = 100,
|
||
pool_per_host: int = 32,
|
||
user_agent: str | None = None,
|
||
referer: str | None = None,
|
||
):
|
||
self._total_timeout = total_timeout
|
||
self._max_bytes = max_bytes
|
||
self._pool_limit = pool_limit
|
||
self._pool_per_host = pool_per_host
|
||
self._default_headers: dict[str, str] = {}
|
||
if user_agent:
|
||
self._default_headers['User-Agent'] = user_agent
|
||
if referer:
|
||
self._default_headers['Referer'] = referer
|
||
self._session: aiohttp.ClientSession | None = None
|
||
self._loop_sessions: dict[int, aiohttp.ClientSession] = {}
|
||
self._lock = asyncio.Lock()
|
||
|
||
async def _session_get(self) -> aiohttp.ClientSession:
|
||
if self._session is not None and not self._session.closed:
|
||
return self._session
|
||
async with self._lock:
|
||
if self._session is None or self._session.closed:
|
||
connector = aiohttp.TCPConnector(
|
||
limit=self._pool_limit,
|
||
limit_per_host=self._pool_per_host,
|
||
ttl_dns_cache=300,
|
||
)
|
||
self._session = aiohttp.ClientSession(
|
||
connector=connector,
|
||
timeout=aiohttp.ClientTimeout(total=self._total_timeout),
|
||
headers=dict(self._default_headers),
|
||
)
|
||
return self._session
|
||
|
||
async def _session_for_loop(self) -> aiohttp.ClientSession:
|
||
"""按事件循环隔离 session。
|
||
|
||
同一进程内可能有两个 loop（应用主 loop + 同步桥的后台 loop）。
|
||
aiohttp 的 ClientSession 不能跨 loop 复用，故按 loop 分别持有。
|
||
"""
|
||
key = id(asyncio.get_running_loop())
|
||
sess = self._loop_sessions.get(key)
|
||
if sess is not None and not sess.closed:
|
||
return sess
|
||
async with self._lock:
|
||
sess = self._loop_sessions.get(key)
|
||
if sess is None or sess.closed:
|
||
connector = aiohttp.TCPConnector(
|
||
limit=self._pool_limit,
|
||
limit_per_host=self._pool_per_host,
|
||
ttl_dns_cache=300,
|
||
)
|
||
sess = aiohttp.ClientSession(
|
||
connector=connector,
|
||
timeout=aiohttp.ClientTimeout(total=self._total_timeout),
|
||
headers=dict(self._default_headers),
|
||
)
|
||
self._loop_sessions[key] = sess
|
||
return sess
|
||
|
||
async def close(self) -> None:
|
||
if self._session is not None and not self._session.closed:
|
||
await self._session.close()
|
||
self._session = None
|
||
for sess in self._loop_sessions.values():
|
||
if not sess.closed:
|
||
await sess.close()
|
||
self._loop_sessions.clear()
|
||
|
||
# ---------- 底层：单次请求（不重试） ----------
|
||
|
||
async def request_once(
|
||
self,
|
||
method: str,
|
||
url: str,
|
||
*,
|
||
headers: Mapping[str, str] | None = None,
|
||
cookies: Mapping[str, str] | None = None,
|
||
params: Mapping[str, Any] | None = None,
|
||
data: Any = None,
|
||
json_body: Any = None,
|
||
timeout: float | None = None,
|
||
allow_redirects: bool = True,
|
||
) -> HTTPResult:
|
||
"""发起一次请求，**不做重试**（自管重试策略的调用方用这个）。
|
||
|
||
非 2xx 一律抛 UpstreamHTTPError（含 5xx）——调用方按 .status 自行决定是否重试。
|
||
request() 在此基础上叠加「网络异常/5xx 退避重试一次」。
|
||
"""
|
||
session = await self._session_for_loop()
|
||
req_timeout = aiohttp.ClientTimeout(total=timeout) if timeout else None
|
||
try:
|
||
async with session.request(
|
||
method,
|
||
url,
|
||
headers=dict(headers) if headers else None,
|
||
cookies=dict(cookies) if cookies else None,
|
||
params=params,
|
||
data=data,
|
||
json=json_body,
|
||
timeout=req_timeout,
|
||
allow_redirects=allow_redirects,
|
||
) as resp:
|
||
body = await resp.read()
|
||
if len(body) > self._max_bytes:
|
||
raise UpstreamTooLarge(f"上游响应体超过上限 {self._max_bytes} 字节")
|
||
if resp.status >= 400:
|
||
raise UpstreamHTTPError(resp.status, resp.reason or '')
|
||
return HTTPResult(
|
||
status=resp.status,
|
||
url=str(resp.url),
|
||
text=body.decode(resp.charset or 'utf-8', errors='replace'),
|
||
headers=dict(resp.headers),
|
||
cookies={k: v.value for k, v in resp.cookies.items()},
|
||
)
|
||
except aiohttp.ClientError as e:
|
||
raise UpstreamNetworkError(str(e)) from e
|
||
except TimeoutError as e:
|
||
raise UpstreamNetworkError('请求超时') from e
|
||
|
||
async def _with_retry(self, fn, *, attempts: int = 2, backoff: float = 1.0) -> HTTPResult:
|
||
"""网络异常/5xx 退避重试；4xx 与超限立即抛出不重试。"""
|
||
last: UpstreamError | None = None
|
||
for i in range(attempts):
|
||
try:
|
||
return await fn()
|
||
except UpstreamHTTPError as e:
|
||
if e.status >= 500 and i + 1 < attempts:
|
||
last = e
|
||
await asyncio.sleep(backoff)
|
||
continue
|
||
raise
|
||
except UpstreamNetworkError as e:
|
||
last = e
|
||
if i + 1 < attempts:
|
||
await asyncio.sleep(backoff)
|
||
continue
|
||
raise
|
||
# 理论上不可达
|
||
raise last or UpstreamError('上游请求失败')
|
||
|
||
# ---------- 高层 API ----------
|
||
|
||
async def request(self, method: str, url: str, **kw) -> HTTPResult:
|
||
return await self._with_retry(lambda: self.request_once(method, url, **kw))
|
||
|
||
async def get_text(self, url: str, **kw) -> str:
|
||
return (await self.request('GET', url, **kw)).text
|
||
|
||
async def post_text(self, url: str, **kw) -> str:
|
||
return (await self.request('POST', url, **kw)).text
|
||
|
||
async def get_json(self, url: str, **kw) -> Any:
|
||
return (await self.request('GET', url, **kw)).json()
|
||
|
||
async def post_json(self, url: str, **kw) -> Any:
|
||
return (await self.request('POST', url, **kw)).json()
|
||
|
||
@asynccontextmanager
|
||
async def stream(
|
||
self,
|
||
method: str,
|
||
url: str,
|
||
*,
|
||
headers: Mapping[str, str] | None = None,
|
||
cookies: Mapping[str, str] | None = None,
|
||
timeout: float | None = None,
|
||
allow_redirects: bool = True,
|
||
) -> AsyncGenerator[aiohttp.ClientResponse, None]:
|
||
"""流式请求（下载/音视频代理用）：yield 原始响应，边读边写不整段入内存。"""
|
||
session = await self._session_for_loop()
|
||
req_timeout = aiohttp.ClientTimeout(total=timeout) if timeout else None
|
||
try:
|
||
async with session.request(
|
||
method,
|
||
url,
|
||
headers=dict(headers) if headers else None,
|
||
cookies=dict(cookies) if cookies else None,
|
||
timeout=req_timeout,
|
||
allow_redirects=allow_redirects,
|
||
) as resp:
|
||
if resp.status >= 400:
|
||
raise UpstreamHTTPError(resp.status, resp.reason or '')
|
||
yield resp
|
||
except aiohttp.ClientError as e:
|
||
raise UpstreamNetworkError(str(e)) from e
|
||
|
||
async def open(
|
||
self,
|
||
method: str,
|
||
url: str,
|
||
*,
|
||
headers: Mapping[str, str] | None = None,
|
||
cookies: Mapping[str, str] | None = None,
|
||
timeout: float | None = None,
|
||
allow_redirects: bool = True,
|
||
) -> aiohttp.ClientResponse:
|
||
"""打开流式响应并返回**未关闭**的 ClientResponse（调用方负责 close）。
|
||
|
||
与 stream() 的区别：不把响应交给 `async with`，便于「先读状态码/头决定是否
|
||
转发，再流式读取」的代理场景（如音频 Range 代理）。
|
||
"""
|
||
session = await self._session_for_loop()
|
||
req_timeout = aiohttp.ClientTimeout(total=timeout) if timeout else None
|
||
try:
|
||
return await session.request(
|
||
method,
|
||
url,
|
||
headers=dict(headers) if headers else None,
|
||
cookies=dict(cookies) if cookies else None,
|
||
timeout=req_timeout,
|
||
allow_redirects=allow_redirects,
|
||
)
|
||
except aiohttp.ClientError as e:
|
||
raise UpstreamNetworkError(str(e)) from e
|
||
|
||
|
||
# 进程级共享客户端（UA/Referer 由 music_api 在初始化后注入）
|
||
client = AsyncHTTPClient()
|