Archived
snowy-yang
65adfc9089
core/aiohttp_compat 是「同步转异步」迁移期的 requests 同形异步垫片：它存在的 唯一理由是让既有 `requests.post/get + raise_for_status()` 风格的代码以最小 改动迁到异步。其两个调用方（music_api/api.py、music_api/client.py）**本就已是 async def**——垫片只剩「保留 requests 语义」这一层包装，属于迁移残留。 改动： - api.py：23 处 requests.post/get → await core.http_client.request()。 该文件每处都在请求后立刻 raise_for_status() 才取 .json()，从不把 4xx 当数据读， 故「_once 对非 2xx 直接抛 UpstreamHTTPError」与旧语义等价（原 raise_for_status 抛的 HTTPError 也一律落入 RequestException 分支）； except requests.RequestException → except UpstreamError。 - client.py：两个自管重试的方法改用新公开的 core.http_client.request_once() （单次、不重试），保留原有「5xx 退避重试一次 / 4xx 立即失败」的分流—— 把 status_code>=500 的判断换成 UpstreamHTTPError.status，行为逐条对齐 （4xx=风控不重试，网络/5xx 重试一次）。响应体上限交由客户端 max_bytes 统一把关。 post_request_full 返回类型改为 HTTPResult（qr.py 用到的 .text/.cookies/.headers 三者在 HTTPResult 上齐备）。 - core/http_client：_once 更名并公开为 request_once（自管重试的调用方需要它）。 注：这是为「删垫片」而补的唯一一个小口子，不是新增抽象。 注意一处**有意的行为变化**：api.py 原经垫片请求零重试，现在 request() 自带 「网络异常/5xx 退避重试一次、4xx 不重试」（core.http_client 既定口径）。 429/460 属 4xx 不重试，不加剧风控。 - 删除 core/aiohttp_compat.py；web/app.py 去掉其 close()；pyproject 去掉其 per-file-ignore；core/http_errors.py 注释更正为「仅供 core.sync_http」。 - 测试：test_aiohttp_compat.py 拆解为 test_async_bridge.py（只留桥的两条契约， 垫片自身的 4 条随垫片消亡）；test_search_music / test_playlist_detail 的替身 边界从 api.requests.post 换成 api._http.request，异常类型换成 UpstreamNetworkError。 门禁：ruff 全过 / pyright 0-0 / pytest 383 项全绿（0 失败；383=387-6+2，账面吻合）。
197 lines
7.1 KiB
Python
197 lines
7.1 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""同步 HTTP 门面（底层 aiohttp）——彻底移除 requests 依赖。
|
||
|
||
为什么需要它：部分代码天然是同步的（worker 线程里的流式下载、纯函数里的短链展开/
|
||
IP 归属地查询）。本模块提供 requests 风格的同步 API，但底层复用 aiohttp 连接池，
|
||
不再引入 requests。
|
||
|
||
流式读取的实现：在后台事件循环起一个读取协程，把块推入有界队列（背压），
|
||
同步侧从队列消费——每次搬运只过一次队列，不做「每块一次跨线程调度」。
|
||
"""
|
||
import asyncio
|
||
import queue
|
||
import threading
|
||
from collections.abc import Iterator, Mapping
|
||
from typing import Any
|
||
|
||
from core import async_bridge
|
||
from core.http_client import client as _client
|
||
from core.http_errors import HTTPError, RequestException
|
||
|
||
# 异常层次见 core.http_errors（与调用方共用的 requests 同形定义）。RequestException
|
||
# 在此再导出：music_downloader.models 的 _RetryFromScratch 继承它，既有 except
|
||
# 与测试也直接导入它。
|
||
__all__ = ['HTTPError', 'RequestException', 'Session', 'SyncResponse', 'get', 'post']
|
||
|
||
_DEFAULT_TIMEOUT = 30
|
||
|
||
|
||
class SyncResponse:
|
||
"""requests.Response 的同步子集（底层 aiohttp）。"""
|
||
|
||
def __init__(self, status: int, headers: Mapping[str, str], content: bytes | None,
|
||
url: str, stream_state: dict | None = None):
|
||
self.status_code = status
|
||
self.headers = dict(headers)
|
||
self.url = url
|
||
self._content = content
|
||
self._stream = stream_state
|
||
|
||
@property
|
||
def content(self) -> bytes:
|
||
if self._content is None:
|
||
self._content = b''.join(self.iter_content(64 * 1024))
|
||
return self._content
|
||
|
||
@property
|
||
def text(self) -> str:
|
||
return self.content.decode('utf-8', errors='replace')
|
||
|
||
def json(self, **kw) -> Any:
|
||
import json as _json
|
||
return _json.loads(self.text, **kw)
|
||
|
||
def raise_for_status(self) -> None:
|
||
if self.status_code >= 400:
|
||
raise HTTPError(f'HTTP {self.status_code}', self)
|
||
|
||
def iter_content(self, chunk_size: int = 64 * 1024) -> Iterator[bytes]:
|
||
"""流式读取响应体（大小写与 requests 一致：chunk_size 为字节数）。"""
|
||
if self._content is not None:
|
||
for i in range(0, len(self._content), chunk_size):
|
||
yield self._content[i:i + chunk_size]
|
||
return
|
||
st = self._stream
|
||
assert st is not None
|
||
q: queue.Queue = st['queue']
|
||
while True:
|
||
item = q.get()
|
||
if item is None: # 正常结束
|
||
break
|
||
if isinstance(item, BaseException): # 读取出错
|
||
raise item
|
||
yield item
|
||
|
||
def close(self) -> None:
|
||
st = self._stream
|
||
if st and st.get('cancel') is not None:
|
||
try:
|
||
async_bridge.run_sync(_cancel(st))
|
||
except Exception:
|
||
pass
|
||
|
||
def __enter__(self) -> 'SyncResponse':
|
||
return self
|
||
|
||
def __exit__(self, *exc) -> bool:
|
||
self.close()
|
||
return False
|
||
|
||
|
||
async def _cancel(st: dict) -> None:
|
||
st['cancelled'] = True
|
||
resp = st.get('resp')
|
||
if resp is not None:
|
||
resp.close()
|
||
|
||
|
||
def _stream_reader(st: dict, chunk_size: int) -> None:
|
||
"""在后台事件循环读块 → 有界队列（背压），同步侧消费。"""
|
||
async def _run():
|
||
try:
|
||
async for chunk in st['resp'].content.iter_chunked(chunk_size):
|
||
if st.get('cancelled'):
|
||
break
|
||
while True: # 背压：队列满则稍等，避免内存无界
|
||
try:
|
||
st['queue'].put(chunk, timeout=0.5)
|
||
break
|
||
except queue.Full:
|
||
if st.get('cancelled'):
|
||
return
|
||
except Exception as e:
|
||
st['queue'].put(e)
|
||
finally:
|
||
try:
|
||
st['resp'].close()
|
||
except Exception:
|
||
pass
|
||
st['queue'].put(None)
|
||
|
||
asyncio.run_coroutine_threadsafe(_run(), async_bridge.get_loop())
|
||
|
||
|
||
class Session:
|
||
"""requests.Session 的同步子集（连接池由底层 aiohttp 统一维护）。"""
|
||
|
||
trust_env = False
|
||
|
||
def __init__(self, **kw) -> None:
|
||
# headers：会话级默认头（与 requests.Session().headers.update 等价）
|
||
self.headers: dict[str, str] = dict(kw.pop('headers', {}))
|
||
|
||
def _merge(self, kw: dict) -> dict:
|
||
h = dict(self.headers)
|
||
h.update(kw.pop('headers', None) or {})
|
||
if h:
|
||
kw['headers'] = h
|
||
return kw
|
||
|
||
def get(self, url: str, **kw) -> SyncResponse:
|
||
return _request('GET', url, **self._merge(kw))
|
||
|
||
def post(self, url: str, **kw) -> SyncResponse:
|
||
return _request('POST', url, **self._merge(kw))
|
||
|
||
def close(self) -> None:
|
||
pass
|
||
|
||
|
||
def _norm_timeout(timeout):
|
||
"""requests 允许 (connect, read) 元组；aiohttp 只接受单一总时长——取较大者。"""
|
||
if isinstance(timeout, (tuple, list)):
|
||
vals = [float(v) for v in timeout if isinstance(v, (int, float))]
|
||
return max(vals) if vals else _DEFAULT_TIMEOUT
|
||
return timeout
|
||
|
||
|
||
def _request(method: str, url: str, *, headers: Mapping[str, str] | None = None,
|
||
cookies: Mapping[str, str] | None = None, params: Mapping[str, Any] | None = None,
|
||
data: Any = None, json: Any = None, timeout: float | None = _DEFAULT_TIMEOUT,
|
||
allow_redirects: bool = True, stream: bool = False) -> SyncResponse:
|
||
timeout = _norm_timeout(timeout)
|
||
if stream:
|
||
return _open_stream(method, url, headers=headers, cookies=cookies, params=params,
|
||
data=data, json=json, timeout=timeout,
|
||
allow_redirects=allow_redirects)
|
||
resp = async_bridge.run_sync(_client.request(
|
||
method, url, headers=headers, cookies=cookies, params=params, data=data,
|
||
json_body=json, timeout=timeout, allow_redirects=allow_redirects))
|
||
return SyncResponse(resp.status, resp.headers, resp.text.encode('utf-8'),
|
||
resp.url)
|
||
|
||
|
||
def _open_stream(method: str, url: str, **kw) -> SyncResponse:
|
||
timeout = _norm_timeout(kw.pop('timeout', _DEFAULT_TIMEOUT))
|
||
# AsyncHTTPClient.open 只接受 headers/cookies/timeout/allow_redirects；
|
||
# 流式场景不支持 params/data/json（调用方直接把查询串拼进 URL）
|
||
for unsupported in ('params', 'data', 'json'):
|
||
kw.pop(unsupported, None)
|
||
resp = async_bridge.run_sync(_client.open(method, url, timeout=timeout, **kw))
|
||
st = {'resp': resp, 'queue': queue.Queue(maxsize=8), 'cancel': object(), 'cancelled': False}
|
||
th = threading.Thread(target=_stream_reader, args=(st, 64 * 1024), daemon=True)
|
||
th.start()
|
||
return SyncResponse(resp.status, resp.headers, None, str(resp.url), stream_state=st)
|
||
|
||
|
||
# 模块级便捷函数（等价于共享 Session 的 get/post）
|
||
_default_session = Session()
|
||
|
||
|
||
def get(url: str, **kw) -> SyncResponse:
|
||
return _default_session.get(url, **kw)
|
||
|
||
|
||
def post(url: str, **kw) -> SyncResponse:
|
||
return _default_session.post(url, **kw)
|