diff --git a/core/http_client.py b/core/http_client.py index eb32166..922d63e 100644 --- a/core/http_client.py +++ b/core/http_client.py @@ -289,5 +289,19 @@ class AsyncHTTPClient: raise UpstreamNetworkError(str(e)) from e +async def iter_chunks(resp: aiohttp.ClientResponse, chunk_size: int) -> AsyncGenerator[bytes, None]: + """逐块读取流式响应体，把 aiohttp 的传输异常统一翻译为 UpstreamNetworkError。 + + 请求建立阶段的失败由 open()/request() 翻成 UpstreamError；但**流式读取期间** + 断流抛的是 aiohttp.ClientPayloadError（ClientError 子类），不在那条路径上—— + 调用方（音频下载、播客落盘）只认 UpstreamError 层次，故在此收敛。 + """ + try: + async for chunk in resp.content.iter_chunked(chunk_size): + yield chunk + except aiohttp.ClientError as e: + raise UpstreamNetworkError(str(e)) from e + + # 进程级共享客户端（UA/Referer 由 music_api 在初始化后注入） client = AsyncHTTPClient() diff --git a/music_api/__init__.py b/music_api/__init__.py index f4cc9ce..5f8c2c0 100644 --- a/music_api/__init__.py +++ b/music_api/__init__.py @@ -22,9 +22,6 @@ from .client import ( _as_dict, _ensure_dict, ) -from .compat import ( - url_v1, -) from .constants import ( APIConstants, QualityLevel, @@ -59,5 +56,4 @@ __all__ = [ '_ensure_dict', 'load_cookies', 'parse_cookie_string', - 'url_v1', ] diff --git a/music_api/compat.py b/music_api/compat.py deleted file mode 100644 index 42a5ab8..0000000 --- a/music_api/compat.py +++ /dev/null @@ -1,25 +0,0 @@ -# -*- coding: utf-8 -*- -"""同步兼容函数接口（内部经 run_sync 桥接异步实现）。 - -（自 music_api.py 拆出，行为与原实现一致。） - -⚠️ 迁移期残留：本模块的定位是「同步代码调用异步实现」的桥接门面。调用方陆续 -改为原生异步（`await NeteaseAPI().xxx()`）后，这里的方法即成为死代码，应删除而 -非新增。当前仅 `url_v1` 仍有唯一调用方（`music_api.podcast` 的同步解析路径）， -故本文件只保留它——其余 16 个同步门面已随调用方异步化一并移除。 -""" - -from typing import Any - -from core.async_bridge import run_sync # 模块级同步兼容接口用的同步/异步桥 - -from .api import ( - NeteaseAPI, -) - - -# 向后兼容的函数接口 -def url_v1(song_id: int, level: str, cookies: dict[str, str]) -> dict[str, Any]: - """获取歌曲URL（向后兼容）""" - api = NeteaseAPI() - return run_sync(api.get_song_url(song_id, level, cookies)) diff --git a/music_api/podcast.py b/music_api/podcast.py index 9cf8aca..8184dcf 100644 --- a/music_api/podcast.py +++ b/music_api/podcast.py @@ -10,10 +10,12 @@ 说明： - 播客单集的音频本质上是网易云曲库中的 song（program.mainSong）， - 因此直链复用 music_api.url_v1（eapi song/enhance/player/url/v1）获取。 + 因此直链复用 NeteaseAPI.get_song_url（eapi song/enhance/player/url/v1）获取。 - 支持格式：mp3 / m4a / aac 等（以接口返回 type 为准）。 - 支持链接协议：music.163.com/djradio?id=（电台）、/program?id= 与 /sound?id=（单集）、 以及 163cn.tv 短链。 +- 本模块**整体异步**：HTTP 统一走 core.http_client（进程级连接池 + 有界重试 + + 8MB 响应体上限），调用方直接 await，不再需要同步门面/线程包装。 """ import json @@ -21,9 +23,10 @@ import re import time from typing import Any -from core import sync_http as requests # 同步门面（底层 aiohttp） +from core.http_client import UpstreamError +from core.http_client import client as _http -from .compat import url_v1 +from .api import NeteaseAPI from .constants import APIConstants from .errors import APIException @@ -51,26 +54,19 @@ class PodcastConstants: DEFAULT_HOT_CATE_IDS = [6, 3, 8, 10001] -# 与 music_api 同口径的上游 JSON 响应体上限 -_MAX_UPSTREAM_JSON_BYTES = 8 * 1024 * 1024 - - -def _post_json(url: str, data: dict[str, Any], cookies: dict[str, str] | None = None) -> dict[str, Any]: - """发送表单 POST 请求并返回 JSON""" +async def _post_json(url: str, data: dict[str, Any], cookies: dict[str, str] | None = None) -> dict[str, Any]: + """发送表单 POST 请求并返回 JSON（响应体上限由 core.http_client 统一把关）""" headers = { 'User-Agent': APIConstants.USER_AGENT, 'Referer': APIConstants.REFERER, } try: - response = requests.post(url, data=data, headers=headers, - cookies=cookies or {}, timeout=30) - response.raise_for_status() - if len(response.content) > _MAX_UPSTREAM_JSON_BYTES: - raise APIException('播客接口响应体异常（超过大小上限）') + response = await _http.request('POST', url, data=data, headers=headers, + cookies=cookies or {}, timeout=30) result = response.json() if not isinstance(result, dict): raise APIException('播客接口响应异常（非标准 JSON 对象）') - except requests.RequestException as e: + except UpstreamError as e: raise APIException(f"播客接口请求失败: {e}") from e except json.JSONDecodeError as e: raise APIException(f"解析播客接口响应失败: {e}") from e @@ -180,7 +176,7 @@ def _norm_program(raw: dict[str, Any]) -> dict[str, Any]: _SHORTLINK_CACHE: dict[str, str] = {} -def _resolve_short_link(url: str) -> str: +async def _resolve_short_link(url: str) -> str: """解析 163cn.tv 短链，返回跳转后的真实链接（仅接受整串短链，子串不触发请求）""" if not re.match(r'^https?://(www\.)?163cn\.tv/\S+$', str(url or '').strip(), re.IGNORECASE): return url @@ -192,10 +188,10 @@ def _resolve_short_link(url: str) -> str: return cached resp = None try: - resp = requests.get(url, allow_redirects=False, timeout=10, - headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}) + resp = await _http.open('GET', url, allow_redirects=False, timeout=10, + headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}) real = resp.headers.get('Location', url) - except requests.RequestException: + except UpstreamError: return url finally: if resp is not None: @@ -210,7 +206,7 @@ def _resolve_short_link(url: str) -> str: return real -def extract_podcast_ref(raw: str) -> dict[str, str]: +async def extract_podcast_ref(raw: str) -> dict[str, str]: """从用户输入中提取播客引用（正规 URL 解析，替代子串匹配与非数字剔除） Returns: @@ -223,7 +219,7 @@ def extract_podcast_ref(raw: str) -> dict[str, str]: text = str(raw or '').strip() # 短链先还原（仅当整串就是一个短链 URL） if '163cn.tv' in text.lower(): - text = _resolve_short_link(text) + text = await _resolve_short_link(text) # 有 URL 结构：按路径判断类型，query 正规解析 id（参数顺序无关） if '://' in text or text.startswith('//'): @@ -256,9 +252,9 @@ def extract_podcast_ref(raw: str) -> dict[str, str]: # ============ 核心 API ============ -def search_podcast(keywords: str, cookies: dict[str, str] | None = None, - limit: int = 20, offset: int = 0, - search_type: str = 'all') -> dict[str, Any]: +async def search_podcast(keywords: str, cookies: dict[str, str] | None = None, + limit: int = 20, offset: int = 0, + search_type: str = 'all') -> dict[str, Any]: """搜索播客 Args: @@ -278,18 +274,17 @@ def search_podcast(keywords: str, cookies: dict[str, str] | None = None, result: dict[str, Any] = {'radios': [], 'voices': []} - def _do_search(stype: int) -> dict[str, Any]: - return _post_json(PodcastConstants.SEARCH_API, - {'s': keywords, 'type': stype, 'limit': limit, 'offset': offset}, - cookies) - if search_type in ('all', 'radio'): - res = _do_search(1009) + res = await _post_json(PodcastConstants.SEARCH_API, + {'s': keywords, 'type': 1009, 'limit': limit, 'offset': offset}, + cookies) radios = (res.get('result') or {}).get('djRadios') or [] result['radios'] = [_norm_radio(r) for r in radios] if search_type in ('all', 'voice'): - res = _do_search(2000) + res = await _post_json(PodcastConstants.SEARCH_API, + {'s': keywords, 'type': 2000, 'limit': limit, 'offset': offset}, + cookies) rdata = res.get('result') or {} voices = rdata.get('songs') or rdata.get('voices') or rdata.get('programs') or [] result['voices'] = [_norm_program(v) for v in voices] @@ -297,13 +292,13 @@ def search_podcast(keywords: str, cookies: dict[str, str] | None = None, return result -def get_radio_detail(radio_id: Any, cookies: dict[str, str] | None = None) -> dict[str, Any]: +async def get_radio_detail(radio_id: Any, cookies: dict[str, str] | None = None) -> dict[str, Any]: """获取播客电台（djradio）详情""" try: rid = int(str(radio_id).strip()) except (TypeError, ValueError): raise APIException('无效的电台ID') from None - result = _post_json(PodcastConstants.DJRADIO_GET_API, {'id': rid}, cookies) + result = await _post_json(PodcastConstants.DJRADIO_GET_API, {'id': rid}, cookies) # 正常返回键为 djRadio；不存在时 code=404（_post_json 已抛错）或 data 为空 data = result.get('djRadio') or result.get('data') or {} if not data: @@ -311,8 +306,8 @@ def get_radio_detail(radio_id: Any, cookies: dict[str, str] | None = None) -> di return _norm_radio(data) -def get_radio_programs(radio_id: Any, cookies: dict[str, str] | None = None, - limit: int = 30, offset: int = 0, asc: bool = True) -> dict[str, Any]: +async def get_radio_programs(radio_id: Any, cookies: dict[str, str] | None = None, + limit: int = 30, offset: int = 0, asc: bool = True) -> dict[str, Any]: """获取电台下的单集列表（分页） Args: @@ -329,10 +324,10 @@ def get_radio_programs(radio_id: Any, cookies: dict[str, str] | None = None, except (TypeError, ValueError): raise APIException('无效的电台ID') from None limit = max(1, min(int(limit or 30), 100)) - result = _post_json(PodcastConstants.DJ_PROGRAM_BYRADIO_API, - {'radioId': rid, 'limit': limit, 'offset': max(0, int(offset or 0)), - 'asc': 'true' if asc else 'false'}, - cookies) + result = await _post_json(PodcastConstants.DJ_PROGRAM_BYRADIO_API, + {'radioId': rid, 'limit': limit, 'offset': max(0, int(offset or 0)), + 'asc': 'true' if asc else 'false'}, + cookies) programs = [_norm_program(p) for p in (result.get('programs') or [])] count = _to_int(result.get('count')) offset_val = max(0, int(offset or 0)) @@ -343,20 +338,20 @@ def get_radio_programs(radio_id: Any, cookies: dict[str, str] | None = None, } -def get_program_detail(program_id: Any, cookies: dict[str, str] | None = None) -> dict[str, Any]: +async def get_program_detail(program_id: Any, cookies: dict[str, str] | None = None) -> dict[str, Any]: """获取单集（program）详情""" try: pid = int(str(program_id).strip()) except (TypeError, ValueError): raise APIException('无效的单集ID') from None - result = _post_json(PodcastConstants.DJ_PROGRAM_DETAIL_API, {'id': pid}, cookies) + result = await _post_json(PodcastConstants.DJ_PROGRAM_DETAIL_API, {'id': pid}, cookies) program = result.get('program') or {} if not program: raise APIException('未找到该播客单集（可能已下架或ID错误）') return _norm_program(program) -def get_categories() -> list[dict[str, Any]]: +async def get_categories() -> list[dict[str, Any]]: """获取播客分类列表（GET 接口） Returns: @@ -367,13 +362,10 @@ def get_categories() -> list[dict[str, Any]]: 'Referer': APIConstants.REFERER, } try: - response = requests.get(PodcastConstants.DJRADIO_CATEGORY_API, - headers=headers, timeout=30) - response.raise_for_status() - if len(response.content) > _MAX_UPSTREAM_JSON_BYTES: - raise APIException('播客接口响应体异常（超过大小上限）') + response = await _http.request('GET', PodcastConstants.DJRADIO_CATEGORY_API, + headers=headers, timeout=30) result = response.json() - except requests.RequestException as e: + except UpstreamError as e: raise APIException(f"获取播客分类失败: {e}") from e except json.JSONDecodeError as e: raise APIException(f"解析播客分类响应失败: {e}") from e @@ -396,9 +388,9 @@ def get_categories() -> list[dict[str, Any]]: return cats -def get_hot_radios(cookies: dict[str, str] | None = None, - limit: int = 30, offset: int = 0, - cate_id: int | None = None) -> list[dict[str, Any]]: +async def get_hot_radios(cookies: dict[str, str] | None = None, + limit: int = 30, offset: int = 0, + cate_id: int | None = None) -> list[dict[str, Any]]: """获取分类下热门播客 Args: @@ -423,9 +415,9 @@ def get_hot_radios(cookies: dict[str, str] | None = None, cate_ids.append(cid) for cid in cate_ids: - result = _post_json(PodcastConstants.DJRADIO_HOT_API, - {'limit': limit, 'offset': max(0, int(offset or 0)), 'cateId': cid}, - cookies) + result = await _post_json(PodcastConstants.DJRADIO_HOT_API, + {'limit': limit, 'offset': max(0, int(offset or 0)), 'cateId': cid}, + cookies) radios = [_norm_radio(r) for r in (result.get('djRadios') or [])] if radios: return radios @@ -455,8 +447,8 @@ def _falldown_order(level: str) -> list[str]: return list(dict.fromkeys(order)) # 去重保序 -def resolve_episode_url(song_id: Any, cookies: dict[str, str] | None = None, - level: str = 'auto') -> dict[str, Any] | None: +async def resolve_episode_url(song_id: Any, cookies: dict[str, str] | None = None, + level: str = 'auto') -> dict[str, Any] | None: """解析单集音频直链（带音质降级） Args: @@ -473,9 +465,10 @@ def resolve_episode_url(song_id: Any, cookies: dict[str, str] | None = None, sid = int(str(song_id).strip()) except (TypeError, ValueError): return None + api = NeteaseAPI() for lv in order: try: - res = url_v1(sid, lv, cookies or {}) + res = await api.get_song_url(sid, lv, cookies or {}) except (APIException, ValueError): continue data = (res or {}).get('data') or [] @@ -495,36 +488,36 @@ def resolve_episode_url(song_id: Any, cookies: dict[str, str] | None = None, # ============ 组合能力 ============ -def parse_podcast_ref(raw: str, cookies: dict[str, str] | None = None, - page_size: int = 30) -> dict[str, Any]: +async def parse_podcast_ref(raw: str, cookies: dict[str, str] | None = None, + page_size: int = 30) -> dict[str, Any]: """智能解析用户输入：电台链接 → 电台详情+单集列表；单集链接 → 单集详情 Returns: {'kind': 'radio', 'radio': {...}, 'programs': [...], 'count': n, 'has_more': bool} 或 {'kind': 'program', 'program': {...}} """ - ref = extract_podcast_ref(raw) + ref = await extract_podcast_ref(raw) if ref['kind'] == 'radio': - radio = get_radio_detail(ref['id'], cookies) - page = get_radio_programs(ref['id'], cookies, limit=page_size, offset=0, asc=True) + radio = await get_radio_detail(ref['id'], cookies) + page = await get_radio_programs(ref['id'], cookies, limit=page_size, offset=0, asc=True) return {'kind': 'radio', 'radio': radio, **page} if ref['kind'] == 'program': - return {'kind': 'program', 'program': get_program_detail(ref['id'], cookies)} + return {'kind': 'program', 'program': await get_program_detail(ref['id'], cookies)} # 纯数字：先试单集，再试电台 try: - return {'kind': 'program', 'program': get_program_detail(ref['id'], cookies)} + return {'kind': 'program', 'program': await get_program_detail(ref['id'], cookies)} except APIException: pass - radio = get_radio_detail(ref['id'], cookies) - page = get_radio_programs(ref['id'], cookies, limit=page_size, offset=0, asc=True) + radio = await get_radio_detail(ref['id'], cookies) + page = await get_radio_programs(ref['id'], cookies, limit=page_size, offset=0, asc=True) return {'kind': 'radio', 'radio': radio, **page} -def list_radio_all_program_ids(radio_id: Any, cookies: dict[str, str] | None = None, - max_count: int = 200) -> list[dict[str, Any]]: +async def list_radio_all_program_ids(radio_id: Any, cookies: dict[str, str] | None = None, + max_count: int = 200) -> list[dict[str, Any]]: """拉取电台全部单集（用于整台打包，正序），返回 [{'id','name','main_song_id','serial_num','cover_url'}] 最多拉取 max_count 集，防止超大电台拖垮磁盘/任务。 @@ -537,7 +530,7 @@ def list_radio_all_program_ids(radio_id: Any, cookies: dict[str, str] | None = N page_size = 100 max_pages = 30 # 有界保护：即使上游 has_more 恒真也最多翻 30 页 for _ in range(max_pages): - page = get_radio_programs(radio_id, cookies, limit=page_size, offset=offset, asc=True) + page = await get_radio_programs(radio_id, cookies, limit=page_size, offset=offset, asc=True) new_count = 0 for p in page['programs']: if p['id'] in seen_ids: diff --git a/music_downloader/__init__.py b/music_downloader/__init__.py index ca3d96c..23c1140 100644 --- a/music_downloader/__init__.py +++ b/music_downloader/__init__.py @@ -2,9 +2,8 @@ """网易云音乐下载器（标签写入基于 mutagen，纯 Python） 提供功能： -- 同步 / 异步下载（本地落盘 / 内存返回） -- 批量并发下载 -- 进度查询 +- 异步下载（本地落盘） +- 批量并发下载（由上层服务编排） - 基于 mutagen 的标签 + 封面写入（MP3/FLAC/M4A/MP4/OGG/Opus） 设计要点： @@ -12,6 +11,7 @@ 2. 传输通道二选一：aria2c 常驻 RPC（多连接）优先，不可用时回落内置 aiohttp 流式 3. 标签写入按容器分派（ID3v2.3 / Vorbis comment / iTunes ilst），全部原子替换 4. 扩展名优先使用 API 返回的 type 字段，URL/Content-Type 兜底 +5. 全链路异步：HTTP 走 core.http_client，阻塞 IO/CPU 经 asyncio.to_thread 卸载 """ # 兼容门面：实现已按职责拆分到同包各子模块，此处原样再导出。 @@ -27,11 +27,9 @@ from ._concurrency import ( get_file_lock, ) from ._http import ( - _SESSION_LOCAL, DEFAULT_UA, DL_CHUNK_SIZE, _sniff_audio_header, - get_http_session, ) from ._log import ( logger, @@ -68,7 +66,6 @@ __all__ = [ '_COVER_CACHE_ORDER', '_FILE_LOCKS', '_FILE_LOCKS_GUARD', - '_SESSION_LOCAL', 'CountingSemaphore', 'DownloadCanceled', 'DownloadException', @@ -82,6 +79,5 @@ __all__ = [ '_sniff_audio_header', 'fetch_cover_bytes', 'get_file_lock', - 'get_http_session', 'logger', ] diff --git a/music_downloader/__main__.py b/music_downloader/__main__.py index b56668f..bc0b5ac 100644 --- a/music_downloader/__main__.py +++ b/music_downloader/__main__.py @@ -3,6 +3,10 @@ from __future__ import annotations +import asyncio +import sys +from dataclasses import asdict + from .downloader import ( MusicDownloader, ) @@ -12,44 +16,45 @@ from .models import ( # ============ CLI 入口 ============ -if __name__ == "__main__": - import sys - from dataclasses import asdict +def _print_usage() -> None: + """无参数时打印模块信息与用法。""" + print("网易云音乐下载器") + print("支持的音质: standard / exhigh / lossless / hires / sky / jyeffect / jymaster / dolby") + print("支持的功能: 单曲/批量下载、标签与封面写入（mutagen 纯 Python）") + print() + print("用法:") + print(" python -m music_downloader [quality] # 仅打印信息") + print(" python -m music_downloader [quality] --download # 打印并下载") + print() + print("示例:") + print(" python -m music_downloader 123456") + print(" python -m music_downloader 123456 lossless") + print(" python -m music_downloader 123456 hires --download") + + +async def _main(argv: list[str]) -> int: + """CLI 主体（下载链路是协程，故整体在 asyncio.run 里跑）。返回进程退出码。""" downloader = MusicDownloader() - # 无参数：打印模块信息 - if len(sys.argv) < 2: - print("网易云音乐下载器") - print("支持的音质: standard / exhigh / lossless / hires / sky / jyeffect / jymaster / dolby") - print("支持的功能: 同步/批量下载、标签与封面写入（mutagen 纯 Python）") - print() - print("用法:") - print(" python music_downloader.py [quality] # 仅打印信息") - print(" python music_downloader.py [quality] --download # 打印并下载") - print() - print("示例:") - print(" python music_downloader.py 123456") - print(" python music_downloader.py 123456 lossless") - print(" python music_downloader.py 123456 hires --download") - sys.exit(0) + if len(argv) < 2: + _print_usage() + return 0 - # 解析参数 try: - music_id = int(sys.argv[1]) + music_id = int(argv[1]) except ValueError: - print(f"错误: music_id 必须是整数，收到: {sys.argv[1]!r}") - sys.exit(1) + print(f"错误: music_id 必须是整数，收到: {argv[1]!r}") + return 1 - quality = sys.argv[2] if len(sys.argv) > 2 and not sys.argv[2].startswith("--") else "lossless" - do_download = "--download" in sys.argv + quality = argv[2] if len(argv) > 2 and not argv[2].startswith("--") else "lossless" + do_download = "--download" in argv - # 打印音乐信息 try: - info = downloader.get_music_info(music_id, quality) + info = await downloader.get_music_info(music_id, quality) except DownloadException as e: print(f"获取音乐信息失败: {e}") - sys.exit(1) + return 1 print() print("=" * 70) @@ -72,13 +77,18 @@ if __name__ == "__main__": print("=" * 70) - # 可选：触发下载 if do_download: print() print("开始下载...") - result = downloader.download_music_file(music_id, quality) + result = await downloader.download_music_file(music_id, quality) if result.success: print(f"下载完成: {result.file_path} ({result.file_size} 字节)") else: print(f"下载失败: {result.error_message}") - sys.exit(1) + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(_main(sys.argv))) diff --git a/music_downloader/_aria2.py b/music_downloader/_aria2.py index dbb8959..82b2bce 100644 --- a/music_downloader/_aria2.py +++ b/music_downloader/_aria2.py @@ -9,12 +9,17 @@ - RPC 只监听 127.0.0.1 并强制随机 secret —— 无鉴权的 aria2 RPC 允许本机任意 进程操纵下载队列（读写任意文件），暴露到容器外等于开了一个后门； - aria2c 缺失 / 进程早退 / RPC 未就绪，一律返回 None 静默降级， - 由调用方回落到既有的 core.sync_http 流式通道； + 由调用方回落到内置的 core.http_client 流式通道； - aria2 没有「单文件体积上限」选项，故用轮询 tellStatus 实现体积守卫与取消响应。 + +异步口径：aria2p 客户端本身是同步阻塞实现（内部走 urllib），故每次 RPC 调用 +（提交/查询/移除）都经 ``asyncio.to_thread`` 卸载；轮询间隔改 ``asyncio.sleep`` +让出事件循环。进程启动（``_try_start``）最长阻塞 8s，调用方同样经 to_thread 触发。 """ from __future__ import annotations +import asyncio import atexit import secrets import shutil @@ -26,7 +31,7 @@ from collections.abc import Callable from pathlib import Path from typing import Any, Protocol -from core import sync_http as requests # 同步门面（底层 aiohttp） +from core.http_client import UpstreamNetworkError from ._http import DEFAULT_UA from ._log import logger @@ -101,7 +106,7 @@ class Aria2Backend: # ---------- 下载 ---------- - def fetch( + async def fetch( self, url: str, target: Path, @@ -118,7 +123,7 @@ class Aria2Backend: Raises: DownloadCanceled: cancel_event 置位（残片一并删除） DownloadException: 体积超限（残片一并删除）、aria2 报告任务失败、总超时 - requests.RequestException: RPC 抖动等可重试的网络类失败（保留残片续传） + UpstreamNetworkError: RPC 抖动等可重试的网络类失败（保留残片续传） """ options = { 'dir': str(target.parent), @@ -137,20 +142,20 @@ class Aria2Backend: options['referer'] = referer try: - download = self._api.add_uris([url], options=options) + download = await asyncio.to_thread(self._api.add_uris, [url], options=options) except Exception as e: - raise requests.RequestException(f'aria2 提交任务失败: {e}') from e + raise UpstreamNetworkError(f'aria2 提交任务失败: {e}') from e deadline = time.monotonic() + timeout while True: if cancel_event is not None and cancel_event.is_set(): - self._abort(download, drop_files=True) + await asyncio.to_thread(self._abort, download, drop_files=True) raise DownloadCanceled('下载已取消') try: - download.update() + await asyncio.to_thread(download.update) except Exception as e: # RPC 抖动按可重试的网络失败上报，交给外层退避重试 - raise requests.RequestException(f'aria2 状态查询失败: {e}') from e + raise UpstreamNetworkError(f'aria2 状态查询失败: {e}') from e done = int(download.completed_length or 0) total = int(download.total_length or 0) @@ -160,7 +165,7 @@ class Aria2Backend: except Exception: pass # 进度回调绝不影响下载主流程 if done > max_bytes: - self._abort(download, drop_files=True) + await asyncio.to_thread(self._abort, download, drop_files=True) raise DownloadException( f'文件超过大小上限 {max_bytes // (1024 * 1024)}MB，已中止') if download.is_complete: @@ -168,12 +173,12 @@ class Aria2Backend: return if download.has_failed: # 保留残片与控制文件，等待外层重试时续传 - raise requests.RequestException( + raise UpstreamNetworkError( f'aria2 下载失败: {download.error_message or download.status}') if time.monotonic() > deadline: - self._abort(download, drop_files=False) - raise requests.RequestException('aria2 下载超时') - time.sleep(_POLL_INTERVAL) + await asyncio.to_thread(self._abort, download, drop_files=False) + raise UpstreamNetworkError('aria2 下载超时') + await asyncio.sleep(_POLL_INTERVAL) # ---------- 内部 ---------- @@ -197,7 +202,9 @@ class Aria2Backend: def get_backend() -> Aria2Backend | None: """返回 aria2 后端单例；aria2c 不可用/启动失败时返回 None（调用方降级）。 - 首次调用尝试拉起常驻进程，之后命中缓存；失败结果进程内记忆，不反复重试。 + 首次调用尝试拉起常驻进程（最长阻塞 _RPC_READY_TIMEOUT 秒），之后命中缓存； + 失败结果进程内记忆，不反复重试。**同步阻塞**——异步调用方须经 + ``asyncio.to_thread(get_backend)`` 触发，避免首次启动期间冻住事件循环。 """ global _backend, _unavailable if _unavailable: diff --git a/music_downloader/_concurrency.py b/music_downloader/_concurrency.py index 91251c8..38abc83 100644 --- a/music_downloader/_concurrency.py +++ b/music_downloader/_concurrency.py @@ -1,11 +1,18 @@ # -*- coding: utf-8 -*- -"""文件锁与计数信号量。 +"""文件锁与并发闸（asyncio 原语）。 -（自 music_downloader.py 拆出，行为与原实现一致。） +（自 music_downloader.py 拆出。） + +下载链路已整体异步化：同一进程、同一事件循环里跑多个任务，互斥不再靠线程 +阻塞，而是靠 ``await`` 让出控制权。故此处两个原语都换成 asyncio 实现—— +用 threading 锁会「持锁方在等事件循环、等待方又在事件循环上」而直接死锁。 +两个计数器字典仍用 threading.Lock 保护：临界区只有字典读写、内部无 await， +不会把事件循环卡住，也让仍在同步线程里跑的监控循环能安全读取。 """ from __future__ import annotations +import asyncio import threading from pathlib import Path @@ -13,7 +20,7 @@ from pathlib import Path # 防止并发下载同名歌曲（不同用户/任务）互相覆盖导致文件损坏 # 锁表带引用计数与回收——无人引用时从表中移除，防止长期运行无界增长 # 计数在注册表守卫锁内完成（"取出引用"即计数 +1），消除"取引用后、acquire 前 -# 被另一个线程回收、第三个线程拿到新锁对象"的互斥破坏窗口 +# 被另一个任务回收、第三个任务拿到新锁对象"的互斥破坏窗口 _FILE_LOCKS: dict[str, RefCountedFileLock] = {} _FILE_LOCKS_GUARD = threading.Lock() @@ -23,11 +30,11 @@ class RefCountedFileLock: """带引用计数的文件锁：refs 表示"已持有或已登记即将持有"的数量，归零可回收""" def __init__(self): - self._lock = threading.Lock() + self._lock = asyncio.Lock() self.refs = 0 - def acquire(self) -> None: - self._lock.acquire() + async def acquire(self) -> None: + await self._lock.acquire() def release(self) -> None: self._lock.release() @@ -37,14 +44,14 @@ class RefCountedFileLock: class _FileLockHandle: - """兼容 `with get_file_lock(path):` 旧用法的句柄：语义与 file_lock_scope 一致""" + """`async with get_file_lock(path):` 句柄：语义与引用计数回收口径同旧实现""" def __init__(self, path: Path): self._path = path self._lock = None self._key = '' - def __enter__(self) -> RefCountedFileLock: + async def __aenter__(self) -> RefCountedFileLock: key = str(self._path) with _FILE_LOCKS_GUARD: lock = _FILE_LOCKS.get(key) @@ -55,7 +62,7 @@ class _FileLockHandle: self._lock = lock self._key = key try: - lock.acquire() + await lock.acquire() except BaseException: # acquire 被打断时回滚引用计数并摘除空表项，防锁表条目永久泄漏 with _FILE_LOCKS_GUARD: @@ -67,7 +74,7 @@ class _FileLockHandle: raise return lock - def __exit__(self, exc_type, exc_val, exc_tb) -> None: + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: if self._lock is None: return self._lock.release() @@ -79,72 +86,42 @@ class _FileLockHandle: def get_file_lock(path: Path) -> _FileLockHandle: - """兼容入口：返回可用 with 语句的锁句柄（引用计数在守卫锁内登记，无回收竞态）""" + """兼容入口：返回可用 `async with` 的锁句柄（引用计数在守卫锁内登记，无回收竞态）""" return _FileLockHandle(path) class CountingSemaphore: - """带活跃/等待计数的信号量（线程安全），用于实时展示排队情况 + """带活跃/等待计数的并发闸（asyncio），用于实时展示排队情况 - 用法与 threading.BoundedSemaphore 一致（支持 with 语句）， - 额外维护 active（正在执行）/ waiting（排队等待）两个计数， - 供前端实时展示排队人数与预计等待时间。 + 容量由 ``active`` 计数实现，等待靠 ``await asyncio.sleep`` 让出事件循环。 + 与旧实现（threading.BoundedSemaphore + 轮询 ``acquire(blocking=False)``） + 同为非公平抢占、同样按 poll 粒度醒来，语义不变；差别只在等待不再阻塞线程。 """ def __init__(self, value: int = 1): self.capacity = max(1, int(value)) - self._semaphore = threading.BoundedSemaphore(self.capacity) self._lock = threading.Lock() self.active = 0 self.waiting = 0 - def acquire(self, blocking: bool = True) -> bool: - if not blocking: - if not self._semaphore.acquire(blocking=False): - return False - with self._lock: - self.active += 1 - return True - with self._lock: - self.waiting += 1 - try: - self._semaphore.acquire() - except BaseException: - with self._lock: - self.waiting = max(0, self.waiting - 1) - raise - with self._lock: - self.waiting = max(0, self.waiting - 1) - self.active += 1 - return True - - def acquire_cancellable(self, cancel_ev, poll: float = 1.0) -> bool: + async def acquire_cancellable(self, cancel_ev, poll: float = 1.0) -> bool: """可取消地等待名额：等待期间 cancel_ev 置位立即返回 False， - 不再无期限阻塞线程与退款。返回 True 表示已获得名额。""" - import time as _time + 不再无期限等待。返回 True 表示已获得名额。""" with self._lock: self.waiting += 1 try: while True: if cancel_ev is not None and cancel_ev.is_set(): return False - if self._semaphore.acquire(blocking=False): - with self._lock: + with self._lock: + if self.active < self.capacity: self.active += 1 - return True - _time.sleep(poll) + return True + await asyncio.sleep(poll) finally: with self._lock: self.waiting = max(0, self.waiting - 1) def release(self) -> None: - self._semaphore.release() with self._lock: self.active = max(0, self.active - 1) - - def __enter__(self) -> CountingSemaphore: - self.acquire() - return self - - def __exit__(self, exc_type, exc_val, exc_tb) -> None: - self.release() diff --git a/music_downloader/_http.py b/music_downloader/_http.py index 20d95e1..ecf3726 100644 --- a/music_downloader/_http.py +++ b/music_downloader/_http.py @@ -1,20 +1,18 @@ # -*- coding: utf-8 -*- -"""HTTP 会话复用与音频内容嗅探。 +"""浏览器 UA、下载分块与音频内容嗅探。 -（自 music_downloader.py 拆出，行为与原实现一致。） +（自 music_downloader.py 拆出。） + +HTTP 本身已改走 core.http_client（进程级异步连接池），本模块不再持有任何 +会话/连接状态——原先"线程本地 Session + keep-alive"的提速手段由 aiohttp +连接池统一承担，也就没有"同步 Session"这一层可留。 """ from __future__ import annotations -import threading - import filetype -from core import sync_http as requests # 同步门面（底层 aiohttp，替代 requests） - -# ============ HTTP 连接复用（提速核心） ============ - -# 常见浏览器 UA，避免直链 CDN 因空 UA 限速/拒绝 +# 常见浏览器 UA，避免直链 CDN 因空 UA 限速/拒绝；封面等旁路请求也复用它 DEFAULT_UA = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" @@ -25,10 +23,6 @@ DEFAULT_UA = ( DL_CHUNK_SIZE = 256 * 1024 -# 线程本地 Session：同线程内复用 TCP/TLS 连接（keep-alive），批量并发显著提速 -_SESSION_LOCAL = threading.local() - - # M4A/MP4 音频在 filetype 里被判为 video/mp4——mp4 是复合容器，仅凭魔数无法 # 区分音频轨与视频轨，故显式放行该 mime。 _CONTAINER_MIMES = frozenset({'video/mp4'}) @@ -51,13 +45,3 @@ def _sniff_audio_header(path) -> bool: return False mime = (kind.mime or '').lower() return mime.startswith('audio/') or mime in _CONTAINER_MIMES - - -def get_http_session() -> requests.Session: - """获取当前线程的共享同步会话（连接池由底层 aiohttp 统一维护）。""" - session: requests.Session | None = getattr(_SESSION_LOCAL, "session", None) - if session is None: - session = requests.Session() - session.headers.update({"User-Agent": DEFAULT_UA}) - _SESSION_LOCAL.session = session - return session diff --git a/music_downloader/cover.py b/music_downloader/cover.py index f6b3225..948c015 100644 --- a/music_downloader/cover.py +++ b/music_downloader/cover.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """封面字节缓存（同专辑只从网络拉一次）。 -（自 music_downloader.py 拆出，行为与原实现一致。） +（自 music_downloader.py 拆出，行为与原实现一致；HTTP 改走 core.http_client。） """ from __future__ import annotations @@ -9,11 +9,14 @@ from __future__ import annotations import threading from collections import deque +from core.http_client import client as _http +from core.http_client import iter_chunks + from ._http import ( - get_http_session, + DEFAULT_UA, ) -# 封面字节缓存：同一专辑封面只从网络拉取一次（ffmpeg 写元数据 与 ZIP 打包 共用） +# 封面字节缓存：同一专辑封面只从网络拉取一次（元数据写入 与 ZIP 打包 共用） # 容量抽成常量：淘汰循环按它判定，不再读 deque.maxlen（该属性类型为 int|None， # 一旦有人把 deque 改成无界，len() >= None 会直接 TypeError） _COVER_CACHE_MAX = 128 @@ -22,10 +25,15 @@ _COVER_CACHE: dict[str, bytes] = {} _COVER_CACHE_ORDER: deque[str] = deque(maxlen=_COVER_CACHE_MAX) +# 只保护字典/双端队列的读写，临界区内无 await——仍用 threading.Lock， +# 便于同步线程（ZIP 组装）读取缓存时一并复用 _COVER_CACHE_LOCK = threading.Lock() +# 封面字节预算：拒绝超大盘点阵/伪装文件 +_COVER_MAX_BYTES = 10 * 1024 * 1024 -def fetch_cover_bytes(pic_url: str) -> bytes | None: + +async def fetch_cover_bytes(pic_url: str) -> bytes | None: """下载并缓存封面图片字节；失败返回 None（字节预算 10MB，拒绝超大盘点阵/伪装文件）""" if not pic_url: return None @@ -34,24 +42,26 @@ def fetch_cover_bytes(pic_url: str) -> bytes | None: if cached: return cached try: - resp = get_http_session().get(pic_url, timeout=15, stream=True) + resp = await _http.open('GET', pic_url, + headers={'User-Agent': DEFAULT_UA}, timeout=15) try: - resp.raise_for_status() + if resp.status >= 400: + return None try: declared = int(resp.headers.get('Content-Length') or 0) except ValueError: declared = 0 # 垃圾 Content-Length 不炸，按未知长度走流式 - if declared > 10 * 1024 * 1024: + if declared > _COVER_MAX_BYTES: return None # bytearray 就地追加（原 bytes += 是 O(n²) 拷贝，10MB 封面最坏多拷上百 MB） data = bytearray() - for chunk in resp.iter_content(chunk_size=64 * 1024): + async for chunk in iter_chunks(resp, 64 * 1024): data += chunk - if len(data) > 10 * 1024 * 1024: + if len(data) > _COVER_MAX_BYTES: return None if not data: return None - data = bytes(data) + payload = bytes(data) finally: # 所有路径统一关闭响应（原 Content-Length 异常路径漏关） resp.close() @@ -63,6 +73,6 @@ def fetch_cover_bytes(pic_url: str) -> bytes | None: while len(_COVER_CACHE_ORDER) >= _COVER_CACHE_MAX: evict = _COVER_CACHE_ORDER.popleft() _COVER_CACHE.pop(evict, None) - _COVER_CACHE[pic_url] = data + _COVER_CACHE[pic_url] = payload _COVER_CACHE_ORDER.append(pic_url) - return data + return payload diff --git a/music_downloader/downloader.py b/music_downloader/downloader.py index 55bc200..89f243b 100644 --- a/music_downloader/downloader.py +++ b/music_downloader/downloader.py @@ -1,19 +1,24 @@ # -*- coding: utf-8 -*- -"""下载主类：落盘、元数据写入与 ZIP 打包。 +"""下载主类：落盘、元数据写入与交付。 -（自 music_downloader.py 拆出，行为与原实现一致。） +（自 music_downloader.py 拆出。） + +异步口径：整个下载链路是协程。网络走 core.http_client（进程级连接池）， +文件落盘/标签写入是阻塞 IO 与纯 Python CPU 工作，经 ``asyncio.to_thread`` 卸载； +互斥改 asyncio 锁（见 _concurrency），取消仍由 threading.Event 承载—— +取消位由请求线程/监控线程置位，只有轮询读取，跨线程安全且无需跨循环唤醒。 """ from __future__ import annotations +import asyncio import os import re import threading -import time from pathlib import Path -from core import sync_http as requests # 同步门面（底层 aiohttp，替代 requests） -from core.async_bridge import run_sync +from core.http_client import UpstreamError, UpstreamHTTPError, iter_chunks +from core.http_client import client as _http from music_api import APIException, NeteaseAPI, load_cookies from ._aria2 import Aria2Backend, get_backend @@ -24,7 +29,6 @@ from ._concurrency import ( from ._http import ( DL_CHUNK_SIZE, _sniff_audio_header, - get_http_session, ) from ._log import ( logger, @@ -52,7 +56,7 @@ class MusicDownloader: 用法: downloader = MusicDownloader(download_dir="downloads") - result = downloader.download_music_file(music_id, quality="lossless") + result = await downloader.download_music_file(music_id, quality="lossless") """ def __init__( @@ -66,7 +70,7 @@ class MusicDownloader: download_dir: 下载目录 max_concurrent: 单个批量任务内的最大并发下载数 global_max_concurrent: 整个服务同时下载的文件数（含标签写入）。 - 所有请求共享该信号量，防止大量用户同时下载导致内存/CPU 耗尽。 + 所有请求共享该闸，防止大量用户同时下载导致内存/CPU 耗尽。 """ self.download_dir = Path(download_dir) self.download_dir.mkdir(exist_ok=True) @@ -74,14 +78,14 @@ class MusicDownloader: self.api = NeteaseAPI() - # 全局并发信号量：所有请求共用，限制同时进行的下载数 + # 全局并发闸：所有请求共用，限制同时进行的下载数 # （带计数，供前端实时展示排队情况） self._download_semaphore = CountingSemaphore(global_max_concurrent) - # 标签写入专用信号量：写标签会重写整个文件（IO 密集）， + # 标签写入专用闸：写标签会重写整个文件（IO 密集）， # 与下载分离限流 —— 写满时只阻塞后来者，不占下载名额、不拖慢其它文件的传输 cpu_count = os.cpu_count() or 2 - self._tag_semaphore = threading.BoundedSemaphore(max(1, min(4, cpu_count))) + self._tag_semaphore = asyncio.Semaphore(max(1, min(4, cpu_count))) # ----------- 工具方法 ----------- @@ -145,24 +149,20 @@ class MusicDownloader: # ----------- 信息获取 ----------- - def get_music_info(self, music_id: int, quality: str = "standard", - cookies: dict[str, str] | None = None) -> MusicInfo: - """获取音乐详细信息（同步，URL/详情/歌词 3 个接口并行请求，加快单曲/批量启动）""" + async def get_music_info(self, music_id: int, quality: str = "standard", + cookies: dict[str, str] | None = None) -> MusicInfo: + """获取音乐详细信息（URL/详情/歌词 3 个接口并发请求，加快单曲/批量启动）""" try: cookies = cookies if cookies is not None else load_cookies() - # 3 个接口并行请求，单曲下载/批量预取都能明显提速 - # URL/详情拿到后立即取消未完成的歌词请求，已知失败不再被拖着等完 - # 异步化：改用 asyncio.gather 并发（api 方法已异步），失败语义与旧线程池一致 - import asyncio - - async def _gather_info(): - url_t = self.api.get_song_url(music_id, quality, cookies) - detail_t = self.api.get_song_detail(music_id) - lyric_t = self.api.get_lyric(music_id, cookies) - return await asyncio.gather(url_t, detail_t, lyric_t, return_exceptions=True) - - url_result, detail_result, lyric_result = run_sync(_gather_info()) + # 3 个接口并发请求，单曲下载/批量预取都能明显提速 + # URL/详情拿到后不再等待失败的歌词请求，已知失败不拖着等完 + url_result, detail_result, lyric_result = await asyncio.gather( + self.api.get_song_url(music_id, quality, cookies), + self.api.get_song_detail(music_id), + self.api.get_lyric(music_id, cookies), + return_exceptions=True, + ) if isinstance(url_result, BaseException): raise url_result if isinstance(detail_result, BaseException): @@ -255,9 +255,9 @@ class MusicDownloader: safe_tag = self._sanitize_filename(tag, max_bytes=tag_budget) return self.download_dir / f"{safe_base}{safe_tag}{ext}" - # ----------- 下载（同步） ----------- + # ----------- 下载 ----------- - def download_music_file( + async def download_music_file( self, music_id: int, quality: str = "standard", @@ -269,7 +269,7 @@ class MusicDownloader: music_info: MusicInfo | None = None, progress_cb=None, ) -> DownloadResult: - """同步下载音乐到本地 + """下载音乐到本地 Args: music_id: 音乐ID @@ -288,15 +288,15 @@ class MusicDownloader: - 流式下载到 .part 临时文件，完整下载后再原子改名，半截文件不会被当作已下载 - 断点续传：失败保留 .part，重试/下次下载从已写字节续传（服务器支持 Range 时） - per-file 锁：同名歌曲并发下载互斥，防止 .part 被交叉写入损坏 - - 连接复用：线程级 Session + keep-alive，避免每首歌重建 TCP/TLS 连接 - - 网络传输与 ffmpeg 元数据写入分离限流，互不抢占对方资源 + - 连接复用：core.http_client 的进程级连接池 + keep-alive + - 网络传输与标签写入分离限流，互不抢占对方资源 本函数只保留编排与重试策略：缓存校验、续传起点、传输通道、完整性判定、 元数据限流各自见对应私有方法的 docstring（此前是 205 行 / 圈复杂度 47 的单体，任何一处改动都要通读全部步骤）。 """ if music_info is None: - music_info = self.get_music_info(music_id, quality, cookies) + music_info = await self.get_music_info(music_id, quality, cookies) file_path = self._build_file_path(music_info) part_path0 = file_path.with_name(file_path.name + ".part") @@ -304,9 +304,9 @@ class MusicDownloader: next_wait = 0.0 for attempt in range(retries + 1): - # 重试退避睡眠在文件锁外执行——同名并发任务不再陪睡 + # 重试退避等待在文件锁外执行——同名并发任务不再陪睡 if next_wait > 0: - time.sleep(next_wait) + await asyncio.sleep(next_wait) next_wait = 0.0 # 重试轮次顶部的取消检查也要清理 .part（原实现在 try 外抛出会绕过清理） if cancel_event is not None and cancel_event.is_set(): @@ -319,24 +319,24 @@ class MusicDownloader: part_path = part_path0 # 同一目标文件的下载者串行化（不同文件之间仍然并行） - with get_file_lock(file_path): + async with get_file_lock(file_path): try: - # 其它线程可能刚好完成此文件，锁内复查（命中即返回，不再下载） - cached = self._cached_result(file_path, music_info, - write_meta, meta_fields) + # 其它任务可能刚好完成此文件，锁内复查（命中即返回，不再下载） + cached = await self._cached_result(file_path, music_info, + write_meta, meta_fields) if cached is not None: return cached # 上次已下完但未改名的 .part：直接转正，不发起网络请求 - promoted = self._promote_complete_part( + promoted = await self._promote_complete_part( part_path, file_path, music_info, write_meta, meta_fields) if promoted is not None: return promoted # part_path 已在文件锁外绑定为 part_path0（同一路径） - self._transfer(music_info, part_path, - self._resume_offset(part_path, music_info), - cancel_event, progress_cb) + await self._transfer(music_info, part_path, + self._resume_offset(part_path, music_info), + cancel_event, progress_cb) # 完整性校验（已知期望大小时必须精确相等；未知期望时至少非空） mismatch = self._size_mismatch_error(part_path, music_info) @@ -347,8 +347,8 @@ class MusicDownloader: continue # 完整性通过 → 嗅探 → 原子转正 → 写元数据 - return self._publish(part_path, file_path, music_info, - cancel_event, write_meta, meta_fields) + return await self._publish(part_path, file_path, music_info, + cancel_event, write_meta, meta_fields) except DownloadCanceled: # 用户明确放弃：清理半截临时文件 @@ -363,7 +363,7 @@ class MusicDownloader: last_error = "续传起点失效，重新下载" next_wait = self._backoff_wait(attempt, retries) continue - except (requests.RequestException, OSError) as e: + except (UpstreamError, OSError) as e: # 网络错误：保留 .part 支持断点续传； # 但若任务已被取消，.part 没有任何续传价值，一并清理 if cancel_event is not None and cancel_event.is_set(): @@ -385,7 +385,7 @@ class MusicDownloader: # ----------- 下载分步实现（download_music_file 的组成件） ----------- - def _cached_result( + async def _cached_result( self, file_path: Path, music_info: MusicInfo, @@ -406,9 +406,9 @@ class MusicDownloader: if cache_bad: self._unlink_quiet(file_path) return None - # 缓存命中也尊重本次的元数据请求（重跑一次 ffmpeg，幂等） + # 缓存命中也尊重本次的元数据请求（重跑一次写标签，幂等） if write_meta: - self._write_metadata_locked(file_path, music_info, meta_fields) + await self._write_metadata_locked(file_path, music_info, meta_fields) return DownloadResult( success=True, file_path=str(file_path), @@ -416,7 +416,7 @@ class MusicDownloader: music_info=music_info, ) - def _promote_complete_part( + async def _promote_complete_part( self, part_path: Path, file_path: Path, @@ -437,7 +437,7 @@ class MusicDownloader: return None os.replace(part_path, file_path) if write_meta: - self._write_metadata_locked(file_path, music_info, meta_fields) + await self._write_metadata_locked(file_path, music_info, meta_fields) return DownloadResult( success=True, file_path=str(file_path), @@ -460,7 +460,7 @@ class MusicDownloader: return 0 return existing - def _transfer( + async def _transfer( self, music_info: MusicInfo, part_path: Path, @@ -474,18 +474,20 @@ class MusicDownloader: """ # 排队用可取消等待：原阻塞式 with 使取消后仍要等全局 10 名额腾出， # 任务卡死、退款延迟 - if not self._download_semaphore.acquire_cancellable(cancel_event, poll=0.5): + if not await self._download_semaphore.acquire_cancellable(cancel_event, poll=0.5): raise DownloadCanceled("下载已取消") try: - aria2_backend = get_backend() + # get_backend 首次调用要拉起 aria2c 并等 RPC 就绪（最长 8s）， + # 经 to_thread 卸载，不在事件循环上原地等 + aria2_backend = await asyncio.to_thread(get_backend) if aria2_backend is not None: # aria2c 通道：多连接 + 原生断点续传 - self._fetch_via_aria2(aria2_backend, music_info, part_path, - cancel_event, progress_cb) + await self._fetch_via_aria2(aria2_backend, music_info, part_path, + cancel_event, progress_cb) else: # 内置通道：aiohttp 流式落盘 + Range 起点校验 - self._fetch_via_http(music_info, part_path, resume_from, - cancel_event, progress_cb) + await self._fetch_via_http(music_info, part_path, resume_from, + cancel_event, progress_cb) finally: # 与 acquire_cancellable 配对释放（原 with 语义） self._download_semaphore.release() @@ -506,7 +508,7 @@ class MusicDownloader: return (f"下载不完整: 期望 {music_info.file_size or '未知'} 字节, " f"实际 {actual_size} 字节") - def _publish( + async def _publish( self, part_path: Path, file_path: Path, @@ -527,14 +529,14 @@ class MusicDownloader: os.replace(part_path, file_path) - # 元数据写入移出下载信号量（CPU 操作不占网络名额）， + # 元数据写入移出下载闸（CPU 操作不占网络名额）， # 但仍在文件锁内执行，保证其它请求看到的是已写入标签的最终文件 if write_meta: # 元数据段入口补取消检查点——已取消的任务不再 - # 进入信号量排队/ffmpeg 等待（有界但可达 120s），收尾不再延迟 + # 进入标签闸排队/写盘等待（有界但可达数十秒），收尾不再延迟 if cancel_event is not None and cancel_event.is_set(): raise DownloadCanceled("下载已取消") - self._write_metadata_locked(file_path, music_info, meta_fields) + await self._write_metadata_locked(file_path, music_info, meta_fields) return DownloadResult( success=True, @@ -543,26 +545,26 @@ class MusicDownloader: music_info=music_info, ) - def _write_metadata_locked( + async def _write_metadata_locked( self, file_path: Path, music_info: MusicInfo, meta_fields: list[str] | None, ) -> None: - """在标签信号量内写元数据：写标签会重写整个文件（IO 密集），与下载分离 + """在标签闸内写元数据：写标签会重写整个文件（IO 密集），与下载分离 限流——写满时只阻塞后来者，不占下载名额、不拖慢其它文件的传输。""" - with self._tag_semaphore: - self._write_metadata(file_path, music_info, meta_fields) + async with self._tag_semaphore: + await self._write_metadata(file_path, music_info, meta_fields) @staticmethod def _backoff_wait(attempt: int, retries: int) -> float: - """失败后的退避秒数；已是最后一轮则不等待。锁外睡眠——同名并发任务不陪睡。""" + """失败后的退避秒数；已是最后一轮则不等待。锁外等待——同名并发任务不陪睡。""" return 0.5 * (attempt + 1) if attempt < retries else 0.0 # ----------- 传输通道（二选一，产物都是 part_path） ----------- - def _fetch_via_http( + async def _fetch_via_http( self, music_info: MusicInfo, part_path: Path, @@ -570,7 +572,7 @@ class MusicDownloader: cancel_event: threading.Event | None, progress_cb, ) -> None: - """内置流式通道：core.sync_http 逐块落盘，支持 Range 断点续传。 + """内置流式通道：core.http_client 逐块落盘，支持 Range 断点续传。 四步：构造 Range 头 → 发请求 → 定追加/重写并校验分片起点 → 流式落盘 （含体积上限与取消）。HTTP 错误在此分类成「确定性失败」或交外层重试。 @@ -579,23 +581,22 @@ class MusicDownloader: DownloadCanceled: 取消事件置位 DownloadException: 体积超限、CDN 4xx 等确定性失败 _RetryFromScratch: 续传起点失效（残片已清，外层从头重下） + UpstreamError: 网络/5xx 等可重试失败（保留残片续传） """ expected_size = music_info.file_size or 0 self._safe_progress(progress_cb, resume_from, expected_size) - resp: requests.SyncResponse | None = None + resp = None try: - resp = get_http_session().get( - music_info.download_url, - stream=True, - timeout=(10, 60), - headers=self._range_headers(resume_from), - ) + headers = self._range_headers(resume_from) + resp = await _http.open('GET', music_info.download_url, + headers=headers or None, timeout=60) mode = self._resolve_write_mode(resp, resume_from, part_path) - resp.raise_for_status() - self._stream_body(resp, part_path, mode, resume_from, - expected_size, cancel_event, progress_cb) - except requests.HTTPError as e: - status = self._http_status(e) + if resp.status >= 400: + raise UpstreamHTTPError(resp.status, resp.reason or '') + await self._stream_body(resp, part_path, mode, resume_from, + expected_size, cancel_event, progress_cb) + except UpstreamHTTPError as e: + status = e.status if status == 416: # Range 起点无效（如 CDN 环节变化）：清除旧 part，从头重新下载 self._unlink_quiet(part_path) @@ -622,7 +623,7 @@ class MusicDownloader: 服务器忽略 Range 返回 200 时若继续追加，会把整段拼在残片后面；206 还须 校验分片起点（见 _verify_content_range）。 """ - if resume_from <= 0 or resp.status_code != 206: + if resume_from <= 0 or resp.status != 206: return "wb" self._verify_content_range(resp, resume_from, part_path) return "ab" @@ -643,7 +644,7 @@ class MusicDownloader: raise _RetryFromScratch( f"Content-Range 起点错位（{m.group(1)} != {resume_from}），已从头重新下载") - def _stream_body( + async def _stream_body( self, resp, part_path: Path, @@ -662,7 +663,7 @@ class MusicDownloader: written = resume_from if mode == "ab" else 0 over_limit = False with open(part_path, mode) as f: - for chunk in resp.iter_content(chunk_size=DL_CHUNK_SIZE): + async for chunk in iter_chunks(resp, DL_CHUNK_SIZE): if cancel_event is not None and cancel_event.is_set(): raise DownloadCanceled("下载已取消") if not chunk: @@ -688,11 +689,6 @@ class MusicDownloader: except Exception: pass - @staticmethod - def _http_status(exc: BaseException) -> int | None: - """从 HTTPError 上取响应状态码（无响应对象时返回 None）。""" - return getattr(getattr(exc, "response", None), "status_code", None) - @staticmethod def _is_deterministic_client_error(status: int | None) -> bool: """确定性 4xx（403 防盗链 / 429 风控等）：对同一失效直链重试只会加剧风控、 @@ -700,7 +696,7 @@ class MusicDownloader: """ return status is not None and 400 <= status < 500 and status != 408 - def _fetch_via_aria2( + async def _fetch_via_aria2( self, backend: Aria2Backend, music_info: MusicInfo, @@ -711,10 +707,10 @@ class MusicDownloader: """aria2c 通道：多连接加速 + 原生断点续传。 异常语义与内置通道对齐：取消抛 DownloadCanceled、体积超限抛 - DownloadException（均清残片），网络类失败抛 RequestException + DownloadException（均清残片），网络类失败抛 UpstreamNetworkError （保留残片，由外层退避重试续传）。 """ - backend.fetch( + await backend.fetch( music_info.download_url, part_path, cancel_event=cancel_event, @@ -728,7 +724,7 @@ class MusicDownloader: # 三个方法已删除——全服务端与验证脚本零调用（CLI 遗留），且 to_memory 不接受 # cookies 参数、真被启用会有行为差异；如需内存下载/异步批量再按当前接口重写 - def _write_metadata( + async def _write_metadata( self, file_path: Path, music_info: MusicInfo, @@ -743,11 +739,13 @@ class MusicDownloader: 封面走共享缓存（与 ZIP 打包共用一次网络请求）；各容器的落盘细节见 tagging.py——按扩展名分派到 ID3v2.3 / Vorbis comment / iTunes ilst。 + 写盘是阻塞 IO + 纯 Python CPU（mutagen 全量重写文件），经 to_thread 卸载。 """ include_cover = fields is None or "cover" in fields cover: bytes | None = None if include_cover and music_info.pic_url: - cover = fetch_cover_bytes(music_info.pic_url) + cover = await fetch_cover_bytes(music_info.pic_url) if not cover: logger.warning("封面下载失败，跳过封面写入") - return write_tags(file_path, build_tags(music_info, fields), cover) + return await asyncio.to_thread( + write_tags, file_path, build_tags(music_info, fields), cover) diff --git a/music_downloader/models.py b/music_downloader/models.py index ff22c6a..379d25c 100644 --- a/music_downloader/models.py +++ b/music_downloader/models.py @@ -9,8 +9,6 @@ from __future__ import annotations from dataclasses import dataclass from enum import Enum -from core import sync_http as requests # 同步门面（底层 aiohttp，替代 requests） - # ============ 类型定义 ============ class QualityLevel(Enum): @@ -64,8 +62,14 @@ class DownloadCanceled(DownloadException): pass -class _RetryFromScratch(requests.RequestException): - """内部信号：断点续传起点失效（HTTP 416 等），清除 .part 后从头重试""" +class _RetryFromScratch(Exception): + """内部信号：断点续传起点失效（HTTP 416 / Content-Range 错位），清除 .part 后从头重试。 + + 刻意**不**继承 DownloadException 或 UpstreamError：外层 + ``except DownloadException`` 的语义是"确定性失败，不重试"，本信号恰恰要求 + 从头重来；``except UpstreamError`` 的语义是"网络失败，保留 .part 续传"， + 而本信号已经把残片清掉了。两条语义都不是它，故独立成类、单独先于二者捕获。 + """ pass diff --git a/pyproject.toml b/pyproject.toml index b4c561b..cc71899 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,19 @@ ignore = [ # ASYNC109：timeout 参数是有意 API——直接转发给 aiohttp 的 ClientTimeout， # 不是「用 sleep 当超时」的反模式。 "core/http_client.py" = ["ASYNC109"] +# ASYNC109：timeout 是**总下载时限**（交给 aria2 轮询循环判定），不是「用 sleep 当超时」。 +"music_downloader/_aria2.py" = ["ASYNC109"] +# ASYNC230/240：残留的阻塞文件系统调用都是**本地元数据/短写**——exists()/stat()/ +# os.replace()/往 .part 顺序写分块，微秒级且不碰网络，卸载到线程的调度开销大于操作 +# 本身。真正的重活（mutagen 全量重写标签、zipfile 整段拷贝、shutil.rmtree、同步 +# SQLite 落库、aria2p 的同步 RPC）都已各自 to_thread 卸载。 +"music_downloader/downloader.py" = ["ASYNC230", "ASYNC240"] +"services/podcast.py" = ["ASYNC230", "ASYNC240"] +"services/batch.py" = ["ASYNC240"] +# ASYNC110：暂停闸门轮询的是**跨线程**置位的 threading.Event 与任务表状态——置位方 +# 在请求线程/监控线程，不属于本事件循环，故不能用 asyncio.Event 取代（见 _taskcore +# 模块 docstring 的取消/暂停载体说明）。 +"services/_taskcore.py" = ["ASYNC110"] [tool.ruff.lint.isort] known-first-party = ["core", "web", "services", "music_api", "auth_system", "music_downloader"] diff --git a/services/__init__.py b/services/__init__.py index d95dcd7..d6501a8 100644 --- a/services/__init__.py +++ b/services/__init__.py @@ -10,8 +10,11 @@ - ``downloads``：单曲下载、账号探测与任务交付 - ``batch``：歌单/专辑批量打包 worker - ``podcast``：播客单集下载与整台打包 worker -- ``streaming``：同步流式响应垫片（音频分段透传） - ``tasks``：任务监控循环、设备心跳与缓存清扫 + +worker 与打包引擎都是协程，由 ``_taskcore._start_task`` 作为 asyncio 任务挂到 +应用事件循环上（不再起线程）；监控循环仍是一条常驻线程——它每轮只做周期性的 +文件/库维护，不参与请求链路，留在线程里既不占事件循环也不额外引入 to_thread。 """ # ============ 依赖自检 ============ diff --git a/services/_taskcore.py b/services/_taskcore.py index e9bb119..36b5531 100644 --- a/services/_taskcore.py +++ b/services/_taskcore.py @@ -1,7 +1,17 @@ # -*- coding: utf-8 -*- """任务生命周期原语（启动 / 终态 / 文件收尾 / 结算）。 + +worker 自本轮起是协程：``_start_task`` 把 ``worker(*args)`` 包成 asyncio 任务挂到 +应用事件循环上（原先起一条线程）。任务字典里的句柄随之由 ``worker_thread`` 换成 +``worker_task``，监控循环对「worker 已结束但没人走终态结算」的兜底判断改为 +``task.done()``——语义不变（settle 幂等，无双退）。 + +取消/暂停位仍是 ``threading.Event``：置位方可能是监控线程或请求线程，读取方是 +worker 协程，只有"置位"与"轮询读取"两种操作，既不需要跨循环唤醒也不会被事件 +循环卡住，比 asyncio.Event 更省心。 """ +import asyncio import threading import time import traceback @@ -21,20 +31,31 @@ from .api_service import ( api_service, ) +# 存活期的后台任务强引用——asyncio 只持弱引用，不留引用的话任务可能中途被 GC +_BACKGROUND_TASKS: set[asyncio.Task] = set() -def _start_task_thread(task_id: str, target, args=()) -> str | None: + +def _log_worker_crash(handle: asyncio.Task) -> None: + """worker 的异常逃逸到任务层时留痕。 + + 正常情况下 worker 自带四路终态收敛，不会抛到这里；真抛出来说明收尾路径本身 + 出了问题，必须留下 traceback，否则任务永远停在 running 且无人知晓。 + """ + if handle.cancelled(): + return + exc = handle.exception() + if exc is not None: + api_service.logger.error(f"任务 worker 异常退出: {exc}", exc_info=exc) + + +def _start_task(task_id: str, worker, args: tuple = ()) -> str | None: """启动任务 worker（启动失败时收敛任务终态并按账本退费，不留僵尸 queued 记录）。 退款只走账本（本处 settle 后装饰器看到账已关，不会重复退）。 - 线程句柄存入任务字典——监控循环对 4h 超时任务若发现 worker 线程已死 + 任务句柄存入任务字典——监控循环对 4h 超时任务若发现 worker 已结束 （无法自行走到终态结算），可代为按账本结算退款（settle 幂等，无双退）。""" try: - th = threading.Thread(target=target, args=args, daemon=True) - with DOWNLOAD_TASKS_LOCK: - t = DOWNLOAD_TASKS.get(task_id) - if t is not None: - t['worker_thread'] = th - th.start() - return None + loop = asyncio.get_running_loop() + handle = loop.create_task(worker(*args), name=f'task:{task_id}') except Exception as e: with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS.pop(task_id, None) @@ -43,14 +64,22 @@ def _start_task_thread(task_id: str, target, args=()) -> str | None: auth_system.ledger_settle(task_id, 0, 'submit_failed') except Exception: pass - api_service.logger.error(f"任务线程启动失败: {task_id} {e}") + api_service.logger.error(f"任务启动失败: {task_id} {e}") return f'任务启动失败：{e}' + _BACKGROUND_TASKS.add(handle) + handle.add_done_callback(_BACKGROUND_TASKS.discard) + handle.add_done_callback(_log_worker_crash) + with DOWNLOAD_TASKS_LOCK: + t = DOWNLOAD_TASKS.get(task_id) + if t is not None: + t['worker_task'] = handle + return None -def _mark_task_terminal(task_id: str, status: str, msg: str = '') -> None: +async def _mark_task_terminal(task_id: str, status: str, msg: str = '') -> None: """任务进入终态的统一入口（记录 completed_at，保留期从完成时刻起算， 而不是从提交时刻起算）。 - /F-E：置 error/canceled 终态时若 worker 线程已死（无人再走正常结算）， + /F-E：置 error/canceled 终态时若 worker 已结束（无人再走正常结算）， 代为按账本结算退款——ledger_settle 幂等，worker 已自行结算过则此处为 0。 终态先到先得——已终态（done/error/canceled）的任务 不再被覆盖（堵住 canceled→error 的非法迁移边：取消后解析阶段的自身失败 @@ -65,15 +94,15 @@ def _mark_task_terminal(task_id: str, status: str, msg: str = '') -> None: t['msg'] = msg or t.get('msg', '') t['completed_at'] = time.time() if status in ('canceled', 'error'): - th = t.get('worker_thread') - if th is not None and not th.is_alive(): + wt = t.get('worker_task') + if wt is not None and wt.done(): need_settle = True - # settle 移到任务锁外——ledger_settle 内部要抢 _DB_LOCK 写库 + # settle 移到任务锁外——ledger_settle 内部要抢 SQLite 写锁 # （busy 最长 15s），持任务锁调用会让全站进度/提交/取消/领取随库停摆； - # settle 幂等，锁外调用无双退 + # settle 幂等，锁外调用无双退。落库是同步阻塞调用，经 to_thread 卸载 if need_settle: try: - auth_system.ledger_settle(task_id, 0, status) + await asyncio.to_thread(auth_system.ledger_settle, task_id, 0, status) except Exception: pass @@ -88,11 +117,12 @@ def _task_files(task: dict[str, Any]) -> list[str]: return files -def _settle_batch_billing(task_id: str, ok_count: int, final_status: str = 'settled') -> int: +async def _settle_batch_billing(task_id: str, ok_count: int, final_status: str = 'settled') -> int: """批量任务结算（走持久账本，按实际预扣-实际交付幂等退款； 重复调用/重启后调用都只会退一次）""" try: - refund = auth_system.ledger_settle(task_id, ok_count, final_status) + refund = await asyncio.to_thread( + auth_system.ledger_settle, task_id, ok_count, final_status) except Exception: # 本函数多在异常处理器内被调用——结算再抛错（库忙超时） # 会杀死 worker 收尾路径，跳过终态标记；失败留周期对账兜底（幂等） @@ -104,11 +134,12 @@ def _settle_batch_billing(task_id: str, ok_count: int, final_status: str = 'sett with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS.get(task_id) or {} uid = int(t.get('billing_user_id') or 0) - u = auth_system.get_user_by_id(uid) if uid else None + u = await asyncio.to_thread(auth_system.get_user_by_id, uid) if uid else None uname = u['username'] if u else 'system' remaining = u['duration_seconds'] if u else '?' - auth_system.add_log('success', uname, 'system', '批量退费', - f'失败部分 {refund} 次已退回，剩余 {remaining} 次') + await asyncio.to_thread( + auth_system.add_log, 'success', uname, 'system', '批量退费', + f'失败部分 {refund} 次已退回，剩余 {remaining} 次') except Exception: pass return refund @@ -127,8 +158,8 @@ def _task_work_dir(task_id: str) -> Path: return path -def _wait_if_paused(task: dict[str, Any], pause_ev: threading.Event, - cancel_ev: threading.Event, done: int) -> None: +async def _wait_if_paused(task: dict[str, Any], pause_ev: threading.Event, + cancel_ev: threading.Event, done: int) -> None: """断线暂停闸门：置 paused 态并等待恢复或取消，恢复后回到 running。 两个打包 worker（音乐歌单 / 播客整台）共用同一套暂停语义——此前各自持有 @@ -147,7 +178,7 @@ def _wait_if_paused(task: dict[str, Any], pause_ev: threading.Event, task['done'] = done task['msg'] = '已暂停（重新打开页面自动恢复）' while not cancel_ev.is_set() and (pause_ev.is_set() or task.get('status') == 'paused'): - time.sleep(1) + await asyncio.sleep(1) if cancel_ev.is_set(): raise DownloadCanceled("任务已取消") pause_ev.clear() diff --git a/services/_zipcore.py b/services/_zipcore.py index 8bf6484..fb3cdf3 100644 --- a/services/_zipcore.py +++ b/services/_zipcore.py @@ -13,25 +13,25 @@ 术语：下文「条目」指一个下载单元——音乐歌单里是一首歌，播客整台里是一集。 -线程模型（本轮最容易踩坏的地方，改动前请先读）： -- 下载 I/O 在 ``ThreadPoolExecutor`` 里跑，窗口大小 = ``job.concurrency``； -- **ZIP 写入与包内名去重只发生在调用线程**（``as_completed`` 消费结果时）， - :class:`zipfile.ZipFile` 不是线程安全的，且去重读改写 ``used_arcnames`` - 需要串行——故下载与落包被拆成 ``download_one`` / ``write_entry`` 两个钩子， - 只有前者进线程池。 +并发模型（本轮最容易踩坏的地方，改动前请先读）： +- 一个窗口 = ``job.concurrency`` 个条目，窗口内 ``asyncio.gather`` 并发下载； +- **ZIP 写入与包内名去重只在引擎协程里串行发生**（按窗口顺序消费结果）—— + :class:`zipfile.ZipFile` 不是并发安全的，且去重读改写 ``used_arcnames`` + 需要串行；因此 ``download_one``（并发）与 ``write_entry``（串行）是两个钩子， + 只有前者进 gather。``write_entry`` 里的重型落盘（几百 MB 的音频条目） + 由实现方自行经 ``asyncio.to_thread`` 卸载，避免卡住事件循环。 ``run_zip_job`` 的四个终态（成功 / 全失败 / 取消 / 异常）原先各写一段 8~20 行的 收尾内联在 try/except 里。此处把「一次运行」的状态收进 :class:`_ZipRun`， 四个终态各自成函数——改账本口径或补一种终态时，不必再在 167 行里找落点。 """ +import asyncio import os import tempfile -import threading import time import traceback import zipfile -from collections.abc import Callable -from concurrent.futures import ThreadPoolExecutor, as_completed +from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -80,7 +80,7 @@ class ZipContext: task_id: str params: dict[str, Any] - cancel_ev: threading.Event + cancel_ev: Any work_dir: Path zf: zipfile.ZipFile progress: Callable[..., None] @@ -106,7 +106,7 @@ class ZipJob: 子类至少实现 :meth:`prepare` / :meth:`download_one` / :meth:`write_entry` / :meth:`zip_filename`，其余钩子按需覆写。**实例每次运行新建**——下载器信息 - 预取池等运行期状态就挂在实例上。 + 预取任务等运行期状态就挂在实例上。 """ tag = '批量打包' # 日志前缀 @@ -119,7 +119,7 @@ class ZipJob: # ---- 必须实现 ---- - def prepare(self, params: dict[str, Any], task_id: str, failed: list[str]) -> list[Any]: + async def prepare(self, params: dict[str, Any], task_id: str, failed: list[str]) -> list[Any]: """组装条目清单。可抛 ``DownloadException`` 让任务按业务失败收敛。 ``failed`` 可被直接追加——清单阶段就有条目已知失败（如按 ID 列表时个别 @@ -127,8 +127,8 @@ class ZipJob: """ raise NotImplementedError - def download_one(self, item: Any, idx: int, ctx: ZipContext) -> Any: - """下载单个条目（**在下载线程池内执行**）。 + async def download_one(self, item: Any, idx: int, ctx: ZipContext) -> Any: + """下载单个条目（**在窗口内并发执行**）。 返回的 ``payload`` 会原样交给 :meth:`write_entry`（放临时文件名、 已取到的元信息等）。失败请抛异常：``ItemFailed`` 用其自带文案， @@ -136,10 +136,12 @@ class ZipJob: """ raise NotImplementedError - def write_entry(self, item: Any, idx: int, payload: Any, ctx: ZipContext) -> None: - """把下载结果写入 ZIP（**在引擎调用线程内执行，天然串行**）。 + async def write_entry(self, item: Any, idx: int, payload: Any, ctx: ZipContext) -> None: + """把下载结果写入 ZIP（**由引擎按窗口顺序串行调用**）。 包内名去重在这里做——需要读写 ``ctx.used_arcnames``，必须串行。 + 重型落盘请自行 ``await asyncio.to_thread(...)``：直接在协程里写 + 几百 MB 会把事件循环卡住。 """ raise NotImplementedError @@ -149,7 +151,7 @@ class ZipJob: # ---- 可选钩子 ---- - def prefetch(self, window: list[Any], ctx: ZipContext) -> None: + async def prefetch(self, window: list[Any], ctx: ZipContext) -> None: """窗口级预热（每窗口一次、提交下载前调用）。默认无操作。""" def fail_label(self, item: Any) -> str: @@ -174,14 +176,14 @@ class ZipJob: return f"部分{self.item_noun}已打包完成，可领取" + ( f"，退回 {refunded} 次" if refunded else '') - def supplement(self, task_id: str, params: dict[str, Any], count: int) -> None: + async def supplement(self, task_id: str, params: dict[str, Any], count: int) -> None: """清单确定后的计费补扣（默认无操作）。""" - def on_delivered(self, items: list[Any], task_id: str, params: dict[str, Any]) -> None: + async def on_delivered(self, items: list[Any], task_id: str, params: dict[str, Any]) -> None: """交付落定后的操作记录（默认无操作）。""" - def cleanup(self) -> None: - """释放运行期资源（如信息预取池）。默认无操作。""" + async def cleanup(self) -> None: + """释放运行期资源（如信息预取任务）。默认无操作。""" def _discard(path: str | None) -> None: @@ -206,12 +208,12 @@ class _ZipRun: job: ZipJob task_id: str - settle: Callable[[str, int, str], int] + settle: Callable[[str, int, str], Awaitable[int]] tally: _Tally = field(default_factory=_Tally) task: dict[str, Any] = field(default_factory=dict) params: dict[str, Any] = field(default_factory=dict) - cancel_ev: threading.Event | None = None - pause_ev: threading.Event | None = None + cancel_ev: Any = None + pause_ev: Any = None items: list[Any] = field(default_factory=list) zip_tmp_path: str | None = None zip_valid: bool = False @@ -232,7 +234,7 @@ def _progress_of(run: _ZipRun) -> Callable[..., None]: return _progress -def _start_zip_run(run: _ZipRun) -> None: +async def _start_zip_run(run: _ZipRun) -> None: """置 running、组装清单、写回 total、执行计费补扣。""" with DOWNLOAD_TASKS_LOCK: task = DOWNLOAD_TASKS[run.task_id] @@ -245,13 +247,13 @@ def _start_zip_run(run: _ZipRun) -> None: # 计耗时只算「下载+打包」这一段（不含排队），故不用 task['start'] run.t0 = time.time() - run.items = run.job.prepare(run.params, run.task_id, run.tally.failed) + run.items = await run.job.prepare(run.params, run.task_id, run.tally.failed) with DOWNLOAD_TASKS_LOCK: DOWNLOAD_TASKS[run.task_id]['total'] = len(run.items) - run.job.supplement(run.task_id, run.params, len(run.items)) + await run.job.supplement(run.task_id, run.params, len(run.items)) -def _build_zip_archive(run: _ZipRun) -> None: +async def _build_zip_archive(run: _ZipRun) -> None: """建临时 ZIP 并逐窗口填充。返回时 ``zip_valid`` 才置位。 ZIP 构建放 downloads 卷（原实现落系统 %TEMP%——500 首无损可达 GB 级， @@ -269,7 +271,7 @@ def _build_zip_archive(run: _ZipRun) -> None: ctx = ZipContext(task_id=run.task_id, params=run.params, cancel_ev=run.cancel_ev, work_dir=work_dir, zf=zf, progress=_progress_of(run), total=len(run.items)) - _fill_zip(job, run.items, ctx, run.tally, run.task, run.pause_ev) + await _fill_zip(job, run.items, ctx, run.tally, run.task, run.pause_ev) # zip_valid 只在 ZIP with 块正常退出（中央目录完整写出）后才置 True。 # 此前两个 worker 在这之后还各留了一段 `if not zip_valid:` 失败处理—— # 紧跟赋值、中间无任何可改zip_valid的语句，是**不可达**的复审残留，已删。 @@ -280,7 +282,7 @@ def _build_zip_archive(run: _ZipRun) -> None: BATCH_DURATIONS.append(time.time() - run.t0) -def _finish_all_failed(run: _ZipRun) -> None: +async def _finish_all_failed(run: _ZipRun) -> None: """全部失败（含清单非空但一条都没成）：不发布空 ZIP、不按交付结算，转 error 并全额退费。 原音乐批量用 ``ok==0 and failed``、播客用早抛 ``DownloadException``——这里 @@ -289,8 +291,8 @@ def _finish_all_failed(run: _ZipRun) -> None: job, tally = run.job, run.tally _discard(run.zip_tmp_path) run.zip_tmp_path = None - refunded = run.settle(run.task_id, 0, 'error') - _mark_task_terminal(run.task_id, 'error', job.all_failed_msg(tally.failed, refunded)) + refunded = await run.settle(run.task_id, 0, 'error') + await _mark_task_terminal(run.task_id, 'error', job.all_failed_msg(tally.failed, refunded)) with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS.get(run.task_id) if t is not None: @@ -300,7 +302,7 @@ def _finish_all_failed(run: _ZipRun) -> None: f"{job.tag}全部失败: {run.task_id} 失败{len(tally.failed)} 退回{refunded}次") -def _publish_delivery(run: _ZipRun) -> None: +async def _publish_delivery(run: _ZipRun) -> None: """正常交付：按入包成功数结算，锁内复查取消/终态后发布结果。 ``'settled'`` 是 ``_settle_batch_billing`` 的默认值，此处显式写出，让 @@ -308,7 +310,7 @@ def _publish_delivery(run: _ZipRun) -> None: """ job, tally = run.job, run.tally assert run.cancel_ev is not None - refunded = run.settle(run.task_id, tally.ok, 'settled') + refunded = await run.settle(run.task_id, tally.ok, 'settled') with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS.get(run.task_id) # 发布前锁内复查取消/失效状态——取消请求落在「检查后、拿锁前」的窗口 @@ -329,24 +331,24 @@ def _publish_delivery(run: _ZipRun) -> None: 'filename': job.zip_filename(tally.ok), 'audio_files': [], # 缓存文件不归任务所有，不进入清理清单 } - job.on_delivered(run.items, run.task_id, run.params) + await job.on_delivered(run.items, run.task_id, run.params) api_service.logger.info( f"{job.tag}完成: {job.log_label()}成功{tally.ok} 失败{len(tally.failed)} " f"退回{refunded}次 ({run.task_id})") -def _finish_canceled(run: _ZipRun) -> None: +async def _finish_canceled(run: _ZipRun) -> None: """部分成功后取消——已入包部分照常交付（成果守恒），未完成部分按账本退费。 ZIP 未正常完成（写坏）时不按 ``ok_count`` 交付——按全额退费收敛。 """ job, tally = run.job, run.tally delivered = tally.ok if run.zip_valid else 0 - refunded = run.settle(run.task_id, delivered, 'canceled') + refunded = await run.settle(run.task_id, delivered, 'canceled') if delivered > 0 and run.zip_tmp_path: - _mark_task_terminal(run.task_id, 'canceled', - f"已取消（已打包 {tally.ok} {job.noun}可领取）" + - (f"，退回 {refunded} 次" if refunded else '')) + await _mark_task_terminal(run.task_id, 'canceled', + f"已取消（已打包 {tally.ok} {job.noun}可领取）" + + (f"，退回 {refunded} 次" if refunded else '')) with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS.get(run.task_id) if t is not None: @@ -358,9 +360,9 @@ def _finish_canceled(run: _ZipRun) -> None: } else: _discard(run.zip_tmp_path) - _mark_task_terminal(run.task_id, 'canceled', - '任务已取消' + ( - f"（已退回 {refunded} 次下载次数）" if refunded else '')) + await _mark_task_terminal(run.task_id, 'canceled', + '任务已取消' + ( + f"（已退回 {refunded} 次下载次数）" if refunded else '')) with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS.get(run.task_id) if t is not None: @@ -369,29 +371,29 @@ def _finish_canceled(run: _ZipRun) -> None: f"{job.tag}已取消: {run.task_id}（成功{tally.ok} 退回{refunded}次）") -def _finish_business_error(run: _ZipRun, exc: DownloadException) -> None: +async def _finish_business_error(run: _ZipRun, exc: DownloadException) -> None: """业务失败（清单阶段：无可下载条目 / 补扣下载次数失败 / 上游不可用）。 文案是我们自己拼的用户可读文本，保留原文（同播客单集下载）。 """ job, tally = run.job, run.tally delivered = tally.ok if run.zip_valid else 0 - refunded = run.settle(run.task_id, delivered, 'error') - _apply_error(run.task_id, str(exc), refunded) + refunded = await run.settle(run.task_id, delivered, 'error') + await _apply_error(run.task_id, str(exc), refunded) _discard(run.zip_tmp_path) api_service.logger.error( f"{job.tag}失败: {run.task_id} {exc}（成功{tally.ok} 退回{refunded}次）") -def _finish_unexpected(run: _ZipRun, exc: Exception) -> None: +async def _finish_unexpected(run: _ZipRun, exc: Exception) -> None: """非预期异常：原文 logger 先行（结算本身可能再抛，先落下 traceback）， 用户可见文案泛化；带上退款结果再补一条，避免排障时只看到「异常」。""" job, tally = run.job, run.tally api_service.logger.error(f"{job.tag}异常: {run.task_id} {exc}\n{traceback.format_exc()}") delivered = tally.ok if run.zip_valid else 0 - refunded = run.settle(run.task_id, delivered, 'error') + refunded = await run.settle(run.task_id, delivered, 'error') if delivered > 0 and run.zip_tmp_path: - _apply_error(run.task_id, job.partial_msg(refunded), refunded) + await _apply_error(run.task_id, job.partial_msg(refunded), refunded) with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS.get(run.task_id) if t is not None: @@ -401,16 +403,16 @@ def _finish_unexpected(run: _ZipRun, exc: Exception) -> None: 'audio_files': [], } else: - _apply_error(run.task_id, '打包失败，请稍后重试' + ( + await _apply_error(run.task_id, '打包失败，请稍后重试' + ( f"（已退回 {refunded} 次下载次数）" if refunded else ''), refunded) _discard(run.zip_tmp_path) api_service.logger.error( f"{job.tag}异常: {run.task_id}（成功{tally.ok} 退回{refunded}次）") -def run_zip_job(job: ZipJob, task_id: str, *, - settle: Callable[[str, int, str], int] = _settle_batch_billing) -> None: - """执行一次批量打包任务（在 worker 线程内调用，自身不向外抛异常）。 +async def run_zip_job(job: ZipJob, task_id: str, *, + settle: Callable[[str, int, str], Awaitable[int]] = _settle_batch_billing) -> None: + """执行一次批量打包任务（在 worker 协程内调用，自身不向外抛异常）。 ``settle`` 由调用方注入，默认走 :func:`services._taskcore._settle_batch_billing`。 做成参数是为了让测试能在**模块级**替换结算函数（包装层在调用时读取模块全局， @@ -424,99 +426,91 @@ def run_zip_job(job: ZipJob, task_id: str, *, """ run = _ZipRun(job=job, task_id=task_id, settle=settle) try: - _start_zip_run(run) - _build_zip_archive(run) + await _start_zip_run(run) + await _build_zip_archive(run) if run.tally.ok == 0: - _finish_all_failed(run) + await _finish_all_failed(run) return - _publish_delivery(run) + await _publish_delivery(run) except DownloadCanceled: - _finish_canceled(run) + await _finish_canceled(run) except DownloadException as e: - _finish_business_error(run, e) + await _finish_business_error(run, e) except Exception as e: - _finish_unexpected(run, e) + await _finish_unexpected(run, e) finally: - # 回收信息预取线程池，避免线程泄漏（未启动的预取任务一并取消） - job.cleanup() + # 回收信息预取任务，避免任务泄漏（未启动的预取任务一并取消） + await job.cleanup() -def _apply_error(task_id: str, msg: str, refunded: int) -> None: +async def _apply_error(task_id: str, msg: str, refunded: int) -> None: """异常/业务失败的终态收敛：置 error（终态先到先得）、记账退款额。 走 :func:`_mark_task_terminal` 而非就地写任务字典——后者是播客侧的旧写法， - 除了重复一遍「已终态不覆盖」判断外，还漏掉了 worker 线程已死时按账本补结算 - 的兜底（``/F-E``）。 + 除了重复一遍「已终态不覆盖」判断外，还漏掉了 worker 已结束未结算时按账本 + 补结算的兜底（``/F-E``）。 """ - _mark_task_terminal(task_id, 'error', msg) + await _mark_task_terminal(task_id, 'error', msg) with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS.get(task_id) if t is not None: t['refunded'] = refunded -def _fill_zip(job: ZipJob, items: list[Any], ctx: ZipContext, tally: _Tally, - task: dict[str, Any], pause_ev: threading.Event) -> None: +async def _fill_zip(job: ZipJob, items: list[Any], ctx: ZipContext, tally: _Tally, + task: dict[str, Any], pause_ev) -> None: """逐窗口下载并写入 ZIP（返回时中央目录尚未写出，由调用方的 with 退出时落盘）。 - 窗口 = ``job.concurrency`` 个条目：下载并行、落包串行。窗口大小取 1 时 + 窗口 = ``job.concurrency`` 个条目：下载并发、落包串行。窗口大小取 1 时 退化为顺序下载（播客整台即如此）。 + + 取消语义与旧的线程池版本对齐：等待名额期间可被取消；窗口内某条抛 + DownloadCanceled 时整批中止（在途下载因每次分块都查取消位而很快退出）。 """ concurrency = max(1, int(job.concurrency)) - pool: ThreadPoolExecutor | None = None - try: - # 不用 with 管理下载池——取消时 with 退出会 wait=True 等在途 future - # （其内部阻塞在预取 HTTP 上最长 30s），改为不等待 + 取消排队任务 - pool = ThreadPoolExecutor(max_workers=concurrency) - offset = 0 - while offset < len(items): - if ctx.cancel_ev.is_set(): - raise DownloadCanceled("任务已取消") - # 断线暂停：释放名额等待恢复/取消，恢复后重新进入排队 - _wait_if_paused(task, pause_ev, ctx.cancel_ev, tally.ok + len(tally.failed)) - # 每个窗口持有 BATCH_ZIP_SEM（同一时刻全局最多 N 个批量任务在下载） - # 等待名额期间可被取消，不再无限阻塞线程与退款 - if not BATCH_ZIP_SEM.acquire_cancellable(ctx.cancel_ev): - raise DownloadCanceled("任务已取消") - try: - # 暂停/取消落在「等待恢复返回」与「拿到名额」之间：立刻放名额回到 - # 循环顶部重新处理暂停（原播客侧缺这一步，会多下完这一集才停） - if (ctx.cancel_ev.is_set() or pause_ev.is_set() - or task['status'] == 'paused'): - continue - window = items[offset:offset + concurrency] - start_idx = offset + 1 # 条目序号从 1 起（进度文案用） - offset += len(window) - job.prefetch(window, ctx) # 只预取本窗口 - futures = { - pool.submit(job.download_one, item, start_idx + i, ctx): (item, start_idx + i) - for i, item in enumerate(window) - } - for future in as_completed(futures): - if ctx.cancel_ev.is_set(): - raise DownloadCanceled("任务已取消") - item, idx = futures[future] - try: - payload = future.result() - # 落包在主线程：ZipFile 非线程安全 + 包内名去重要串行 - job.write_entry(item, idx, payload, ctx) - except DownloadCanceled: - raise - except ItemFailed as f: - tally.failed.append(f.label) - except Exception as e: - tally.failed.append(f"{job.fail_label(item)}({e})") - else: - tally.ok += 1 - ctx.progress(job.done_msg(item, idx, ctx)) - finally: - BATCH_ZIP_SEM.release() - finally: - if pool is not None: - try: - # 取消/异常时不等待在途下载线程（其可能阻塞在预取 HTTP 上）， - # 排队任务一并取消；在途线程自行收尾（.part 残片由续传/清扫兜底） - pool.shutdown(wait=False, cancel_futures=True) - except TypeError: - pool.shutdown(wait=False) + offset = 0 + while offset < len(items): + if ctx.cancel_ev.is_set(): + raise DownloadCanceled("任务已取消") + # 断线暂停：释放名额等待恢复/取消，恢复后重新进入排队 + await _wait_if_paused(task, pause_ev, ctx.cancel_ev, tally.ok + len(tally.failed)) + # 每个窗口持有 BATCH_ZIP_SEM（同一时刻全局最多 N 个批量任务在下载） + # 等待名额期间可被取消，不再无限等待与延迟退款 + if not await BATCH_ZIP_SEM.acquire_cancellable(ctx.cancel_ev): + raise DownloadCanceled("任务已取消") + try: + # 暂停/取消落在「等待恢复返回」与「拿到名额」之间：立刻放名额回到 + # 循环顶部重新处理暂停（原播客侧缺这一步，会多下完这一集才停） + if (ctx.cancel_ev.is_set() or pause_ev.is_set() + or task['status'] == 'paused'): + continue + window = items[offset:offset + concurrency] + start_idx = offset + 1 # 条目序号从 1 起（进度文案用） + offset += len(window) + await job.prefetch(window, ctx) # 只预取本窗口 + results = await asyncio.gather( + *(job.download_one(item, start_idx + i, ctx) + for i, item in enumerate(window)), + return_exceptions=True, + ) + for i, result in enumerate(results): + if ctx.cancel_ev.is_set(): + raise DownloadCanceled("任务已取消") + item, idx = window[i], start_idx + i + try: + if isinstance(result, BaseException): + raise result + # 落包串行：ZipFile 非并发安全 + 包内名去重要按序 + await job.write_entry(item, idx, result, ctx) + except DownloadCanceled: + raise + except ItemFailed as f: + tally.failed.append(f.label) + except Exception as e: + tally.failed.append(f"{job.fail_label(item)}({e})") + else: + tally.ok += 1 + ctx.progress(job.done_msg(item, idx, ctx)) + finally: + BATCH_ZIP_SEM.release() diff --git a/services/batch.py b/services/batch.py index e214b71..44a1058 100644 --- a/services/batch.py +++ b/services/batch.py @@ -5,11 +5,8 @@ 本模块只保留「音乐歌单」这一侧的策略：窗口化预取与并行下载、每首三条目 （音频 + 歌词 + 封面）入包、同名去重、包名与文案。 """ +import asyncio import zipfile -from concurrent.futures import ( - Future, - ThreadPoolExecutor, -) from dataclasses import dataclass from pathlib import Path from typing import Any @@ -38,7 +35,7 @@ from .api_service import ( @dataclass(frozen=True) class _SongPayload: - """一首歌下载完成后的入包素材（下载线程产出，落包线程消费）。""" + """一首歌下载完成后的入包素材（下载协程产出，落包协程消费）。""" path: str name: str @@ -59,36 +56,41 @@ class _SonglistZipJob(ZipJob): record_duration = True def __init__(self) -> None: - self._info_pool: ThreadPoolExecutor | None = None - self._info_futures: dict[Any, Future[Any]] = {} + self._info_sem: asyncio.Semaphore | None = None + self._info_tasks: dict[Any, asyncio.Task[Any]] = {} - def prepare(self, params: dict[str, Any], task_id: str, failed: list[str]) -> list[Any]: + async def prepare(self, params: dict[str, Any], task_id: str, failed: list[str]) -> list[Any]: self.concurrency = int(params['concurrency']) return list(params['ids']) - def prefetch(self, window: list[Any], ctx: ZipContext) -> None: + async def prefetch(self, window: list[Any], ctx: ZipContext) -> None: """按窗口预取歌曲信息。 原实现一次性提交全部（最多 500 首 ×3 接口）预取任务，大歌单瞬间以 10 并发 连发上游约 1500 次请求打风控；改为每个下载窗口只预取本窗口的歌（与下载节奏 - 对齐），``download_one`` 语义不变（等预取结果再下载）。 + 对齐），并发另用信号量钳在 4，``download_one`` 语义不变（等预取结果再下载）。 """ - if self._info_pool is None: - self._info_pool = ThreadPoolExecutor(max_workers=min(4, max(1, self.concurrency))) + if self._info_sem is None: + self._info_sem = asyncio.Semaphore(min(4, max(1, self.concurrency))) quality = ctx.params['quality'] cookies = ctx.params['cookies'] for mid in window: - if mid not in self._info_futures: - self._info_futures[mid] = self._info_pool.submit( - api_service.downloader.get_music_info, mid, quality, cookies - ) + if mid not in self._info_tasks: + self._info_tasks[mid] = asyncio.create_task( + self._load_info(mid, quality, cookies)) - def download_one(self, item: Any, idx: int, ctx: ZipContext) -> _SongPayload: + async def _load_info(self, mid: Any, quality: str, cookies: dict[str, str]) -> Any: + """取单曲信息；并发由 _info_sem 钳制（避免预取风暴打风控）。""" + assert self._info_sem is not None + async with self._info_sem: + return await api_service.downloader.get_music_info(mid, quality, cookies) + + async def download_one(self, item: Any, idx: int, ctx: ZipContext) -> _SongPayload: """并行下载单元：等待预取信息 → 下载 → 复用预取歌词与缓存封面（零重复请求）。""" mid = item - info = self._info_futures[mid].result() + info = await self._info_tasks[mid] params = ctx.params - result = api_service.downloader.download_music_file( + result = await api_service.downloader.download_music_file( mid, params['quality'], write_meta=params['write_meta'], meta_fields=params['meta_fields'], @@ -103,7 +105,7 @@ class _SonglistZipJob(ZipJob): lyric = getattr(info, 'lyric', '') or '' # 封面走共享缓存（同一专辑只拉一次网络）；取消后不再发起新请求 if not ctx.cancel_ev.is_set() and info.pic_url: - cover = fetch_cover_bytes(info.pic_url) or b'' + cover = await fetch_cover_bytes(info.pic_url) or b'' if not result.success: raise ItemFailed(str(mid)) assert result.file_path is not None @@ -118,7 +120,7 @@ class _SonglistZipJob(ZipJob): return _SongPayload(path=result.file_path, name=mi.name, artists=mi.artists, file_type=mi.file_type, lyric=lyric, cover=cover) - def write_entry(self, item: Any, idx: int, payload: _SongPayload, ctx: ZipContext) -> None: + async def write_entry(self, item: Any, idx: int, payload: _SongPayload, ctx: ZipContext) -> None: mid = item base = _build_download_base(ctx.params['filename_format'], {'name': payload.name, 'artist_string': payload.artists}) @@ -139,13 +141,15 @@ class _SonglistZipJob(ZipJob): k += 1 ctx.used_arcnames.update((final_name, lrc_name, jpg_name)) # F-D：exists 与 zf.write 之间的缓存竞态（另一任务缓存 miss 删文件） - # 按单曲失败计，不再让整批转 error + # 按单曲失败计，不再让整批转 error。音频条目可达上百 MB，落盘经 to_thread + # 卸载，避免在事件循环里做整段文件拷贝 try: - ctx.zf.write(payload.path, arcname=final_name, compress_type=zipfile.ZIP_STORED) + await asyncio.to_thread(ctx.zf.write, payload.path, arcname=final_name, + compress_type=zipfile.ZIP_STORED) except OSError as e: raise ItemFailed(f"{mid}(缓存文件读取失败: {e})") from e # 缓存文件是跨任务共享资源，任务不得删除；磁盘回收交给监控线程的下载缓存 - # 清扫（按文件年龄）。歌词与封面已在并行下载线程中拉取，这里直接写包（无网络等待） + # 清扫（按文件年龄）。歌词与封面已在并行下载协程中拉取，这里直接写包（无网络等待） if payload.lyric: try: ctx.zf.writestr(lrc_name, _format_lyric(payload.lyric, lyric_format), @@ -161,19 +165,23 @@ class _SonglistZipJob(ZipJob): def zip_filename(self, ok_count: int) -> str: return f"批量下载_{ok_count}首.zip" - def cleanup(self) -> None: - if self._info_pool is not None: - try: - self._info_pool.shutdown(wait=False, cancel_futures=True) - except TypeError: - self._info_pool.shutdown(wait=False) + async def cleanup(self) -> None: + """取消在途预取任务（窗口因暂停/取消被跳过时它们没人 await）。 + + 取消后不再 await：asyncio 任务被 cancel 不会产生"异常未被取用"告警， + 而 await 会在此处引入最长一次上游请求的等待。 + """ + for task in self._info_tasks.values(): + if not task.done(): + task.cancel() + self._info_tasks.clear() -def _batch_zip_worker(task_id: str) -> None: +async def _batch_zip_worker(task_id: str) -> None: """后台批量打包：窗口化下载 → 写 ZIP（音频免压缩），支持取消与断线暂停 - 断线暂停：设备心跳超时后，当前窗口下载完即释放 BATCH_ZIP_SEM 名额给下一位， 保存进度进入暂停等待；重新打开网站（心跳恢复）后自动回到队列继续下载 - 音频入包即删，缓解磁盘占用 """ - run_zip_job(_SonglistZipJob(), task_id, settle=_settle_batch_billing) + await run_zip_job(_SonglistZipJob(), task_id, settle=_settle_batch_billing) diff --git a/services/downloads.py b/services/downloads.py index aef3b30..471198b 100644 --- a/services/downloads.py +++ b/services/downloads.py @@ -7,8 +7,13 @@ ``_single_download_worker`` 原是 177 行 / 圈复杂度 30 的单函数，三个 except 分支 各写一遍「结算 + 删任务目录」，正文里又散着四处取消检查。此处按「一步一个具名 函数」拆开：主函数只留线性流程，失败收尾收敛到 ``_fail_single_task`` 一处。 + +worker 自本轮起是协程：下载链路本身已异步，落库/删目录等阻塞调用经 +``asyncio.to_thread`` 卸载（auth_system 仍是同步 SQLite，DB 线异步化后即可 +直接 await）。 """ +import asyncio import os import shutil import threading @@ -114,13 +119,13 @@ def _begin_single_task(task_id: str) -> SingleTaskParams: ) -def _fetch_single_metadata(p: SingleTaskParams) -> tuple[MusicInfo, dict[str, Any], str, str]: - """一次预取歌曲完整信息（URL/详情/歌词三接口并行），并算出交付基础名。 +async def _fetch_single_metadata(p: SingleTaskParams) -> tuple[MusicInfo, dict[str, Any], str, str]: + """一次预取歌曲完整信息（URL/详情/歌词三接口并发），并算出交付基础名。 替代原先 name_v1 + url_v1 + 下载器内部 get_music_info 的重复串行调用。 返回 ``(MusicInfo, 下载器口径的 music_info, base, filename)``。 """ - mi = api_service.downloader.get_music_info(p.music_id, p.quality, p.cookies) + mi = await api_service.downloader.get_music_info(p.music_id, p.quality, p.cookies) music_info = { 'id': p.music_id, 'name': mi.name, @@ -188,37 +193,47 @@ def _single_file_result(file_path_str: str, filename: str, } -def _pack_single_zip(file_path_str: str, base: str, filename: str, mi: MusicInfo, - music_info: dict[str, Any], p: SingleTaskParams) -> dict[str, Any]: +async def _pack_single_zip(file_path_str: str, base: str, filename: str, mi: MusicInfo, + music_info: dict[str, Any], p: SingleTaskParams) -> dict[str, Any]: """把音频 + 歌词 + 封面 + info.txt 打成单曲 ZIP 交付包。 歌词与封面都是**尽力而为**：歌词在预取时已一并拿到（无需再请求一次）， 封面走共享缓存；两者拿不到就不写对应条目，不影响音频交付。 + ZIP 组装是整段文件拷贝的阻塞 IO，经 to_thread 卸载。 """ import tempfile as _tempfile + # 封面走共享缓存（与元数据写入共用同一次网络请求）——先取好再落包， + # 网络等待留在事件循环里（协程），文件写入整段交给线程 + cover: bytes = b'' + if mi.pic_url: + cover = await fetch_cover_bytes(mi.pic_url) or b'' + fd, zip_tmp_path = _tempfile.mkstemp(prefix='single_zip_', suffix='.zip', dir=str(Path(file_path_str).parent)) os.close(fd) - with zipfile.ZipFile(zip_tmp_path, 'w', zipfile.ZIP_DEFLATED) as zf: - zf.write(file_path_str, arcname=filename) - # 歌词已在预取时一并拿到，无需再请求一次 - if mi.lyric: - try: - zf.writestr(f"{base}.{p.lyric_format}", _format_lyric(mi.lyric, p.lyric_format)) - except Exception: - pass - # 封面走共享缓存（与元数据写入共用同一次网络请求） - cover = fetch_cover_bytes(mi.pic_url) - if cover: - zf.writestr(f"{base}.jpg", cover) - info_txt = ( - f"歌曲: {music_info['name']}\n" - f"歌手: {music_info['artist_string']}\n" - f"专辑: {music_info['album']}\n" - f"音质: {api_service._get_quality_display_name(p.quality)}\n" - ) - zf.writestr("info.txt", info_txt) + + def _write_zip() -> None: + with zipfile.ZipFile(zip_tmp_path, 'w', zipfile.ZIP_DEFLATED) as zf: + zf.write(file_path_str, arcname=filename) + # 歌词已在预取时一并拿到，无需再请求一次 + if mi.lyric: + try: + zf.writestr(f"{base}.{p.lyric_format}", + _format_lyric(mi.lyric, p.lyric_format)) + except Exception: + pass + if cover: + zf.writestr(f"{base}.jpg", cover) + info_txt = ( + f"歌曲: {music_info['name']}\n" + f"歌手: {music_info['artist_string']}\n" + f"专辑: {music_info['album']}\n" + f"音质: {api_service._get_quality_display_name(p.quality)}\n" + ) + zf.writestr("info.txt", info_txt) + + await asyncio.to_thread(_write_zip) return { 'file_path': file_path_str, 'zip_path': zip_tmp_path, @@ -228,8 +243,8 @@ def _pack_single_zip(file_path_str: str, base: str, filename: str, mi: MusicInfo } -def _publish_single_task(task_id: str, result: dict[str, Any], remote_ip: str, - name: str, filename: str) -> None: +async def _publish_single_task(task_id: str, result: dict[str, Any], remote_ip: str, + name: str, filename: str) -> None: """最终发布：置 done（已终态不覆盖）→ 结算 → 记操作日志。 交付已落定；结算失败**不再**把任务改标 error——钱由重启对账兜底， @@ -246,7 +261,7 @@ def _publish_single_task(task_id: str, result: dict[str, Any], remote_ip: str, t['msg'] = '下载完成' t['result'] = result try: - auth_system.ledger_settle(task_id, 1, 'delivered') + await asyncio.to_thread(auth_system.ledger_settle, task_id, 1, 'delivered') except Exception as _e: api_service.logger.error(f"单曲交付结算失败（重启对账兜底）: {task_id} {_e}") try: @@ -256,30 +271,30 @@ def _publish_single_task(task_id: str, result: dict[str, Any], remote_ip: str, api_service.logger.info(f"单曲下载完成: {filename} ({task_id})") -def _fail_single_task(task_id: str, status: str, msg: str, ledger_status: str, - task_dir_path: str | None) -> None: +async def _fail_single_task(task_id: str, status: str, msg: str, ledger_status: str, + task_dir_path: str | None) -> None: """失败/取消的统一收尾：置终态 → 退款 → 清掉任务目录。 这三个 except 分支原先各写一遍（共 9 行 × 3 处）；收敛后新增一种失败 只需要挑一个 ``ledger_status``，不会再漏掉清理或退款。 """ - _mark_task_terminal(task_id, status, msg) + await _mark_task_terminal(task_id, status, msg) try: - auth_system.ledger_settle(task_id, 0, ledger_status) + await asyncio.to_thread(auth_system.ledger_settle, task_id, 0, ledger_status) except Exception: pass if task_dir_path: - shutil.rmtree(task_dir_path, ignore_errors=True) + await asyncio.to_thread(shutil.rmtree, task_dir_path, ignore_errors=True) -def _single_download_worker(task_id: str) -> None: +async def _single_download_worker(task_id: str) -> None: """后台单曲下载：下载音频（可选 ZIP 打包），支持取消；完成后由 /download/result 交付，35 分钟保留期清理""" task_dir_path: str | None = None try: p = _begin_single_task(task_id) - mi, music_info, base, filename = _fetch_single_metadata(p) + mi, music_info, base, filename = await _fetch_single_metadata(p) - download_result = api_service.downloader.download_music_file( + download_result = await api_service.downloader.download_music_file( p.music_id, p.quality, write_meta=p.write_meta, meta_fields=p.meta_fields, @@ -293,25 +308,26 @@ def _single_download_worker(task_id: str) -> None: _abort_if_canceled(p.cancel_ev) assert download_result.file_path is not None # 交付文件用任务专属副本，共享缓存不进清理清单 - file_path_str = _task_delivery_copy(task_id, Path(download_result.file_path), filename) + file_path_str = await asyncio.to_thread( + _task_delivery_copy, task_id, Path(download_result.file_path), filename) task_dir_path = str(Path(file_path_str).parent) if p.need_zip: - result = _pack_single_zip(file_path_str, base, filename, mi, music_info, p) + result = await _pack_single_zip(file_path_str, base, filename, mi, music_info, p) else: result = _single_file_result(file_path_str, filename, music_info) # 打包阶段同样以取消检查收口 _abort_if_canceled(p.cancel_ev) - _publish_single_task(task_id, result, p.remote_ip, music_info['name'], filename) + await _publish_single_task(task_id, result, p.remote_ip, music_info['name'], filename) except DownloadCanceled: api_service.logger.info(f"单曲下载已取消: {task_id}") - _fail_single_task(task_id, 'canceled', '任务已取消', 'canceled', task_dir_path) + await _fail_single_task(task_id, 'canceled', '任务已取消', 'canceled', task_dir_path) except DownloadException as e: api_service.logger.error(f"单曲下载失败: {task_id} {e}") - _fail_single_task(task_id, 'error', str(e), 'error', task_dir_path) + await _fail_single_task(task_id, 'error', str(e), 'error', task_dir_path) except Exception as e: # 非预期异常的 repr 可能含上游主机/连接细节——用户可见文案 # 泛化为固定文案，原文 logger 先行留档（教训：先记录再泛化） api_service.logger.error(f"单曲下载异常: {task_id} {e}\n{traceback.format_exc()}") - _fail_single_task(task_id, 'error', '下载失败，请稍后重试', 'error', task_dir_path) + await _fail_single_task(task_id, 'error', '下载失败，请稍后重试', 'error', task_dir_path) diff --git a/services/podcast.py b/services/podcast.py index 0d2dfe2..a4f176e 100644 --- a/services/podcast.py +++ b/services/podcast.py @@ -4,7 +4,12 @@ 整台打包的骨架（状态机 / 暂停 / 名额 / 终态收敛 / 结算）在 :mod:`services._zipcore`，本模块只保留播客侧策略：清单组装（整台或指定单集）、 逐集顺序下载、按实际集数补扣、单条目入包（首集附封面）。 + +下载走 core.http_client 的流式响应（原先是 services.streaming 这个同步流垫片）， +落盘写文件是阻塞 IO、ZIP 组装是整段拷贝，均经 ``asyncio.to_thread`` 卸载。 """ + +import asyncio import os import shutil import tempfile @@ -17,6 +22,8 @@ from pathlib import Path from typing import Any import auth_system +from core.http_client import client as _http +from core.http_client import iter_chunks from core.state import ( DOWNLOAD_TASKS, DOWNLOAD_TASKS_LOCK, @@ -49,9 +56,13 @@ from .api_service import ( from .downloads import ( _abort_if_canceled, ) -from .streaming import ( - _stream_open, -) + +# 播客流式下载的上游头（与音频直链 CDN 的口径一致） +_STREAM_HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Safari/537.36', + 'Referer': 'https://music.163.com/', +} def _podcast_safe_name(s: str, fallback: str = 'unknown') -> str: @@ -92,20 +103,21 @@ def _report_progress(task_id: str, progress_label: str, done: int, total: int) - t['live_percent'] = min(99.0, float(pct)) -def _stream_to_part(url: str, part: Path, total_size: int, cancel_ev: threading.Event, - task_id: str, progress_label: str) -> tuple[int, int]: +async def _stream_to_part(url: str, part: Path, total_size: int, cancel_ev: threading.Event, + task_id: str, progress_label: str) -> tuple[int, int]: """把响应体流式写入 part，返回 (已写字节数, 期望总大小)。 期望总大小取入参，缺失时回落到 Content-Length（仍可能为 0，表示上游没给）。 """ done = 0 last_update = 0.0 - with _stream_open(url, timeout=60) as r: - if r.status >= 400: - raise DownloadException(f"上游返回 {r.status}") - total = total_size or int(r.headers.get('Content-Length') or 0) + resp = await _http.open('GET', url, headers=_STREAM_HEADERS, timeout=60) + try: + if resp.status >= 400: + raise DownloadException(f"上游返回 {resp.status}") + total = total_size or int(resp.headers.get('Content-Length') or 0) with open(part, 'wb') as f: - for chunk in r.iter_content(chunk_size=256 * 1024): + async for chunk in iter_chunks(resp, 256 * 1024): if cancel_ev.is_set(): raise DownloadCanceled("任务已取消") if not chunk: @@ -116,13 +128,15 @@ def _stream_to_part(url: str, part: Path, total_size: int, cancel_ev: threading. if now - last_update > 0.5: last_update = now _report_progress(task_id, progress_label, done, total) + finally: + resp.close() return done, total def _verify_downloaded(target: Path, done: int, total_size: int) -> None: """完整性校验：不达标则删掉已落地的文件并抛 DownloadException。 - 为什么非要校验：CDN 提前断流时 iter_content 会「正常」结束，文件静默截断。 + 为什么非要校验：CDN 提前断流时流式读取会「正常」结束，文件静默截断。 已知期望大小时要求**精确相等**（API 报的大小与实际一致；放宽到 95% 容差会 交付截断的坏文件还照扣次数）；期望未知时退化为非空校验——Content-Length 兜底值可能是压缩口径，不做精确比对。 @@ -135,21 +149,21 @@ def _verify_downloaded(target: Path, done: int, total_size: int) -> None: raise DownloadException("下载内容为空，请重试") -def _podcast_stream_download(url: str, target: Path, total_size: int, - cancel_ev: threading.Event, - task_id: str = '', progress_label: str = '') -> int: +async def _podcast_stream_download(url: str, target: Path, total_size: int, + cancel_ev: threading.Event, + task_id: str = '', progress_label: str = '') -> int: """流式下载播客音频到 target（先写 .part 再改名），返回实际字节数。 - 自动加入全局下载信号量排队；每 0.5s 更新一次任务进度消息。 + 自动加入全局下载闸排队；每 0.5s 更新一次任务进度消息。 取消时 .part 会留存（不删）——下次下载同名文件直接覆盖，是既有行为。 """ part = target.with_name(target.name + '.part') # 等待全局下载名额期间可被取消（与音乐批量 BATCH_ZIP_SEM 同语义） sem = api_service.downloader._download_semaphore - if not sem.acquire_cancellable(cancel_ev): + if not await sem.acquire_cancellable(cancel_ev): raise DownloadCanceled("任务已取消") try: - done, _total = _stream_to_part(url, part, total_size, cancel_ev, task_id, progress_label) + done, _total = await _stream_to_part(url, part, total_size, cancel_ev, task_id, progress_label) finally: sem.release() if part.exists(): @@ -158,13 +172,13 @@ def _podcast_stream_download(url: str, target: Path, total_size: int, return done -def _podcast_resolve_audio(program: dict[str, Any], quality: str, - cookies: dict[str, str]) -> dict[str, Any]: +async def _podcast_resolve_audio(program: dict[str, Any], quality: str, + cookies: dict[str, str]) -> dict[str, Any]: """解析单集音频直链，失败抛 DownloadException""" song_id = program.get('main_song_id') if not song_id: raise DownloadException("该单集缺少音频信息（mainSong）") - url_info = podcast.resolve_episode_url(song_id, cookies, quality) + url_info = await podcast.resolve_episode_url(song_id, cookies, quality) if not url_info or not url_info.get('url'): raise DownloadException("未获取到音频直链（单集可能已下架或受版权限制）") return url_info @@ -242,21 +256,16 @@ def _prepare_podcast_target(task_id: str, program_id: Any, program: dict[str, An def _write_podcast_zip(zip_path: str, file_path_str: str, arcname: str, base: str, name: str, program: dict[str, Any], url_info: dict[str, Any], - fallback_quality: str) -> None: + fallback_quality: str, cover: bytes = b'') -> None: """单集入包：音频 + 封面（可选，失败不炸任务）+ info.txt。 info.txt 不再写直链（真实 URL 不应随交付文件外泄）。 + 封面字节由调用方先取好（网络等待留在协程里），本函数只做阻塞的落盘组装。 """ with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf: zf.write(file_path_str, arcname=arcname) - cover_url = program.get('cover_url') or '' - if cover_url: - try: - cover = fetch_cover_bytes(cover_url) - if cover: - zf.writestr(f"{base}.jpg", cover) - except Exception: - pass + if cover: + zf.writestr(f"{base}.jpg", cover) info_txt = ( f"单集: {name}\n" f"播客: {program.get('radio_name') or ''}\n" @@ -277,16 +286,24 @@ def _podcast_file_result(file_path_str: str, filename: str, ext: str) -> dict[st } -def _podcast_zip_result(prep: _PodcastTarget, file_path_str: str, - program: dict[str, Any], url_info: dict[str, Any], - spec: _PodcastTaskSpec) -> dict[str, Any]: +async def _podcast_zip_result(prep: _PodcastTarget, file_path_str: str, + program: dict[str, Any], url_info: dict[str, Any], + spec: _PodcastTaskSpec) -> dict[str, Any]: """打包交付形状：ZIP 中间文件建在任务目录下（失败收尾 rmtree 一并覆盖）。""" + cover: bytes = b'' + cover_url = program.get('cover_url') or '' + if cover_url: + try: + cover = await fetch_cover_bytes(cover_url) or b'' + except Exception: + cover = b'' fd, zip_tmp_path = tempfile.mkstemp(prefix='podcast_zip_', suffix='.zip', dir=str(prep.task_dir)) os.close(fd) - _write_podcast_zip(zip_tmp_path, file_path_str, arcname=prep.filename, - base=prep.base, name=prep.name, program=program, - url_info=url_info, fallback_quality=spec.quality) + await asyncio.to_thread(_write_podcast_zip, zip_tmp_path, file_path_str, + arcname=prep.filename, base=prep.base, name=prep.name, + program=program, url_info=url_info, + fallback_quality=spec.quality, cover=cover) return { 'file_path': file_path_str, 'zip_path': zip_tmp_path, @@ -296,8 +313,8 @@ def _podcast_zip_result(prep: _PodcastTarget, file_path_str: str, } -def _publish_podcast_task(task_id: str, result: dict[str, Any], remote_ip: str, - name: str, filename: str) -> None: +async def _publish_podcast_task(task_id: str, result: dict[str, Any], remote_ip: str, + name: str, filename: str) -> None: """最终发布：置 done（已终态不覆盖）→ 结算 → 记操作日志。 交付已落定；结算失败**不再**把任务改标 error——钱由重启对账兜底， @@ -313,7 +330,7 @@ def _publish_podcast_task(task_id: str, result: dict[str, Any], remote_ip: str, t['msg'] = '下载完成' t['result'] = result try: - auth_system.ledger_settle(task_id, 1, 'delivered') + await asyncio.to_thread(auth_system.ledger_settle, task_id, 1, 'delivered') except Exception as _e: api_service.logger.error(f"播客单集交付结算失败（重启对账兜底）: {task_id} {_e}") try: @@ -323,19 +340,19 @@ def _publish_podcast_task(task_id: str, result: dict[str, Any], remote_ip: str, api_service.logger.info(f"播客单集下载完成: {filename} ({task_id})") -def _fail_podcast_task(task_id: str, status: str, msg: str, ledger_status: str, - task_dir: Path | None) -> None: +async def _fail_podcast_task(task_id: str, status: str, msg: str, ledger_status: str, + task_dir: Path | None) -> None: """失败/取消的统一收尾：置终态 → 退款 → 清掉任务目录（含 ZIP 中间件）。""" - _mark_task_terminal(task_id, status, msg) + await _mark_task_terminal(task_id, status, msg) try: - auth_system.ledger_settle(task_id, 0, ledger_status) + await asyncio.to_thread(auth_system.ledger_settle, task_id, 0, ledger_status) except Exception: pass if task_dir is not None: - shutil.rmtree(str(task_dir), ignore_errors=True) + await asyncio.to_thread(shutil.rmtree, str(task_dir), ignore_errors=True) -def _podcast_download_worker(task_id: str) -> None: +async def _podcast_download_worker(task_id: str) -> None: """后台播客单集下载：解析单集 → 音质降级取直链 → 流式下载（可选 ZIP 打包） 任务 type 为 'single'，完成后经 /download/result 交付，与单曲下载共用交付通道。 @@ -344,37 +361,37 @@ def _podcast_download_worker(task_id: str) -> None: task_dir: Path | None = None try: spec = _begin_podcast_task(task_id) - program = podcast.get_program_detail(spec.program_id, spec.cookies) - url_info = _podcast_resolve_audio(program, spec.quality, spec.cookies) + program = await podcast.get_program_detail(spec.program_id, spec.cookies) + url_info = await _podcast_resolve_audio(program, spec.quality, spec.cookies) prep = _prepare_podcast_target(task_id, spec.program_id, program, url_info) task_dir = prep.task_dir with DOWNLOAD_TASKS_LOCK: t = DOWNLOAD_TASKS[task_id] t['msg'] = '下载中...' - _podcast_stream_download(url_info['url'], prep.target, url_info.get('size', 0), - spec.cancel_ev, task_id, '下载中') + await _podcast_stream_download(url_info['url'], prep.target, url_info.get('size', 0), + spec.cancel_ev, task_id, '下载中') _abort_if_canceled(spec.cancel_ev) file_path_str = str(prep.target) if spec.need_zip: - result = _podcast_zip_result(prep, file_path_str, program, url_info, spec) + result = await _podcast_zip_result(prep, file_path_str, program, url_info, spec) else: result = _podcast_file_result(file_path_str, prep.filename, prep.ext) # 打包阶段同样以取消检查收口（末段取消不得覆盖为 done） _abort_if_canceled(spec.cancel_ev) - _publish_podcast_task(task_id, result, spec.remote_ip, prep.name, prep.filename) + await _publish_podcast_task(task_id, result, spec.remote_ip, prep.name, prep.filename) except DownloadCanceled: api_service.logger.info(f"播客单集下载已取消: {task_id}") - _fail_podcast_task(task_id, 'canceled', '任务已取消', 'canceled', task_dir) + await _fail_podcast_task(task_id, 'canceled', '任务已取消', 'canceled', task_dir) except DownloadException as e: api_service.logger.error(f"播客单集下载失败: {task_id} {e}") - _fail_podcast_task(task_id, 'error', str(e), 'error', task_dir) + await _fail_podcast_task(task_id, 'error', str(e), 'error', task_dir) except Exception as e: # 非预期异常原文 logger 先行，用户可见文案泛化（教训） api_service.logger.error(f"播客单集下载异常: {task_id} {e}\n{traceback.format_exc()}") - _fail_podcast_task(task_id, 'error', '下载失败，请稍后重试', 'error', task_dir) + await _fail_podcast_task(task_id, 'error', '下载失败，请稍后重试', 'error', task_dir) def _podcast_cleanup_files(paths: list[str | None]) -> None: @@ -397,7 +414,7 @@ def _podcast_cleanup_files(paths: list[str | None]) -> None: @dataclass(frozen=True) class _EpisodePayload: - """一集下载完成后的入包素材（下载线程产出，落包线程消费）。""" + """一集下载完成后的入包素材（下载协程产出，落包协程消费）。""" path: str base: str @@ -409,7 +426,7 @@ class _PodcastZipJob(ZipJob): 并发刻意等于音乐批量：同样占用全局批量名额（BATCH_ZIP_SEM，全局 3）， 逐集 acquire/release——暂停等待/取消时不长期占名额；音频本身仍受全局 - 下载信号量约束。原实现不受批量并发控制，多用户同时打包会无限并发。 + 下载闸约束。原实现不受批量并发控制，多用户同时打包会无限并发。 """ tag = '播客打包' @@ -422,20 +439,20 @@ class _PodcastZipJob(ZipJob): def __init__(self) -> None: self._radio_name = '' - def prepare(self, params: dict[str, Any], task_id: str, failed: list[str]) -> list[Any]: + async def prepare(self, params: dict[str, Any], task_id: str, failed: list[str]) -> list[Any]: cookies = params['cookies'] radio_id = params['radio_id'] limit = params['limit'] # 组装单集清单：整台电台（正序全量）或指定单集ID列表 if radio_id: - episodes = podcast.list_radio_all_program_ids(radio_id, cookies, max_count=limit) + episodes = await podcast.list_radio_all_program_ids(radio_id, cookies, max_count=limit) if not episodes: raise DownloadException("该电台暂无可下载的单集") else: episodes = [] for pid in (params.get('ids') or []): try: - prog = podcast.get_program_detail(pid, cookies) + prog = await podcast.get_program_detail(pid, cookies) episodes.append({ 'id': prog['id'], 'name': prog['name'], @@ -453,7 +470,7 @@ class _PodcastZipJob(ZipJob): self._radio_name = _podcast_safe_name(params.get('radio_name') or '', '播客') return episodes - def supplement(self, task_id: str, params: dict[str, Any], count: int) -> None: + async def supplement(self, task_id: str, params: dict[str, Any], count: int) -> None: """按次计费补扣：整台电台提交时只预扣 1 次，此处按实际单集数补足差额。 补扣失败（余额不足/数据库异常）任务必须失败并按账本退费——不允许无账本继续交付。 @@ -462,7 +479,7 @@ class _PodcastZipJob(ZipJob): _buid = params.get('billing_uid') _extra = count - int(params.get('billed') or 1) if _buid and _extra > 0: - if not auth_system.ledger_supplement(task_id, _extra): + if not await asyncio.to_thread(auth_system.ledger_supplement, task_id, _extra): raise DownloadException( f"补扣下载次数失败（需补扣 {_extra} 次），任务已取消，预扣部分将退回") if _buid: @@ -476,13 +493,13 @@ class _PodcastZipJob(ZipJob): except Exception as e: raise DownloadException(f"补扣下载次数异常: {e}") from e - def download_one(self, item: Any, idx: int, ctx: ZipContext) -> _EpisodePayload: + async def download_one(self, item: Any, idx: int, ctx: ZipContext) -> _EpisodePayload: ep = item label = f"[{idx}/{ctx.total}] {ep['name'][:24]}" ctx.progress(f"{label} 解析中...") cookies = ctx.params['cookies'] - prog = ep.get('detail') or podcast.get_program_detail(ep['id'], cookies) - url_info = _podcast_resolve_audio(prog, ctx.params['quality'], cookies) + prog = ep.get('detail') or await podcast.get_program_detail(ep['id'], cookies) + url_info = await _podcast_resolve_audio(prog, ctx.params['quality'], cookies) ctx.progress(f"{label} 下载中...") ext = (url_info.get('type') or 'mp3').lower().lstrip('.') serial = int(ep.get('serial_num') or 0) @@ -492,15 +509,15 @@ class _PodcastZipJob(ZipJob): # 单集（Linux 上 unlink 打开中的文件） tmp_audio = ctx.work_dir / f"podcast_zip_tmp_{ctx.task_id}_{idx}.{ext}" try: - _podcast_stream_download(url_info['url'], tmp_audio, - url_info.get('size', 0), ctx.cancel_ev) + await _podcast_stream_download(url_info['url'], tmp_audio, + url_info.get('size', 0), ctx.cancel_ev) except Exception: # 下载失败即清（.part 残片同清）；成功时由 write_entry 的 finally 清 _podcast_cleanup_files([str(tmp_audio)]) raise return _EpisodePayload(path=str(tmp_audio), base=base, ext=ext) - def write_entry(self, item: Any, idx: int, payload: _EpisodePayload, ctx: ZipContext) -> None: + async def write_entry(self, item: Any, idx: int, payload: _EpisodePayload, ctx: ZipContext) -> None: ep = item try: # 重名条目自动追加序号（同名单集/同号不同集不得互相覆盖） @@ -510,11 +527,12 @@ class _PodcastZipJob(ZipJob): arcname = f"{payload.base} ({k}).{payload.ext}" k += 1 ctx.used_arcnames.add(arcname) - ctx.zf.write(payload.path, arcname=arcname, compress_type=zipfile.ZIP_STORED) + await asyncio.to_thread(ctx.zf.write, payload.path, arcname=arcname, + compress_type=zipfile.ZIP_STORED) # 首集附带封面，便于识别 if idx == 1 and ep.get('cover_url'): try: - cover = fetch_cover_bytes(ep['cover_url']) + cover = await fetch_cover_bytes(ep['cover_url']) if cover: ctx.zf.writestr("cover.jpg", cover, compress_type=zipfile.ZIP_STORED) except Exception: @@ -541,7 +559,7 @@ class _PodcastZipJob(ZipJob): return f"全部单集下载失败: {'; '.join(failed[:3])}" + ( f"（已退回 {refunded} 次下载次数）" if refunded else '') - def on_delivered(self, items: list[Any], task_id: str, params: dict[str, Any]) -> None: + async def on_delivered(self, items: list[Any], task_id: str, params: dict[str, Any]) -> None: try: first_name = items[0].get('name') if items else '' _record_log('下载', params.get('remote_ip') or '-', @@ -550,6 +568,6 @@ class _PodcastZipJob(ZipJob): pass -def _podcast_zip_worker(task_id: str) -> None: +async def _podcast_zip_worker(task_id: str) -> None: """整台播客打包：逐集 下载→入包即删，支持取消与断线暂停（复用批量任务机制）""" - run_zip_job(_PodcastZipJob(), task_id, settle=_settle_batch_billing) + await run_zip_job(_PodcastZipJob(), task_id, settle=_settle_batch_billing) diff --git a/services/streaming.py b/services/streaming.py deleted file mode 100644 index a64beb2..0000000 --- a/services/streaming.py +++ /dev/null @@ -1,61 +0,0 @@ -# -*- coding: utf-8 -*- -"""同步流式响应垫片（音频分段透传）。 -""" - - - -class _SyncStreamResponse: - """同步流式响应包装（把 aiohttp 的异步流按需拉成同步迭代）。 - - 供流式下载使用：worker 是同步线程，这里经 core.async_bridge 在后台事件循环 - 逐块取数据，避免引入 requests 依赖（Flask 已删除、requests 已移出依赖）。 - """ - - def __init__(self, url: str, timeout: float): - from core import async_bridge - self._bridge = async_bridge - self._url = url - self._timeout = timeout - self._agent = None - self.status = 0 - self.headers: dict[str, str] = {} - self._resp = None - - def __enter__(self): - self._agent = self._bridge.run_sync(self._open()) - self.status = self._agent[0] - self.headers = self._agent[1] - return self - - async def _open(self): - from core.http_client import client - resp = await client.open('GET', self._url, headers={ - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 ' - '(KHTML, like Gecko) Safari/537.36', - 'Referer': 'https://music.163.com/', - }, timeout=self._timeout) - return resp.status, dict(resp.headers), resp - - def iter_content(self, chunk_size: int = 256 * 1024): - assert self._agent is not None - resp = self._agent[2] - it = resp.content.iter_chunked(chunk_size) - while True: - try: - chunk = self._bridge.run_sync(it.__anext__()) - except StopAsyncIteration: - break - yield chunk - - def __exit__(self, *exc): - if self._agent is not None and self._agent[2] is not None: - try: - self._agent[2].close() - except Exception: - pass - return False - - -def _stream_open(url: str, timeout: float = 60) -> _SyncStreamResponse: - """打开同步可迭代的流式响应（见 _SyncStreamResponse）。""" - return _SyncStreamResponse(url, timeout) diff --git a/services/tasks.py b/services/tasks.py index 913e422..0c6654f 100644 --- a/services/tasks.py +++ b/services/tasks.py @@ -172,7 +172,7 @@ def _reap_finished_tasks(now: float) -> None: def _cancel_overtime_tasks(now: float) -> None: """运行/排队/暂停超过硬上限仍未结束的任务 → 置取消并唤醒 cancel Event。 - worker 线程已死则无人再走终态结算，监控代为按账本退款——ledger_settle 幂等， + worker 已结束则无人再走终态结算，监控代为按账本退款——ledger_settle 幂等， worker 若已结算过此处返回 0，无双退。结算移到任务锁外（同 _mark_task_terminal 口径）：库忙 15s 不再随任务锁波及全站任务端点。 """ @@ -188,9 +188,9 @@ def _cancel_overtime_tasks(now: float) -> None: task['cancel'].set() except Exception: pass - # worker 线程已死则无人再走终态结算——监控代为按账本结算退款 - th = task.get('worker_thread') - if th is not None and not th.is_alive(): + # worker 已结束则无人再走终态结算——监控代为按账本结算退款 + wt = task.get('worker_task') + if wt is not None and wt.done(): timeout_settle.append(tid) api_service.logger.warning(f"任务超时自动取消: {tid}") for tid in timeout_settle: diff --git a/tests/test_aria2.py b/tests/test_aria2.py index dd6f353..86d36ea 100644 --- a/tests/test_aria2.py +++ b/tests/test_aria2.py @@ -14,7 +14,7 @@ import threading import pytest import music_downloader._aria2 as a2 -from core.sync_http import RequestException +from core.http_client import UpstreamNetworkError from music_downloader import DownloadCanceled, DownloadException @@ -82,7 +82,8 @@ class _FakeProc: def _backend(monkeypatch, steps, **kw): - monkeypatch.setattr(a2.time, 'sleep', lambda _s: None) + # fetch 的轮询改 asyncio.sleep（不再 time.sleep）：把间隔压到 0 让状态机快速走完 + monkeypatch.setattr(a2, '_POLL_INTERVAL', 0.0) dl = _FailUpdateDownload([]) if kw.get('fail_update') else _FakeDownload(steps) api = _FakeAPI(dl, fail_submit=kw.get('fail_submit', False)) return a2.Aria2Backend(api, _FakeProc(), 6800), api @@ -104,11 +105,11 @@ def test_free_port_is_in_valid_range(): # ---------- 完成路径 ---------- -def test_fetch_reports_progress_and_completes(monkeypatch, tmp_path): +async def test_fetch_reports_progress_and_completes(monkeypatch, tmp_path): be, api = _backend(monkeypatch, [(50, False, False), (100, True, False)]) seen = [] - be.fetch('http://x/y.mp3', tmp_path / 'y.part', - progress_cb=lambda done, total: seen.append((done, total))) + await be.fetch('http://x/y.mp3', tmp_path / 'y.part', + progress_cb=lambda done, total: seen.append((done, total))) assert seen == [(50, 100), (100, 100)] assert api.removed == [] # 完成不需要中止 assert not (tmp_path / 'y.part.aria2').exists() # 控制文件已兜底清理 @@ -119,57 +120,57 @@ def test_fetch_reports_progress_and_completes(monkeypatch, tmp_path): assert options['max-connection-per-server'] == str(a2._SPLIT) -def test_progress_callback_exception_is_swallowed(monkeypatch, tmp_path): +async def test_progress_callback_exception_is_swallowed(monkeypatch, tmp_path): be, _api = _backend(monkeypatch, [(100, True, False)]) def boom(_d, _t): raise RuntimeError('cb') - be.fetch('http://x/y.mp3', tmp_path / 'y.part', progress_cb=boom) # 不应抛出 + await be.fetch('http://x/y.mp3', tmp_path / 'y.part', progress_cb=boom) # 不应抛出 # ---------- 取消 / 守卫 / 失败 ---------- -def test_fetch_maps_cancel_and_drops_partial(monkeypatch, tmp_path): +async def test_fetch_maps_cancel_and_drops_partial(monkeypatch, tmp_path): be, api = _backend(monkeypatch, [(10, False, False)]) ev = threading.Event() ev.set() with pytest.raises(DownloadCanceled): - be.fetch('http://x/y.mp3', tmp_path / 'y.part', cancel_event=ev) + await be.fetch('http://x/y.mp3', tmp_path / 'y.part', cancel_event=ev) assert api.removed == [('g1', True, True)] -def test_fetch_guards_size_and_drops_partial(monkeypatch, tmp_path): +async def test_fetch_guards_size_and_drops_partial(monkeypatch, tmp_path): be, api = _backend(monkeypatch, [(700, False, False)]) with pytest.raises(DownloadException): - be.fetch('http://x/y.mp3', tmp_path / 'y.part', max_bytes=500) + await be.fetch('http://x/y.mp3', tmp_path / 'y.part', max_bytes=500) assert api.removed == [('g1', True, True)] -def test_fetch_failure_is_retryable_and_keeps_partial(monkeypatch, tmp_path): +async def test_fetch_failure_is_retryable_and_keeps_partial(monkeypatch, tmp_path): be, api = _backend(monkeypatch, [(10, False, True)]) - with pytest.raises(RequestException): - be.fetch('http://x/y.mp3', tmp_path / 'y.part') + with pytest.raises(UpstreamNetworkError): + await be.fetch('http://x/y.mp3', tmp_path / 'y.part') assert api.removed == [] # 残片保留，供下轮续传 -def test_fetch_timeout_is_retryable(monkeypatch, tmp_path): +async def test_fetch_timeout_is_retryable(monkeypatch, tmp_path): be, api = _backend(monkeypatch, [(10, False, False)]) # 永不完成 - with pytest.raises(RequestException): - be.fetch('http://x/y.mp3', tmp_path / 'y.part', timeout=-1) + with pytest.raises(UpstreamNetworkError): + await be.fetch('http://x/y.mp3', tmp_path / 'y.part', timeout=-1) assert api.removed == [('g1', True, False)] # 超时中止但保留残片 -def test_fetch_submit_failure_is_retryable(monkeypatch, tmp_path): +async def test_fetch_submit_failure_is_retryable(monkeypatch, tmp_path): be, _api = _backend(monkeypatch, [], fail_submit=True) - with pytest.raises(RequestException): - be.fetch('http://x/y.mp3', tmp_path / 'y.part') + with pytest.raises(UpstreamNetworkError): + await be.fetch('http://x/y.mp3', tmp_path / 'y.part') -def test_fetch_update_failure_is_retryable(monkeypatch, tmp_path): +async def test_fetch_update_failure_is_retryable(monkeypatch, tmp_path): be, _api = _backend(monkeypatch, [], fail_update=True) - with pytest.raises(RequestException): - be.fetch('http://x/y.mp3', tmp_path / 'y.part') + with pytest.raises(UpstreamNetworkError): + await be.fetch('http://x/y.mp3', tmp_path / 'y.part') def test_close_is_idempotent(monkeypatch): diff --git a/tests/test_download_sync.py b/tests/test_download_sync.py index fea873d..9318ba8 100644 --- a/tests/test_download_sync.py +++ b/tests/test_download_sync.py @@ -51,12 +51,12 @@ def env(tmp_path, monkeypatch): state: dict = {'info_exc': None, 'dl_result': _dl(), 'dl_exc': None} class _FakeDownloader: - def get_music_info(self, music_id, quality, cookies): + async def get_music_info(self, music_id, quality, cookies): if state['info_exc'] is not None: raise state['info_exc'] return state['info'] - def download_music_file(self, music_id, quality, **kw): + async def download_music_file(self, music_id, quality, **kw): if state['dl_exc'] is not None: raise state['dl_exc'] return state['dl_result'] diff --git a/tests/test_downloader.py b/tests/test_downloader.py index c3d23d3..a9fa001 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -9,64 +9,83 @@ ZIP 都经它，约 205 行 / 圈复杂度 47，此前**零直接测试覆盖** 断言只依赖「调用后世界里发生了什么」——返回值、磁盘上的文件、发往网络的请求—— 不引用任何内部函数名，因此本文件可作为后续拆分的等价性验证器。 -替身约定（三处，均为模块级符号，替换点在 downloader 模块命名空间）： -- 网络：``get_http_session`` / ``get_backend``（后者返回 None = 走内置 HTTP 通道） +替身约定（均为模块级符号，替换点在 downloader 模块命名空间）： +- 网络：``_http``（core.http_client 的进程级异步客户端）——替身提供 + ``async open(method, url, **kw)``，返回带 ``status``/``headers``/``content`` + 的未关闭响应；分块读取走**真实的** ``iter_chunks``，顺带钉住「读取期异常 + 收敛为 UpstreamNetworkError」这条边界。 +- aria2：``get_backend``（返回 None = 强制走内置 HTTP 通道） - 内容嗅探：``_sniff_audio_header``（默认放行，按需置 False） - 元数据写入：``MusicDownloader._write_metadata``（不真跑 mutagen） """ +from __future__ import annotations + import threading from pathlib import Path from typing import Any +import aiohttp import pytest -from core import sync_http from music_downloader import DownloadCanceled, DownloadException, MusicInfo from music_downloader.downloader import MusicDownloader # ============ 替身 ============ -class _FakeResponse: - """core.sync_http.SyncResponse 的替身。 +class _FakeContent: + """``aiohttp.ClientResponse.content`` 的替身。 chunks 保留原始 iterable（**不要** list() 归一化）：生成器的副作用 （如中途置取消位）若在构造时刻就求值，「中途取消」会被悄悄测成 「开始前取消」——两条不同的路径混成一条，而测试仍是绿的。 + + 元素约定：可调用对象 → 调用一次（注入副作用，如 cancel.set）； + 异常实例 → 抛出（模拟读取期断流）；其余 → 作为分块产出。 """ - def __init__(self, chunks, status=200, headers=None): + def __init__(self, chunks): self._chunks = chunks - self.status_code = status + + async def iter_chunked(self, chunk_size=0): + for item in self._chunks: + if callable(item): + item() + continue + if isinstance(item, BaseException): + raise item + yield item + + +class _FakeResponse: + """``core.http_client.open`` 返回的未关闭流式响应替身。""" + + def __init__(self, chunks, status=200, headers=None, reason=''): + self.status = status + self.reason = reason self.headers = dict(headers or {}) + self.content = _FakeContent(chunks) self.closed = False - def iter_content(self, chunk_size=0): - yield from self._chunks - - def raise_for_status(self): - if self.status_code >= 400: - raise sync_http.HTTPError(f'HTTP {self.status_code}', self) - def close(self): self.closed = True -class _FakeSession: - """记录每次 GET（含 headers）并按序产出响应。""" +class _FakeClient: + """记录每次 open（含 kw）并按序产出响应；responder 可随用例重装。""" - def __init__(self, responder): - self._responder = responder + def __init__(self): + self._responder = None self.calls: list[tuple[str, dict[str, Any]]] = [] - self.closed = False - def get(self, url, **kw): + def set_responder(self, responder): + self._responder = responder + + async def open(self, method, url, **kw): self.calls.append((url, kw)) + assert self._responder is not None, '本用例未安装网络替身' return self._responder(len(self.calls), url, kw) - def close(self): - self.closed = True - def _body(size, status=200, content_range=None): """固定返回 size 字节的响应工厂（每次调用产出新响应对象）。""" @@ -78,6 +97,11 @@ def _body(size, status=200, content_range=None): return _mk +def _headers(kw) -> dict[str, str]: + """取一次调用的请求头——没带 Range 时 downloader 传的是 None。""" + return kw.get('headers') or {} + + def _info(mid=101, size=20, file_type='mp3'): return MusicInfo( id=mid, name='测试曲', artists='歌手', album='专辑', pic_url='', @@ -98,21 +122,20 @@ def env(tmp_path, monkeypatch): - state['sniff']：内容嗅探结果，默认 True - state['meta']：_write_metadata 的调用记录 [(文件路径, fields), ...] - - install(responder) -> _FakeSession：安装网络替身，返回该替身以便断言 + - install(responder) -> _FakeClient：安装网络替身，返回它以便断言 """ state: dict[str, Any] = {'sniff': True, 'meta': []} - box: dict[str, _FakeSession] = {} + client = _FakeClient() def install(responder): - box['session'] = _FakeSession(responder) - return box['session'] + client.set_responder(responder) + return client - def _record_meta(self, file_path, music_info, fields=None): + async def _record_meta(self, file_path, music_info, fields=None): state['meta'].append((str(file_path), fields)) return True - monkeypatch.setattr('music_downloader.downloader.get_http_session', - lambda: box['session']) + monkeypatch.setattr('music_downloader.downloader._http', client) monkeypatch.setattr('music_downloader.downloader.get_backend', lambda: None) monkeypatch.setattr('music_downloader.downloader._sniff_audio_header', lambda _p: state['sniff']) @@ -126,48 +149,48 @@ def env(tmp_path, monkeypatch): # ============ 正常落盘 ============ -def test_success_promotes_part_to_final(env): +async def test_success_promotes_part_to_final(env): dl, _state, install = env info = _info(size=20) fp, part = _paths(dl, info) install(_body(20)) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) assert r.success and r.error_message == '' - assert Path(r.file_path) == fp + assert Path(r.file_path or '') == fp assert r.file_size == 20 and fp.read_bytes() == b'x' * 20 assert not part.exists(), '完成后 .part 应改名转正' -def test_empty_chunks_skipped(env): +async def test_empty_chunks_skipped(env): dl, _state, install = env info = _info(size=20) fp, _ = _paths(dl, info) install(lambda a, u, kw: _FakeResponse([b'', b'x' * 20, b''])) seen: list[int] = [] - r = dl.download_music_file(101, music_info=info, - progress_cb=lambda w, _t: seen.append(w)) + r = await dl.download_music_file(101, music_info=info, + progress_cb=lambda w, _t: seen.append(w)) assert r.success and fp.stat().st_size == 20 assert seen == [0, 20], '空分块不写盘也不触发进度回调' -def test_progress_reports_resume_start_then_each_chunk(env): +async def test_progress_reports_resume_start_then_each_chunk(env): dl, _state, install = env info = _info(size=20) install(lambda a, u, kw: _FakeResponse([b'a' * 8, b'b' * 12])) seen: list[tuple[int, int]] = [] - r = dl.download_music_file(101, music_info=info, - progress_cb=lambda w, t: seen.append((w, t))) + r = await dl.download_music_file(101, music_info=info, + progress_cb=lambda w, t: seen.append((w, t))) assert r.success assert seen == [(0, 20), (8, 20), (20, 20)], '起始回调带续传起点，此后每块一次' -def test_progress_callback_exception_never_breaks_download(env): +async def test_progress_callback_exception_never_breaks_download(env): dl, _state, install = env info = _info(size=20) install(_body(20)) @@ -175,20 +198,24 @@ def test_progress_callback_exception_never_breaks_download(env): def _boom(_w, _t): raise RuntimeError('回调炸了') - r = dl.download_music_file(101, music_info=info, progress_cb=_boom) + r = await dl.download_music_file(101, music_info=info, progress_cb=_boom) assert r.success, '进度回调异常绝不影响下载主流程' -def test_music_info_fetched_when_not_provided(env, monkeypatch): +async def test_music_info_fetched_when_not_provided(env, monkeypatch): dl, _state, install = env info = _info(size=20) seen: list[tuple] = [] - monkeypatch.setattr(dl, 'get_music_info', - lambda mid, q, c=None: (seen.append((mid, q)), info)[1]) + + async def _fake_info(mid, q, c=None): + seen.append((mid, q)) + return info + + monkeypatch.setattr(dl, 'get_music_info', _fake_info) install(_body(20)) - r = dl.download_music_file(101, 'lossless') + r = await dl.download_music_file(101, 'lossless') assert r.success and seen == [(101, 'lossless')] @@ -196,40 +223,40 @@ def test_music_info_fetched_when_not_provided(env, monkeypatch): # ============ 缓存校验 ============ -def test_cache_hit_skips_network(env): +async def test_cache_hit_skips_network(env): dl, _state, install = env info = _info(size=20) fp, _ = _paths(dl, info) fp.write_bytes(b'c' * 20) session = install(lambda *a: pytest.fail('缓存命中不应发网络请求')) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) assert r.success and r.file_size == 20 assert session.calls == [] -def test_cache_size_mismatch_triggers_redownload(env): +async def test_cache_size_mismatch_triggers_redownload(env): dl, _state, install = env info = _info(size=20) fp, _ = _paths(dl, info) fp.write_bytes(b'c' * 15) # 与本次期望不符 = 缓存 miss session = install(_body(20)) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) assert r.success and fp.read_bytes() == b'x' * 20 assert len(session.calls) == 1, '坏缓存应被删除并重下，而不是当结果交付' -def test_zero_byte_cache_rejected_when_expected_size_unknown(env): +async def test_zero_byte_cache_rejected_when_expected_size_unknown(env): dl, _state, install = env info = _info(size=0) fp, _ = _paths(dl, info) fp.write_bytes(b'') session = install(lambda a, u, kw: _FakeResponse([b'good'])) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) assert r.success and fp.read_bytes() == b'good' assert len(session.calls) == 1, '期望未知时至少拒绝 0 字节坏文件' @@ -238,7 +265,7 @@ def test_zero_byte_cache_rejected_when_expected_size_unknown(env): # ============ 断点续传 ============ -def test_resume_sends_range_and_appends(env): +async def test_resume_sends_range_and_appends(env): dl, _state, install = env info = _info(size=20) fp, part = _paths(dl, info) @@ -247,67 +274,67 @@ def test_resume_sends_range_and_appends(env): [b'q' * 10], status=206, headers={'Content-Range': 'bytes 10-19/20'})) seen: list[tuple[int, int]] = [] - r = dl.download_music_file(101, music_info=info, - progress_cb=lambda w, t: seen.append((w, t))) + r = await dl.download_music_file(101, music_info=info, + progress_cb=lambda w, t: seen.append((w, t))) assert r.success - assert session.calls[0][1]['headers'].get('Range') == 'bytes=10-' + assert _headers(session.calls[0][1]).get('Range') == 'bytes=10-' assert fp.read_bytes() == b'p' * 10 + b'q' * 10, '续传应追加而非覆盖' assert seen[0] == (10, 20), '起始进度应带续传起点' -def test_server_ignoring_range_rewrites_from_zero(env): +async def test_server_ignoring_range_rewrites_from_zero(env): dl, _state, install = env info = _info(size=20) fp, part = _paths(dl, info) part.write_bytes(b'p' * 10) install(lambda a, u, kw: _FakeResponse([b'r' * 20], status=200)) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) assert r.success and fp.read_bytes() == b'r' * 20, '服务器返回 200 须从头重写' -def test_complete_part_promoted_without_network(env): +async def test_complete_part_promoted_without_network(env): dl, _state, install = env info = _info(size=20) fp, part = _paths(dl, info) part.write_bytes(b'z' * 20) # 已下完但未改名 session = install(lambda *a: pytest.fail('完整残片不应发网络请求')) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) assert r.success and r.file_size == 20 assert session.calls == [] assert fp.read_bytes() == b'z' * 20 and not part.exists() -def test_oversized_part_discarded_starts_from_zero(env): +async def test_oversized_part_discarded_starts_from_zero(env): dl, _state, install = env info = _info(size=20) _fp, part = _paths(dl, info) part.write_bytes(b'o' * 30) # 超尺寸残片没有续传价值 session = install(_body(20)) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) assert r.success - assert 'Range' not in session.calls[0][1]['headers'] + assert 'Range' not in _headers(session.calls[0][1]) -def test_zero_byte_part_discarded(env): +async def test_zero_byte_part_discarded(env): dl, _state, install = env info = _info(size=20) _fp, part = _paths(dl, info) part.write_bytes(b'') session = install(_body(20)) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) - assert r.success and 'Range' not in session.calls[0][1]['headers'] + assert r.success and 'Range' not in _headers(session.calls[0][1]) -def test_content_range_offset_mismatch_restarts_from_scratch(env): +async def test_content_range_offset_mismatch_restarts_from_scratch(env): dl, _state, install = env info = _info(size=20) fp, part = _paths(dl, info) @@ -321,14 +348,14 @@ def test_content_range_offset_mismatch_restarts_from_scratch(env): session = install(responder) - r = dl.download_music_file(101, music_info=info, retries=1) + r = await dl.download_music_file(101, music_info=info, retries=1) assert r.success and len(session.calls) == 2 - assert 'Range' not in session.calls[1][1]['headers'], '清残片后应从头下' + assert 'Range' not in _headers(session.calls[1][1]), '清残片后应从头下' assert fp.read_bytes() == b'f' * 20 -def test_206_without_content_range_restarts_from_scratch(env): +async def test_206_without_content_range_restarts_from_scratch(env): dl, _state, install = env info = _info(size=20) _fp, part = _paths(dl, info) @@ -341,13 +368,13 @@ def test_206_without_content_range_restarts_from_scratch(env): session = install(responder) - r = dl.download_music_file(101, music_info=info, retries=1) + r = await dl.download_music_file(101, music_info=info, retries=1) assert r.success and len(session.calls) == 2 - assert 'Range' not in session.calls[1][1]['headers'] + assert 'Range' not in _headers(session.calls[1][1]) -def test_http_416_restarts_from_scratch(env): +async def test_http_416_restarts_from_scratch(env): dl, _state, install = env info = _info(size=20) fp, part = _paths(dl, info) @@ -360,45 +387,45 @@ def test_http_416_restarts_from_scratch(env): session = install(responder) - r = dl.download_music_file(101, music_info=info, retries=1) + r = await dl.download_music_file(101, music_info=info, retries=1) assert r.success and len(session.calls) == 2 - assert 'Range' not in session.calls[1][1]['headers'] + assert 'Range' not in _headers(session.calls[1][1]) assert fp.read_bytes() == b'f' * 20 # ============ 失败与重试 ============ -def test_cdn_4xx_is_deterministic_and_not_retried(env): +async def test_cdn_4xx_is_deterministic_and_not_retried(env): dl, _state, install = env info = _info(size=20) session = install(lambda a, u, kw: _FakeResponse([], status=403)) with pytest.raises(DownloadException) as ei: - dl.download_music_file(101, music_info=info, retries=2) + await dl.download_music_file(101, music_info=info, retries=2) assert 'HTTP 403' in str(ei.value) assert len(session.calls) == 1, '确定性失败对同一失效直链白重试毫无意义' -def test_408_is_retried_then_reported_as_failure(env): +async def test_408_is_retried_then_reported_as_failure(env): dl, _state, install = env info = _info(size=20) session = install(lambda a, u, kw: _FakeResponse([], status=408)) - r = dl.download_music_file(101, music_info=info, retries=1) + r = await dl.download_music_file(101, music_info=info, retries=1) assert not r.success assert '下载请求失败' in r.error_message assert len(session.calls) == 2, '408 请求超时应保留可重试' -def test_5xx_is_retried_and_keeps_part(env): +async def test_5xx_is_retried_and_keeps_part(env): """5xx 是服务端瞬时故障（非确定性失败）：应可重试，且保留 .part 供续传。 与 4xx 的分界在这里钉住：error 分类只看 400<=status<500，5xx 落回裸重抛， - 由外层 RequestException 分支退避重试。此前这条路径无测试覆盖。 + 由外层 UpstreamError 分支退避重试。此前这条路径无测试覆盖。 """ dl, _state, install = env info = _info(size=100) @@ -406,7 +433,7 @@ def test_5xx_is_retried_and_keeps_part(env): part.write_bytes(b'p' * 10) session = install(lambda a, u, kw: _FakeResponse([], status=500)) - r = dl.download_music_file(101, music_info=info, retries=1) + r = await dl.download_music_file(101, music_info=info, retries=1) assert not r.success and '下载请求失败' in r.error_message assert len(session.calls) == 2, '5xx 非确定性失败，应重试' @@ -414,20 +441,20 @@ def test_5xx_is_retried_and_keeps_part(env): assert part.stat().st_size == 10, '.part 应保留供下轮续传' -def test_incomplete_size_keeps_part_and_reports(env): +async def test_incomplete_size_keeps_part_and_reports(env): dl, _state, install = env info = _info(size=100) fp, part = _paths(dl, info) install(_body(10)) - r = dl.download_music_file(101, music_info=info, retries=0) + r = await dl.download_music_file(101, music_info=info, retries=0) assert not r.success and '下载不完整' in r.error_message assert part.exists() and part.stat().st_size == 10, '.part 应保留以供续传' assert not fp.exists(), '不完整的下载不得转正交付' -def test_incomplete_then_resumes_on_next_attempt(env): +async def test_incomplete_then_resumes_on_next_attempt(env): dl, _state, install = env info = _info(size=100) _fp, part = _paths(dl, info) @@ -440,25 +467,24 @@ def test_incomplete_then_resumes_on_next_attempt(env): session = install(responder) - r = dl.download_music_file(101, music_info=info, retries=1) + r = await dl.download_music_file(101, music_info=info, retries=1) assert not r.success and '下载不完整' in r.error_message - assert session.calls[1][1]['headers'].get('Range') == 'bytes=10-', '下轮应续传' + assert _headers(session.calls[1][1]).get('Range') == 'bytes=10-', '下轮应续传' assert part.stat().st_size == 15 -def test_network_error_keeps_part_for_resume(env): +async def test_network_error_keeps_part_for_resume(env): + """读取期断流：真实的 iter_chunks 把 aiohttp.ClientError 收敛成 + UpstreamNetworkError，外层据此保留 .part 供续传。""" dl, _state, install = env info = _info(size=100) fp, part = _paths(dl, info) - def chunks(): - yield b'a' * 10 - raise sync_http.RequestException('连接中断') + install(lambda a, u, kw: _FakeResponse( + [b'a' * 10, aiohttp.ClientPayloadError('连接中断')])) - install(lambda a, u, kw: _FakeResponse(chunks())) - - r = dl.download_music_file(101, music_info=info, retries=0) + r = await dl.download_music_file(101, music_info=info, retries=0) assert not r.success and '下载请求失败' in r.error_message assert part.exists() and part.stat().st_size == 10, '网络错误应保留 .part' @@ -468,7 +494,7 @@ def test_network_error_keeps_part_for_resume(env): # ============ 内容校验与体积上限 ============ -def test_unknown_size_non_audio_content_rejected(env): +async def test_unknown_size_non_audio_content_rejected(env): dl, state, install = env info = _info(size=0) fp, part = _paths(dl, info) @@ -476,25 +502,25 @@ def test_unknown_size_non_audio_content_rejected(env): install(lambda a, u, kw: _FakeResponse([b'blocked'])) with pytest.raises(DownloadException) as ei: - dl.download_music_file(101, music_info=info) + await dl.download_music_file(101, music_info=info) assert '非音频数据' in str(ei.value) assert not part.exists() and not fp.exists() -def test_unknown_size_audio_content_accepted(env): +async def test_unknown_size_audio_content_accepted(env): dl, state, install = env info = _info(size=0) fp, _ = _paths(dl, info) state['sniff'] = True install(lambda a, u, kw: _FakeResponse([b'ID3audio'])) - r = dl.download_music_file(101, music_info=info) + r = await dl.download_music_file(101, music_info=info) assert r.success and fp.read_bytes() == b'ID3audio' -def test_size_limit_discards_part_and_fails(env, monkeypatch): +async def test_size_limit_discards_part_and_fails(env, monkeypatch): dl, _state, install = env monkeypatch.setattr('music_downloader.downloader._MAX_DOWNLOAD_BYTES', 16) info = _info(size=0) @@ -502,7 +528,7 @@ def test_size_limit_discards_part_and_fails(env, monkeypatch): install(lambda a, u, kw: _FakeResponse([b'x' * 100])) with pytest.raises(DownloadException) as ei: - dl.download_music_file(101, music_info=info) + await dl.download_music_file(101, music_info=info) assert '大小上限' in str(ei.value) assert not part.exists() and not fp.exists(), '超限残片无续传价值，须清理' @@ -511,7 +537,7 @@ def test_size_limit_discards_part_and_fails(env, monkeypatch): # ============ 取消 ============ -def test_cancel_before_transfer_clears_part_and_skips_network(env): +async def test_cancel_before_transfer_clears_part_and_skips_network(env): dl, _state, install = env info = _info(size=20) _fp, part = _paths(dl, info) @@ -521,51 +547,42 @@ def test_cancel_before_transfer_clears_part_and_skips_network(env): session = install(lambda *a: pytest.fail('已取消不应发网络请求')) with pytest.raises(DownloadCanceled): - dl.download_music_file(101, music_info=info, cancel_event=cancel) + await dl.download_music_file(101, music_info=info, cancel_event=cancel) assert session.calls == [] assert not part.exists(), '取消须清掉 .part' -def test_cancel_midway_clears_half_written_file(env): +async def test_cancel_midway_clears_half_written_file(env): dl, _state, install = env info = _info(size=20) fp, part = _paths(dl, info) cancel = threading.Event() - def chunks(): - yield b'a' * 5 - cancel.set() - yield b'b' * 15 - - install(lambda a, u, kw: _FakeResponse(chunks())) + # 第二块产出**之前**置取消位：取消发生在迭代过程中（.part 已建立） + install(lambda a, u, kw: _FakeResponse([b'a' * 5, cancel.set, b'b' * 15])) with pytest.raises(DownloadCanceled): - dl.download_music_file(101, music_info=info, cancel_event=cancel) + await dl.download_music_file(101, music_info=info, cancel_event=cancel) assert not part.exists() and not fp.exists() -def test_network_error_while_canceled_reports_canceled(env): +async def test_network_error_while_canceled_reports_canceled(env): dl, _state, install = env info = _info(size=20) fp, part = _paths(dl, info) cancel = threading.Event() - def chunks(): - yield b'a' * 5 - cancel.set() - raise OSError('句柄异常') - - install(lambda a, u, kw: _FakeResponse(chunks())) + install(lambda a, u, kw: _FakeResponse([b'a' * 5, cancel.set, OSError('句柄异常')])) with pytest.raises(DownloadCanceled): - dl.download_music_file(101, music_info=info, cancel_event=cancel, retries=2) + await dl.download_music_file(101, music_info=info, cancel_event=cancel, retries=2) assert not part.exists() and not fp.exists(), '已取消时残片没有续传价值' -def test_cancel_after_promote_keeps_file_but_skips_metadata(env): +async def test_cancel_after_promote_keeps_file_but_skips_metadata(env): dl, state, install = env info = _info(size=20) fp, part = _paths(dl, info) @@ -578,8 +595,8 @@ def test_cancel_after_promote_keeps_file_but_skips_metadata(env): install(_body(20)) with pytest.raises(DownloadCanceled): - dl.download_music_file(101, music_info=info, cancel_event=cancel, - progress_cb=cb) + await dl.download_music_file(101, music_info=info, cancel_event=cancel, + progress_cb=cb) assert fp.exists() and not part.exists(), '已转正的文件不回滚' assert state['meta'] == [], '取消后不应再写元数据' @@ -588,14 +605,14 @@ def test_cancel_after_promote_keeps_file_but_skips_metadata(env): # ============ 元数据 ============ -def test_metadata_written_only_when_requested(env): +async def test_metadata_written_only_when_requested(env): dl, state, install = env install(_body(20)) - dl.download_music_file(101, music_info=_info(mid=101), write_meta=False) + await dl.download_music_file(101, music_info=_info(mid=101), write_meta=False) assert state['meta'] == [], 'write_meta=False 不应写元数据' - r = dl.download_music_file(102, music_info=_info(mid=102), write_meta=True, - meta_fields=['title']) + r = await dl.download_music_file(102, music_info=_info(mid=102), write_meta=True, + meta_fields=['title']) assert r.success assert [f for _p, f in state['meta']] == [['title']], 'fields 应原样透传' diff --git a/tests/test_monitor_jobs.py b/tests/test_monitor_jobs.py index d1a3b17..2e30eec 100644 --- a/tests/test_monitor_jobs.py +++ b/tests/test_monitor_jobs.py @@ -180,23 +180,39 @@ def test_reap_retries_then_gives_up(sandbox): # ============ ② 超时任务取消 ============ -def _finished_thread() -> threading.Thread: - """已结束的线程替身——对应生产里 worker 崩溃/退出的情形（is_alive() 为 False）。""" - th = threading.Thread(target=lambda: None) - th.start() - th.join() - return th +class _WorkerHandle: + """worker 句柄替身——生产里是 asyncio.Task，监控只看它的 ``done()``。 + + 这里用轻量替身而不是真造一个已结束的 Task：Task 的构造依赖运行中的事件循环， + 而监控循环跑在常驻线程里、与任务所在的循环无关；本文件测的是「句柄状态 → + 是否代结算」这段判断，只要 done() 可控即可。 + """ + + def __init__(self, done: bool) -> None: + self._done = done + + def done(self) -> bool: + return self._done + + +def _finished_worker() -> _WorkerHandle: + """已结束的 worker 替身——对应生产里 worker 崩溃/退出的情形（done() 为 True）。""" + return _WorkerHandle(True) + + +def _live_worker() -> _WorkerHandle: + return _WorkerHandle(False) def test_cancel_overtime_settles_dead_worker(sandbox, monkeypatch): - """超硬上限且 worker 线程已死：置取消、唤醒 cancel Event、代为退款。""" + """超硬上限且 worker 已结束：置取消、唤醒 cancel Event、代为退款。""" settled: list[tuple] = [] monkeypatch.setattr(auth, 'ledger_settle', lambda tid, ok, st: settled.append((tid, ok, st))) task = _add_task('t-slow', status='running', start=time.time() - _HARD_TIMEOUT - 60, - worker_thread=_finished_thread()) + worker_task=_finished_worker()) services_tasks._cancel_overtime_tasks(time.time()) assert task['status'] == 'canceled' @@ -204,9 +220,9 @@ def test_cancel_overtime_settles_dead_worker(sandbox, monkeypatch): assert settled == [('t-slow', 0, 'timeout')] -def test_cancel_overtime_without_worker_thread_does_not_settle(sandbox, monkeypatch): - """无 worker_thread 字段的任务不代结算——代结算的前提是「线程曾存在且已死」， - 没有线程句柄就无法判断 worker 是否已在别处结算过，贸然退款会双退。""" +def test_cancel_overtime_without_worker_task_does_not_settle(sandbox, monkeypatch): + """无 worker_task 字段的任务不代结算——代结算的前提是「句柄曾存在且已结束」， + 没有句柄就无法判断 worker 是否已在别处结算过，贸然退款会双退。""" settled: list[tuple] = [] monkeypatch.setattr(auth, 'ledger_settle', lambda tid, ok, st: settled.append((tid, ok, st))) @@ -225,18 +241,11 @@ def test_cancel_overtime_skips_live_worker(sandbox, monkeypatch): monkeypatch.setattr(auth, 'ledger_settle', lambda tid, ok, st: settled.append((tid, ok, st))) - stop = threading.Event() - alive = threading.Thread(target=stop.wait, daemon=True) - alive.start() - try: - task = _add_task('t-live', status='running', - start=time.time() - _HARD_TIMEOUT - 60, worker_thread=alive) - services_tasks._cancel_overtime_tasks(time.time()) - assert task['status'] == 'canceled' - assert settled == [], 'worker 活着 → 不应代结算' - finally: - stop.set() - alive.join(timeout=5) + task = _add_task('t-live', status='running', + start=time.time() - _HARD_TIMEOUT - 60, worker_task=_live_worker()) + services_tasks._cancel_overtime_tasks(time.time()) + assert task['status'] == 'canceled' + assert settled == [], 'worker 活着 → 不应代结算' def test_cancel_overtime_ignores_recent(sandbox): diff --git a/tests/test_podcast_stream.py b/tests/test_podcast_stream.py index c8a232e..56ecc6e 100644 --- a/tests/test_podcast_stream.py +++ b/tests/test_podcast_stream.py @@ -8,6 +8,9 @@ 断言只看「调用后世界里发生了什么」（文件系统 + 任务表），不引用私有函数名， 故可跨后续重构复用。 + +替身边界：monkeypatch ``services.podcast._http``（core.http_client 的进程级异步 +客户端），测试用 ``holder['resp']`` 指定本次响应。 """ import threading @@ -18,8 +21,19 @@ from music_downloader import DownloadCanceled, DownloadException from services import podcast as podcast_svc +class _FakeContent: + """``aiohttp.ClientResponse.content`` 的替身（逐块产出，惰性求值）。""" + + def __init__(self, chunks): + self._chunks = chunks + + async def iter_chunked(self, chunk_size=0): + for item in self._chunks: + yield item + + class _FakeResp: - """`_stream_open` 的替身：可控状态码、Content-Length 与分块序列。 + """``core.http_client.open`` 的替身：可控状态码、Content-Length 与分块序列。 注意不能用 list(chunks) 归一化——那会把传入的生成器**立即求值**，使「迭代中 才触发的副作用」（如中途置取消位）提前到构造时刻，测出来的就不是想测的路径。 @@ -28,16 +42,13 @@ class _FakeResp: def __init__(self, chunks, status=200, content_length=None): self._chunks = chunks self.status = status + self.reason = '' self.headers = {} if content_length is None else {'Content-Length': str(content_length)} + self.content = _FakeContent(chunks) + self.closed = False - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def iter_content(self, chunk_size=0): - yield from self._chunks + def close(self): + self.closed = True class _CancelAfterFirstChunk: @@ -66,14 +77,15 @@ class _CancelAfterFirstChunk: @pytest.fixture() def fake_stream(monkeypatch): - """替换 _stream_open；测试通过 holder['resp'] 指定本次响应。""" + """替换 podcast 模块的 _http 客户端；测试通过 holder['resp'] 指定本次响应。""" holder: dict = {} - def _open(url, timeout=60): - assert url == holder.get('url', url) - return holder['resp'] + class _Client: + async def open(self, method, url, **kw): + assert url == holder.get('url', url) + return holder['resp'] - monkeypatch.setattr(podcast_svc, '_stream_open', _open) + monkeypatch.setattr(podcast_svc, '_http', _Client()) return holder @@ -97,91 +109,93 @@ def task_table(): # ============ 正常路径 ============ -def test_writes_target_and_renames_part(tmp_path, fake_stream): +async def test_writes_target_and_renames_part(tmp_path, fake_stream): """正常下载：返回实际字节数，落地 target，不残留 .part。""" fake_stream['resp'] = _FakeResp([b'a' * 600, b'b' * 400]) target = tmp_path / 'ep.mp3' - done = podcast_svc._podcast_stream_download('http://u', target, 1000, threading.Event()) + done = await podcast_svc._podcast_stream_download('http://u', target, 1000, threading.Event()) assert done == 1000 assert target.read_bytes() == b'a' * 600 + b'b' * 400 assert not target.with_name('ep.mp3.part').exists() -def test_empty_chunks_are_skipped(tmp_path, fake_stream): +async def test_empty_chunks_are_skipped(tmp_path, fake_stream): """空分块不计入字节数（上游偶发发空 chunk）。""" fake_stream['resp'] = _FakeResp([b'', b'x' * 50, b'']) target = tmp_path / 'ep.mp3' - assert podcast_svc._podcast_stream_download('http://u', target, 50, threading.Event()) == 50 + assert await podcast_svc._podcast_stream_download( + 'http://u', target, 50, threading.Event()) == 50 -def test_content_length_used_when_total_unknown(tmp_path, fake_stream): +async def test_content_length_used_when_total_unknown(tmp_path, fake_stream): """未传期望大小时用 Content-Length 兜底，并据此校验完整性。""" fake_stream['resp'] = _FakeResp([b'x' * 30], content_length=30) target = tmp_path / 'ep.mp3' - assert podcast_svc._podcast_stream_download('http://u', target, 0, threading.Event()) == 30 + assert await podcast_svc._podcast_stream_download( + 'http://u', target, 0, threading.Event()) == 30 # ============ 完整性校验 ============ -def test_size_mismatch_removes_delivered_file(tmp_path, fake_stream): +async def test_size_mismatch_removes_delivered_file(tmp_path, fake_stream): """字节数不符：删掉已落地的文件并报错（不交付截断的坏文件）。""" fake_stream['resp'] = _FakeResp([b'x' * 100]) target = tmp_path / 'ep.mp3' with pytest.raises(DownloadException) as ei: - podcast_svc._podcast_stream_download('http://u', target, 200, threading.Event()) + await podcast_svc._podcast_stream_download('http://u', target, 200, threading.Event()) assert '下载不完整' in str(ei.value) assert not target.exists(), '坏文件必须被清掉' -def test_empty_download_raises_and_removes(tmp_path, fake_stream): +async def test_empty_download_raises_and_removes(tmp_path, fake_stream): """期望大小未知且一个字节都没拿到 → 报「下载内容为空」。""" fake_stream['resp'] = _FakeResp([]) target = tmp_path / 'ep.mp3' with pytest.raises(DownloadException) as ei: - podcast_svc._podcast_stream_download('http://u', target, 0, threading.Event()) + await podcast_svc._podcast_stream_download('http://u', target, 0, threading.Event()) assert '下载内容为空' in str(ei.value) assert not target.exists() -def test_known_size_does_not_tolerate_partial(tmp_path, fake_stream): +async def test_known_size_does_not_tolerate_partial(tmp_path, fake_stream): """已知期望大小时要求精确相等——95% 容差会放行截断的坏文件。""" fake_stream['resp'] = _FakeResp([b'x' * 95]) target = tmp_path / 'ep.mp3' with pytest.raises(DownloadException): - podcast_svc._podcast_stream_download('http://u', target, 100, threading.Event()) + await podcast_svc._podcast_stream_download('http://u', target, 100, threading.Event()) # ============ 上游与取消 ============ -def test_upstream_error_status_raises(tmp_path, fake_stream): +async def test_upstream_error_status_raises(tmp_path, fake_stream): fake_stream['resp'] = _FakeResp([], status=404) target = tmp_path / 'ep.mp3' with pytest.raises(DownloadException) as ei: - podcast_svc._podcast_stream_download('http://u', target, 100, threading.Event()) + await podcast_svc._podcast_stream_download('http://u', target, 100, threading.Event()) assert '上游返回 404' in str(ei.value) assert not target.exists() -def test_cancel_before_first_chunk(tmp_path, fake_stream): +async def test_cancel_before_first_chunk(tmp_path, fake_stream): """取消已置位：抛 DownloadCanceled，不发布 target。""" cancel = threading.Event() cancel.set() fake_stream['resp'] = _FakeResp([b'x' * 10]) target = tmp_path / 'ep.mp3' with pytest.raises(DownloadCanceled): - podcast_svc._podcast_stream_download('http://u', target, 10, cancel) + await podcast_svc._podcast_stream_download('http://u', target, 10, cancel) assert not target.exists() -def test_cancel_midway_leaves_part_file(tmp_path, fake_stream): +async def test_cancel_midway_leaves_part_file(tmp_path, fake_stream): """下载中途取消：.part 残留（不删）——改名不发生，下次同名下载直接覆盖。""" cancel = threading.Event() fake_stream['resp'] = _FakeResp(_CancelAfterFirstChunk(cancel)) target = tmp_path / 'ep.mp3' with pytest.raises(DownloadCanceled): - podcast_svc._podcast_stream_download('http://u', target, 20, cancel) + await podcast_svc._podcast_stream_download('http://u', target, 20, cancel) assert not target.exists() assert target.with_name('ep.mp3.part').exists(), '既有行为：取消时 .part 留存' @@ -189,7 +203,7 @@ def test_cancel_midway_leaves_part_file(tmp_path, fake_stream): # ============ 进度消息 ============ -def test_progress_written_for_running_single_task(tmp_path, fake_stream, task_table): +async def test_progress_written_for_running_single_task(tmp_path, fake_stream, task_table): """running 的单集任务：写 msg，并按字节级口径写 live_percent。 单块即完成 → 百分比被 min(99, ...) 封在 99（进度条不预支 100%，收尾靠终态）。 @@ -197,24 +211,25 @@ def test_progress_written_for_running_single_task(tmp_path, fake_stream, task_ta task = task_table() fake_stream['resp'] = _FakeResp([b'x' * 100]) target = tmp_path / 'ep.mp3' - podcast_svc._podcast_stream_download('http://u', target, 100, threading.Event(), - task_id=task['id'], progress_label='下载中') + await podcast_svc._podcast_stream_download('http://u', target, 100, threading.Event(), + task_id=task['id'], progress_label='下载中') assert '下载中' in task['msg'] and '99%' in task['msg'] assert task['live_percent'] == 99.0 -def test_progress_not_written_for_finished_task(tmp_path, fake_stream, task_table): +async def test_progress_not_written_for_finished_task(tmp_path, fake_stream, task_table): """已终态的任务不再被写进度（避免覆盖终态消息），但下载照常完成。""" task = task_table(status='done', msg='原消息') fake_stream['resp'] = _FakeResp([b'x' * 50]) target = tmp_path / 'ep.mp3' - assert podcast_svc._podcast_stream_download('http://u', target, 50, threading.Event(), - task_id=task['id']) == 50 + assert await podcast_svc._podcast_stream_download( + 'http://u', target, 50, threading.Event(), task_id=task['id']) == 50 assert task['msg'] == '原消息' -def test_progress_skipped_without_task_id(tmp_path, fake_stream): +async def test_progress_skipped_without_task_id(tmp_path, fake_stream): """task_id 为空时只下载不写任务表（不得因缺 id 而报错）。""" fake_stream['resp'] = _FakeResp([b'x' * 50]) target = tmp_path / 'ep.mp3' - assert podcast_svc._podcast_stream_download('http://u', target, 50, threading.Event()) == 50 + assert await podcast_svc._podcast_stream_download( + 'http://u', target, 50, threading.Event()) == 50 diff --git a/tests/test_podcast_worker.py b/tests/test_podcast_worker.py index 870216c..905e80f 100644 --- a/tests/test_podcast_worker.py +++ b/tests/test_podcast_worker.py @@ -19,6 +19,7 @@ except 分支各写一遍「结算 + 删任务目录」，正文里散着三处 账本口径：一律用 ``reserve_and_open`` 真预扣（与 test_single_worker.py 同理由）。 """ +import asyncio import threading import time import zipfile @@ -83,9 +84,9 @@ class _FakeStream: self.on_call = on_call self.calls: list[dict[str, Any]] = [] - def __call__(self, url: str, target: Path, total_size: int, - cancel_ev: threading.Event, task_id: str = '', - progress_label: str = '') -> int: + async def __call__(self, url: str, target: Path, total_size: int, + cancel_ev: threading.Event, task_id: str = '', + progress_label: str = '') -> int: self.calls.append({'url': url, 'target': Path(target), 'total_size': total_size}) if self.on_call is not None: @@ -172,13 +173,20 @@ def _register_podcast(sandbox, *, program_id: int = 777, user_id: int, def _install_upstream(monkeypatch, *, program: dict[str, Any] | None = None, url_info: dict[str, Any] | None = None, stream: Any = None, - cover: Any = lambda url: b'\xff\xd8\xff cover') -> None: + cover: Any = None) -> None: """装上播客上游替身（节目详情 / 直链 / 流式下载 / 封面）。""" - monkeypatch.setattr(podcast_svc.podcast, 'get_program_detail', - lambda pid, cookies: program if program is not None else _program()) - monkeypatch.setattr(podcast_svc.podcast, 'resolve_episode_url', - lambda sid, cookies, quality: - url_info if url_info is not None else _url_info()) + async def _detail(pid: Any, cookies: Any) -> dict[str, Any]: + return program if program is not None else _program() + + async def _resolve(sid: Any, cookies: Any, quality: str) -> dict[str, Any]: + return url_info if url_info is not None else _url_info() + + monkeypatch.setattr(podcast_svc.podcast, 'get_program_detail', _detail) + monkeypatch.setattr(podcast_svc.podcast, 'resolve_episode_url', _resolve) + if cover is None: + async def _default_cover(_url: str) -> bytes: + return b'\xff\xd8\xff cover' + cover = _default_cover if stream is None: stream = _FakeStream() monkeypatch.setattr(podcast_svc, '_podcast_stream_download', stream) @@ -186,8 +194,11 @@ def _install_upstream(monkeypatch, *, program: dict[str, Any] | None = None, def _run(task_id: str) -> threading.Thread: - th = threading.Thread(target=podcast_svc._podcast_download_worker, - args=(task_id,), daemon=True) + """把 worker 协程跑在独立线程的私有事件循环里（等价于生产的任务循环）。""" + def _target() -> None: + asyncio.run(podcast_svc._podcast_download_worker(task_id)) + + th = threading.Thread(target=_target, daemon=True) th.start() return th @@ -332,7 +343,7 @@ def test_cover_fetch_failure_still_delivers_zip(sandbox, monkeypatch): """取封面抛异常：被吞掉，ZIP 照常交付（历史行为：封面失败不炸任务）。""" uid = _uid() - def _boom(url: str) -> bytes: + async def _boom(url: str) -> bytes: raise RuntimeError('cover cdn down') _install_upstream(monkeypatch, cover=_boom) @@ -402,7 +413,7 @@ def test_unexpected_exception_uses_generic_message(sandbox, monkeypatch): uid = _uid() before = _balance() - def _boom(pid: Any, cookies: Any) -> dict[str, Any]: + async def _boom(pid: Any, cookies: Any) -> dict[str, Any]: raise RuntimeError('upstream host=10.0.0.9 refused') _install_upstream(monkeypatch) @@ -468,7 +479,7 @@ def test_cancel_during_zip_stage(sandbox, monkeypatch): before = _balance() holder: dict[str, Any] = {} - def _cancel_in_cover(url: str) -> bytes: + async def _cancel_in_cover(url: str) -> bytes: holder['task']['cancel'].set() return b'\xff\xd8\xff cover' diff --git a/tests/test_resolve_episode_url.py b/tests/test_resolve_episode_url.py index 9a0c7fa..cade047 100644 --- a/tests/test_resolve_episode_url.py +++ b/tests/test_resolve_episode_url.py @@ -8,8 +8,9 @@ - 未知音质 → 该音质优先 + 通用阶梯 - song_id 垃圾值 → None 且不发任何请求 -替身边界：monkeypatch ``podcast.url_v1``（模块级名字，podcast.py 从 -.compat 导入后的绑定），按 level 返回结果或抛异常，并记录调用序列。 +替身边界：monkeypatch ``music_api.podcast.NeteaseAPI``——resolve_episode_url 内部 +现构造该客户端并 ``await api.get_song_url(...)``，替身按 level 返回结果或抛异常， +并记录调用序列。 """ import pytest @@ -23,14 +24,18 @@ def url_calls(monkeypatch): calls: list[tuple[int, str]] = [] state: dict = {'results': {}} - def _fake_url_v1(sid, level, cookies): + async def _fake_get_song_url(sid, level, cookies): calls.append((sid, level)) r = state['results'].get(level) if isinstance(r, Exception): raise r return r - monkeypatch.setattr(podcast_mod, 'url_v1', _fake_url_v1) + class _FakeAPI: + async def get_song_url(self, sid, level, cookies): + return await _fake_get_song_url(sid, level, cookies) + + monkeypatch.setattr(podcast_mod, 'NeteaseAPI', _FakeAPI) state['calls'] = calls return state @@ -42,38 +47,38 @@ def _ok(level: str = 'standard') -> dict: async def test_auto_tries_quality_ladder_in_order(url_calls): """auto：按 QUALITY_LADDER 原序尝试，全失败返回 None。""" - assert resolve_episode_url(101, {}, 'auto') is None + assert await resolve_episode_url(101, {}, 'auto') is None assert url_calls['calls'] == [(101, 'exhigh'), (101, 'standard')] async def test_explicit_level_falls_down_only(url_calls): """显式 lossless：向后降级 exhigh/standard，绝不向上尝试 hires。""" - assert resolve_episode_url(101, {}, 'lossless') is None + assert await resolve_episode_url(101, {}, 'lossless') is None assert url_calls['calls'] == [(101, 'lossless'), (101, 'exhigh'), (101, 'standard')] async def test_standard_never_falls_up(url_calls): """显式 standard：只试一次（历史坑：原实现会回退到更高的 exhigh）。""" - assert resolve_episode_url(101, {}, 'standard') is None + assert await resolve_episode_url(101, {}, 'standard') is None assert url_calls['calls'] == [(101, 'standard')] async def test_unknown_level_prepends_generic_ladder(url_calls): """未知音质：先试该音质，再退通用阶梯（去重保序）。""" - assert resolve_episode_url(101, {}, 'sky') is None + assert await resolve_episode_url(101, {}, 'sky') is None assert url_calls['calls'] == [(101, 'sky'), (101, 'exhigh'), (101, 'standard')] async def test_first_hit_returns_mapped_result(url_calls): url_calls['results']['exhigh'] = _ok('hires') - out = resolve_episode_url(101, {}, 'auto') + out = await resolve_episode_url(101, {}, 'auto') assert out == {'url': 'https://x/1.mp3', 'level': 'hires', 'type': 'mp3', 'size': 123, 'br': 128000, 'md5': 'abc', 'requested_level': 'exhigh'} async def test_level_defaults_to_requested_when_missing(url_calls): url_calls['results']['exhigh'] = {'data': [{'url': 'https://x/1.mp3'}]} - out = resolve_episode_url(101, {}, 'auto') + out = await resolve_episode_url(101, {}, 'auto') assert out is not None assert out['level'] == 'exhigh' assert out['type'] == 'mp3' @@ -84,7 +89,7 @@ async def test_level_defaults_to_requested_when_missing(url_calls): async def test_api_exception_tries_next_level(url_calls): url_calls['results']['exhigh'] = APIException('无版权') url_calls['results']['standard'] = _ok() - out = resolve_episode_url(101, {}, 'auto') + out = await resolve_episode_url(101, {}, 'auto') assert out is not None assert out['requested_level'] == 'standard' assert url_calls['calls'] == [(101, 'exhigh'), (101, 'standard')] @@ -94,7 +99,7 @@ async def test_missing_url_tries_next_level(url_calls): """data[0] 无 url：视同失败，继续降级。""" url_calls['results']['exhigh'] = {'data': [{'url': None}]} url_calls['results']['standard'] = _ok() - out = resolve_episode_url(101, {}, 'auto') + out = await resolve_episode_url(101, {}, 'auto') assert out is not None assert out['requested_level'] == 'standard' @@ -102,7 +107,7 @@ async def test_missing_url_tries_next_level(url_calls): async def test_empty_data_tries_next_level(url_calls): url_calls['results']['exhigh'] = None # res 为 None → data 为 [] url_calls['results']['standard'] = _ok() - out = resolve_episode_url(101, {}, 'auto') + out = await resolve_episode_url(101, {}, 'auto') assert out is not None assert out['requested_level'] == 'standard' @@ -110,12 +115,12 @@ async def test_empty_data_tries_next_level(url_calls): async def test_bad_song_id_returns_none_without_requests(url_calls): """song_id 垃圾值：None，且不发任何请求（防御原 TypeError 500）。""" for bad in (None, 'abc', ''): - assert resolve_episode_url(bad, {}, 'auto') is None + assert await resolve_episode_url(bad, {}, 'auto') is None assert url_calls['calls'] == [] async def test_numeric_string_song_id_accepted(url_calls): url_calls['results']['exhigh'] = _ok() - out = resolve_episode_url(' 101 ', {}, 'auto') + out = await resolve_episode_url(' 101 ', {}, 'auto') assert out is not None assert url_calls['calls'][0] == (101, 'exhigh') diff --git a/tests/test_single_worker.py b/tests/test_single_worker.py index 1a29915..4ce34d7 100644 --- a/tests/test_single_worker.py +++ b/tests/test_single_worker.py @@ -13,6 +13,7 @@ 账本口径：一律用 ``reserve_and_open`` 真预扣（``ledger_open`` 只记账不扣款， 会让「退款」凭空给余额加次数，守恒断言就失去意义）。 """ +import asyncio import threading import time import zipfile @@ -62,7 +63,7 @@ class _FakeDownloader: self.info_calls = 0 self.download_calls = 0 - def get_music_info(self, mid: Any, quality: str, cookies: Any) -> MusicInfo: + async def get_music_info(self, mid: Any, quality: str, cookies: Any) -> MusicInfo: self.info_calls += 1 if self.info_error is not None: raise self.info_error @@ -73,7 +74,7 @@ class _FakeDownloader: quality='standard', lyric=self.lyric, ) - def download_music_file(self, mid: Any, quality: str, **kw: Any) -> DownloadResult: + async def download_music_file(self, mid: Any, quality: str, **kw: Any) -> DownloadResult: self.download_calls += 1 if self.on_download is not None: self.on_download(kw) @@ -110,7 +111,10 @@ def sandbox(tmp_path, monkeypatch): monkeypatch.setattr(api_service, 'downloads_path', downloads) monkeypatch.setattr(api_service, 'logger', _NullLogger()) # 封面走网络——测试里默认返回固定字节，个例可覆盖 - monkeypatch.setattr(downloads_svc, 'fetch_cover_bytes', lambda url: b'\xff\xd8\xff cover') + async def _cover(_url: str) -> bytes: + return b'\xff\xd8\xff cover' + + monkeypatch.setattr(downloads_svc, 'fetch_cover_bytes', _cover) # 操作记录写进程内共享列表——隔离掉，断言改由个例捕获 monkeypatch.setattr(downloads_svc, '_record_log', lambda *a: None) @@ -167,8 +171,11 @@ def _register_single(sandbox, *, music_id: int = 1, user_id: int, def _run(task_id: str) -> threading.Thread: - th = threading.Thread(target=downloads_svc._single_download_worker, - args=(task_id,), daemon=True) + """把 worker 协程跑在独立线程的私有事件循环里（等价于生产的任务循环）。""" + def _target() -> None: + asyncio.run(downloads_svc._single_download_worker(task_id)) + + th = threading.Thread(target=_target, daemon=True) th.start() return th @@ -275,7 +282,11 @@ def test_cover_failure_keeps_zip_without_cover(sandbox, monkeypatch): """ uid = _uid() monkeypatch.setattr(api_service, 'downloader', _FakeDownloader(sandbox['cache'])) - monkeypatch.setattr(downloads_svc, 'fetch_cover_bytes', lambda url: None) + + async def _no_cover(_url: str) -> None: + return None + + monkeypatch.setattr(downloads_svc, 'fetch_cover_bytes', _no_cover) tid, task = _register_single(sandbox, music_id=105, user_id=uid, need_zip=True) th = _run(tid) th.join(timeout=30) diff --git a/tests/test_zip_workers.py b/tests/test_zip_workers.py index b655a4f..d4e0e5c 100644 --- a/tests/test_zip_workers.py +++ b/tests/test_zip_workers.py @@ -16,6 +16,7 @@ ZIP 内容、账本守恒、暂停/恢复/取消语义。断言只依赖「调 账本口径：一律用 ``reserve_and_open`` 真预扣（``ledger_open`` 只记账不扣款， 会让「退款」凭空给余额加次数，守恒断言就失去意义）。 """ +import asyncio import threading import time import zipfile @@ -59,7 +60,7 @@ class _FakeDownloader: self.calls: list[int] = [] self._lock = threading.Lock() - def get_music_info(self, mid: Any, quality: str, cookies: Any) -> MusicInfo: + async def get_music_info(self, mid: Any, quality: str, cookies: Any) -> MusicInfo: return MusicInfo( id=int(mid), name=f'歌{mid}', artists='歌手A', album='专辑A', pic_url='', duration=200, track_number=1, download_url='', @@ -67,7 +68,7 @@ class _FakeDownloader: lyric='[00:01.00]第一行\n[00:02.00]第二行', ) - def download_music_file(self, mid: Any, quality: str, **kw: Any) -> DownloadResult: + async def download_music_file(self, mid: Any, quality: str, **kw: Any) -> DownloadResult: with self._lock: self.calls.append(int(mid)) seq = len(self.calls) @@ -199,7 +200,11 @@ def _register_podcast(sandbox, pids: list[int], *, reserved: int, user_id: int) def _run(worker, task_id: str) -> threading.Thread: - th = threading.Thread(target=worker, args=(task_id,), daemon=True) + """把 worker 协程跑在独立线程的私有事件循环里（等价于生产的任务循环）。""" + def _target() -> None: + asyncio.run(worker(task_id)) + + th = threading.Thread(target=_target, daemon=True) th.start() return th @@ -346,8 +351,9 @@ def test_batch_zip_cancel_at_publish_checkpoint_delivers(sandbox, monkeypatch): holder: dict[str, Any] = {} real_settle = batch_svc._settle_batch_billing - def settle_then_cancel(task_id: str, ok_count: int, final_status: str = 'settled') -> int: - refunded = real_settle(task_id, ok_count, final_status) + async def settle_then_cancel(task_id: str, ok_count: int, + final_status: str = 'settled') -> int: + refunded = await real_settle(task_id, ok_count, final_status) holder['task']['cancel'].set() # 结算已完成、发布前一刻取消 return refunded @@ -371,24 +377,27 @@ def test_batch_zip_cancel_at_publish_checkpoint_delivers(sandbox, monkeypatch): @pytest.fixture() def podcast_stub(monkeypatch, sandbox): """把播客上游与流式下载替换为本地确定性实现。""" - def fake_detail(pid: Any, cookies: Any) -> dict[str, Any]: + async def fake_detail(pid: Any, cookies: Any) -> dict[str, Any]: n = int(pid) return {'id': n, 'name': f'单集{n}', 'main_song_id': 1, 'serial_num': n, 'cover_url': '', 'songs': []} - def fake_resolve(prog: dict[str, Any], quality: str, cookies: Any) -> dict[str, Any]: + async def fake_resolve(prog: dict[str, Any], quality: str, cookies: Any) -> dict[str, Any]: return {'url': f"http://example.invalid/{prog['id']}.mp3", 'size': 2048, 'type': 'mp3'} - def fake_stream(url: str, target: Path, total_size: int, cancel_ev: Any, - task_id: str = '', progress_label: str = '') -> int: + async def fake_stream(url: str, target: Path, total_size: int, cancel_ev: Any, + task_id: str = '', progress_label: str = '') -> int: if cancel_ev is not None and cancel_ev.is_set(): raise DownloadCanceled('任务已取消') target.write_bytes(b'\xff\xfb\x90\x00' + b'\x00' * 128) return target.stat().st_size + async def _no_cover(_u: str) -> None: + return None + monkeypatch.setattr(podcast_svc, '_podcast_resolve_audio', fake_resolve) monkeypatch.setattr(podcast_svc, '_podcast_stream_download', fake_stream) - monkeypatch.setattr(podcast_svc, 'fetch_cover_bytes', lambda _u: None) + monkeypatch.setattr(podcast_svc, 'fetch_cover_bytes', _no_cover) monkeypatch.setattr(podcast_svc.podcast, 'get_program_detail', fake_detail) return sandbox @@ -418,7 +427,7 @@ def test_podcast_zip_pause_then_resume(podcast_stub, monkeypatch): holder: dict[str, Any] = {} fired = {'n': 0} - def _pause_once(url, target, total_size, cancel_ev, task_id='', progress_label=''): + async def _pause_once(url, target, total_size, cancel_ev, task_id='', progress_label=''): target.write_bytes(b'\xff\xfb\x90\x00' + b'\x00' * 128) fired['n'] += 1 if fired['n'] == 1: # 只在第一集触发——否则恢复后立刻又被暂停 @@ -447,7 +456,7 @@ def test_podcast_zip_all_fail_refunds(podcast_stub, monkeypatch): uid = _uid() before = _balance() - def _boom(prog, quality, cookies): + async def _boom(prog, quality, cookies): raise RuntimeError('上游不可用') monkeypatch.setattr(podcast_svc, '_podcast_resolve_audio', _boom) @@ -470,9 +479,9 @@ def test_podcast_zip_all_fail_refunds(podcast_stub, monkeypatch): class _SameNameDownloader(_FakeDownloader): """所有歌曲同名同歌手、且都带封面——用于验证包内三条目共享序号后缀。""" - def get_music_info(self, mid: Any, quality: str, cookies: Any) -> MusicInfo: - return replace(super().get_music_info(mid, quality, cookies), - name='同名歌', artists='同歌手', + async def get_music_info(self, mid: Any, quality: str, cookies: Any) -> MusicInfo: + base = await super().get_music_info(mid, quality, cookies) + return replace(base, name='同名歌', artists='同歌手', pic_url='http://example.invalid/cover.jpg') @@ -485,7 +494,10 @@ def test_batch_zip_dedups_three_arcnames_with_shared_suffix(sandbox, monkeypatch """ uid = _uid() monkeypatch.setattr(api_service, 'downloader', _SameNameDownloader(sandbox['cache'])) - monkeypatch.setattr(batch_svc, 'fetch_cover_bytes', lambda _u: b'\xff\xd8\xff\xe0') + async def _cover(_u: str) -> bytes: + return b'\xff\xd8\xff\xe0' + + monkeypatch.setattr(batch_svc, 'fetch_cover_bytes', _cover) tid, task = _register_batch(sandbox, [701, 702], reserved=2, user_id=uid) th = _run(batch_svc._batch_zip_worker, tid) @@ -528,7 +540,7 @@ def test_podcast_all_fail_message_and_ledger_status(podcast_stub, monkeypatch): uid = _uid() before = _balance() - def _boom(prog, quality, cookies): + async def _boom(prog, quality, cookies): raise RuntimeError('上游不可用') monkeypatch.setattr(podcast_svc, '_podcast_resolve_audio', _boom) @@ -557,8 +569,8 @@ def test_podcast_cleans_temp_files_on_episode_failure(podcast_stub, monkeypatch) uid = _uid() before = _balance() - def _flaky(url: str, target: Path, total_size: int, cancel_ev: Any, - task_id: str = '', progress_label: str = '') -> int: + async def _flaky(url: str, target: Path, total_size: int, cancel_ev: Any, + task_id: str = '', progress_label: str = '') -> int: if '/102.' in url: target.write_bytes(b'\x00' * 64) # 正文残留 target.with_name(target.name + '.part').write_bytes(b'\x00' * 32) # 残片 diff --git a/web/app.py b/web/app.py index 14352e7..79ca67d 100644 --- a/web/app.py +++ b/web/app.py @@ -40,7 +40,7 @@ def _make_lifespan(start_monitor: bool): @asynccontextmanager async def _lifespan(app: FastAPI): - """应用生命周期：建表 → 启动任务监控 → 关闭时释放连接池与后台 loop。 + """应用生命周期：建表 → 启动任务监控 → 关闭时释放连接池。 启动必须建表。全新部署时宿主机的 data/ 往往是个空目录（compose 用 `./data:/app/data` 挂载，会盖掉镜像内的任何内容），此时若只等首次查询 @@ -62,12 +62,9 @@ def _make_lifespan(start_monitor: bool): await asyncio.to_thread(start_task_monitor) yield try: - from core import async_bridge, http_client + from core import http_client await http_client.client.close() - # 停掉 sync↔async 桥的后台事件循环（daemon 线程不调也能随进程退出， - # 但显式关闭更干净） - async_bridge.shutdown() except Exception: pass diff --git a/web/routers/download.py b/web/routers/download.py index a01d338..25af033 100644 --- a/web/routers/download.py +++ b/web/routers/download.py @@ -12,7 +12,6 @@ settle/rename 消费过的行天然豁免）；任务登记与启动由 web.tasks_registry 承担； worker 复用 services 层既有实现，账本与交付语义不变。 """ -import asyncio import logging import threading import time @@ -72,19 +71,17 @@ async def _sync_download_json(downloader, music_id: str, quality: str, cookies: 只负责在成功路径 settle(1, 'delivered')（结算后自动退自然豁免）。 """ try: - info = await asyncio.to_thread( - downloader.get_music_info, int(music_id), quality, cookies) + info = await downloader.get_music_info(int(music_id), quality, cookies) except Exception as e: logger.error('获取音乐信息失败: %s', e) return error('获取失败：歌曲不存在或暂无版权', 404) from services._shared import _build_download_base - base = await asyncio.to_thread( - _build_download_base, filename_format, - {'name': info.name, 'artist_string': info.artists}) + base = _build_download_base( + filename_format, {'name': info.name, 'artist_string': info.artists}) filename = f'{base}.{info.file_type}' try: - dl = await asyncio.to_thread( - downloader.download_music_file, int(music_id), quality, + dl = await downloader.download_music_file( + int(music_id), quality, write_meta=write_meta, meta_fields=meta_fields, cookies=cookies, music_info=info) if not dl.success: @@ -190,7 +187,7 @@ async def download(user: Annotated[SiteUser | None, Depends(current_user)], requ from services import downloads as services_downloads from services import tasks as services_tasks device_id = str(data.get('device_id') or '').strip()[:64] - await asyncio.to_thread(services_tasks._touch_device, device_id) + services_tasks._touch_device(device_id) task = _single_task({ 'music_id': music_id, 'quality': quality, 'filename_format': filename_format, 'write_meta': write_meta, @@ -273,7 +270,7 @@ async def download_batch_zip(user: Annotated[SiteUser | None, Depends(current_us device_id = str(data.get('device_id') or '').strip()[:64] from services import batch as services_batch from services import tasks as services_tasks - await asyncio.to_thread(services_tasks._touch_device, device_id) + services_tasks._touch_device(device_id) task = { 'type': 'batch', 'total': len(ids), 'done': 0, 'failed': 0, @@ -337,7 +334,7 @@ async def podcast_download(user: Annotated[SiteUser | None, Depends(current_user from music_api import podcast quality = _podcast_quality(data) need_zip = str(data.get('zip', 'false')).lower() in ('1', 'true', 'yes') - ref = podcast.extract_podcast_ref(raw) + ref = await podcast.extract_podcast_ref(raw) if ref['kind'] == 'radio': return error('这是电台链接，请先解析电台后选择单集下载，或使用「打包整个播客」') @@ -352,7 +349,7 @@ async def podcast_download(user: Annotated[SiteUser | None, Depends(current_user device_id = str(data.get('device_id') or '').strip()[:64] from services import podcast as services_podcast from services import tasks as services_tasks - await asyncio.to_thread(services_tasks._touch_device, device_id) + services_tasks._touch_device(device_id) task = { 'type': 'single', 'total': 1, 'done': 0, 'failed': 0, @@ -403,13 +400,13 @@ async def podcast_batch_zip(user: Annotated[SiteUser | None, Depends(current_use from music_api import podcast if raw_radio: - ref = podcast.extract_podcast_ref(raw_radio) + ref = await podcast.extract_podcast_ref(raw_radio) if ref['kind'] == 'program': return error('这是单集链接，请使用「下载单集」或直接输入电台链接/ID') radio_id = ref['id'] radio_name = '' try: - radio_name = podcast.get_radio_detail(radio_id, cookies).get('name', '') + radio_name = (await podcast.get_radio_detail(radio_id, cookies)).get('name', '') except APIException: pass else: @@ -423,7 +420,7 @@ async def podcast_batch_zip(user: Annotated[SiteUser | None, Depends(current_use device_id = str(data.get('device_id') or '').strip()[:64] from services import podcast as services_podcast from services import tasks as services_tasks - await asyncio.to_thread(services_tasks._touch_device, device_id) + services_tasks._touch_device(device_id) task = { 'type': 'batch', 'total': 0, 'done': 0, 'failed': 0, diff --git a/web/routers/podcast.py b/web/routers/podcast.py index fb1ad59..820fef4 100644 --- a/web/routers/podcast.py +++ b/web/routers/podcast.py @@ -7,10 +7,9 @@ - POST/GET /api/v1/podcast/search ↔ Flask /podcast/search - POST/GET /api/v1/podcast/hot ↔ Flask /podcast/hot -说明：music_api.podcast 是同步模块（底层 core.sync_http），此处以 asyncio.to_thread 包装， -让阻塞的上游调用不占用事件循环——行为不变，且不阻塞其它请求。 +说明：music_api.podcast 是原生异步模块（底层 core.http_client），此处直接 await； +解析与直链检查同属一次请求，不再需要线程卸载。 """ -import asyncio from typing import Annotated from fastapi import APIRouter, Depends, Request @@ -51,8 +50,7 @@ async def podcast_parse(_user: LoginUser, request: Request): cookies = _cookies() try: - result = await asyncio.to_thread( - podcast.parse_podcast_ref, raw, cookies, page_size=page_size) + result = await podcast.parse_podcast_ref(raw, cookies, page_size=page_size) except APIException as e: return error(str(e), 404) except Exception: @@ -64,8 +62,7 @@ async def podcast_parse(_user: LoginUser, request: Request): url_info = None if song_id: try: - url_info = await asyncio.to_thread( - podcast.resolve_episode_url, song_id, cookies, quality) + url_info = await podcast.resolve_episode_url(song_id, cookies, quality) except Exception: url_info = None if url_info: @@ -110,9 +107,8 @@ async def podcast_programs(_user: LoginUser, request: Request): if not str(radio_id).isdigit(): return error('无效的电台 ID') try: - result = await asyncio.to_thread( - podcast.get_radio_programs, radio_id, _cookies(), - limit=limit, offset=offset, asc=asc) + result = await podcast.get_radio_programs(radio_id, _cookies(), + limit=limit, offset=offset, asc=asc) return success(result, '获取单集列表成功') except APIException as e: return error(str(e), 404) @@ -134,9 +130,8 @@ async def podcast_search(_user: LoginUser, request: Request): if search_type not in ('all', 'radio', 'voice'): search_type = 'all' try: - result = await asyncio.to_thread( - podcast.search_podcast, keyword, _cookies(), - limit=limit, offset=offset, search_type=search_type) + result = await podcast.search_podcast(keyword, _cookies(), + limit=limit, offset=offset, search_type=search_type) return success(result, '播客搜索完成') except APIException as e: return error(str(e), 404) @@ -153,14 +148,13 @@ async def podcast_hot(_user: LoginUser, request: Request): offset = max(0, _safe_int(data.get('offset'), 0)) cate_id = data.get('cate_id') or None try: - radios = await asyncio.to_thread( - podcast.get_hot_radios, _cookies(), limit=limit, offset=offset, cate_id=cate_id) + radios = await podcast.get_hot_radios(_cookies(), limit=limit, offset=offset, cate_id=cate_id) except APIException as e: return error(str(e), 404) except Exception: return error('服务暂时不可用，请稍后再试', 500) try: - categories = await asyncio.to_thread(podcast.get_categories) + categories = await podcast.get_categories() except APIException: categories = [] # 满页视为可能还有更多 diff --git a/web/routers/tasks.py b/web/routers/tasks.py index a8beef5..ab20ab5 100644 --- a/web/routers/tasks.py +++ b/web/routers/tasks.py @@ -12,7 +12,6 @@ 这些端点读写 core.state 的共享任务表（与任务执行侧同一批对象），因此 看到的是同一份任务状态。 """ -import asyncio import time from typing import Annotated, Any @@ -173,7 +172,7 @@ async def task_heartbeat(user: LoginUser, request: Request): device_id = str(data.get('device_id') or '').strip()[:64] if not device_id: return error('缺少 device_id 参数') - await asyncio.to_thread(services_tasks._touch_device, device_id) + services_tasks._touch_device(device_id) resumed = 0 with DOWNLOAD_TASKS_LOCK: for task in DOWNLOAD_TASKS.values(): @@ -199,7 +198,7 @@ async def task_list(user: LoginUser, request: Request): device_id = str(data.get('device_id') or '').strip()[:64] if not device_id: return error('缺少 device_id 参数') - await asyncio.to_thread(services_tasks._touch_device, device_id) + services_tasks._touch_device(device_id) my_tasks = [] with DOWNLOAD_TASKS_LOCK: for tid, task in DOWNLOAD_TASKS.items(): diff --git a/web/tasks_registry.py b/web/tasks_registry.py index 338cb68..92024ef 100644 --- a/web/tasks_registry.py +++ b/web/tasks_registry.py @@ -7,7 +7,7 @@ import logging import re import uuid -from collections.abc import Callable +from collections.abc import Awaitable, Callable from typing import Any from core.config import MAX_ACTIVE_TASKS_PER_USER, MAX_TERMINAL_TASKS, MAX_TOTAL_TASKS @@ -71,9 +71,12 @@ async def register_download_task(billing: Billing, task: dict[str, Any], return None -async def start_task(task_id: str, worker: Callable, args: tuple = ()) -> str | None: - """启动 worker 线程（复用 services 的 _start_task_thread，失败收敛终态并退费）。""" - import asyncio +async def start_task(task_id: str, worker: Callable[..., Awaitable[None]], + args: tuple = ()) -> str | None: + """启动 worker 协程（复用 services 的 _start_task，失败收敛终态并退费）。 - from services._taskcore import _start_task_thread - return await asyncio.to_thread(_start_task_thread, task_id, worker, args) + worker 已是协程函数：直接作为 asyncio 任务挂到当前事件循环（不再起线程）， + 任务句柄记在任务字典里供监控循环判断「worker 已结束但没人结算」的兜底场景。 + """ + from services._taskcore import _start_task + return _start_task(task_id, worker, args)