This repository has been archived on 2026-10-02. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
NeteaseUrl/music_downloader/downloader.py
T
snowy-yang 69a50c2c31 refactor: worker 子系统全面异步化,拆掉同步转异步的桥接层
拆掉「同步调异步 / 异步桥同步」两类垫片的第二半(HTTP 线 B2 见前序提交)。
worker 侧此前是「起线程跑同步函数、函数体内部再用 asyncio.to_thread 把活塞给
另一条循环」——线程与协程互等,取消/暂停只能靠轮询。本轮把 worker 自身变成
协程,由 _start_task 直接挂到应用事件循环上,桥接层随之失去存在理由。

改动(按子系统):
- music_downloader:downloader / _aria2 / _concurrency / cover / _http /
  __main__ 全链路协程化。per-file 锁改 asyncio.Lock;计数闸改
  「active 计数 + await asyncio.sleep(0.5)」的可取消等待(原阻塞式 with 会让
  已取消任务仍要等全局名额腾出,卡死并拖后腿退款);aria2p 的同步 RPC 仍经
  to_thread,轮询改 asyncio.sleep;新增 core.http_client.iter_chunks,把**流式
  读取期**的 aiohttp.ClientError 收敛成 UpstreamNetworkError(请求建立阶段早已
  有这条路径,读取阶段此前没有,断流会绕过调用方的 UpstreamError 分类)。
  删除 get_http_session / _SESSION_LOCAL。
  models._RetryFromScratch 不再继承 requests.RequestException 而继承 Exception:
  它是「续传起点失效、残片已清、从头重下」,既不是确定性失败
  (DownloadException),也不是残片已保留的网络失败(UpstreamError),必须独立
  成类且在 except 链中先于二者捕获。
- music_api:podcast.py 全部函数协程化(直链解析内部改
  await NeteaseAPI.get_song_url);compat.py 删除——16 个死掉的同步门面早已
  清空,文件只剩空壳。
- services:_taskcore / _zipcore / batch / downloads / podcast / tasks 协程化。
  threading.Thread → loop.create_task(任务句柄字段 worker_thread →
  worker_task),ThreadPoolExecutor → asyncio.gather + asyncio.Semaphore,
  终态函数与结算回调全部改 await;监控线程里「worker 已死」判定改 task.done()。
  删除 services/streaming.py(同步流垫片,唯一消费者已迁到
  core.http_client.open)。zipfile 实例仍非线程安全,故「下载」并行、
  「入包」严格串行。
- web:路由层 130+ 处 to_thread 拆除——被包住的目标自己已是协程,再包一层只是
  把协程丢进线程里跑一圈。tasks_registry.start_task 直接 create_task。
  web/app.py 关闭段去掉 async_bridge.shutdown()(不再有第二条约循环)。
  web/routers/download.py 的播客端点补上 3 处漏掉的 await
  (extract_podcast_ref / get_radio_detail)——漏掉时不报错、只静默返回协程对象,
  靠 pyright 抓出来后修正。

留在 to_thread 上的都是**正当的阻塞卸载**:同步 SQLite 落库与账本结算、
mutagen 全量重写标签、zipfile 整段拷贝、shutil.rmtree、aria2p 的同步 RPC、
PBKDF2 的 10 万次迭代。ruff 对残留的本地元数据操作(exists/stat/os.replace/
往 .part 顺序写分块)按「微秒级、卸载的调度开销大于操作本身」放行,pyproject
逐条注明理由。

取消/暂停仍由 threading.Event 承载:置位方是请求线程/监控线程,读取方是 worker
协程,只有「置位」与「轮询读取」两种操作,既不需要跨循环唤醒也不会卡住事件
循环——比 asyncio.Event 更省心(ASYNC110 同理按此豁免)。

测试:10 个文件的替身边界同步迁移——网络替身从 core.sync_http 的
get_http_session 换成 downloader/服务模块的 _http.open + **真实** iter_chunks
(顺带钉住读取期异常收敛);`_run` 改为在独立线程里 asyncio.run(worker(...));
aria2 的假时钟从 time.sleep 换成 _POLL_INTERVAL=0;monitor 的线程句柄替身换成
done() 可控的轻量替身(asyncio.Task 的构造需要运行中的事件循环,而监控循环跑在
常驻线程里);podcast / 封面 / 结算包装等替身一律改 async。

门禁:ruff 全过 / pyright 0-0 / pytest 383 项全绿(0 失败)。

下一步 B2:auth_system 12 个模块改 aiosqlite 背板、web 层剩余 to_thread 收尾;
B3:删 core/sync_http.py、core/async_bridge.py、core/http_errors.py(现仅
sync_http 自用)。
2026-09-28 19:36:56 +08:00

752 lines
33 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""下载主类:落盘、元数据写入与交付。
(自 music_downloader.py 拆出。)
异步口径:整个下载链路是协程。网络走 core.http_client(进程级连接池),
文件落盘/标签写入是阻塞 IO 与纯 Python CPU 工作,经 ``asyncio.to_thread`` 卸载;
互斥改 asyncio 锁(见 _concurrency),取消仍由 threading.Event 承载——
取消位由请求线程/监控线程置位,只有轮询读取,跨线程安全且无需跨循环唤醒。
"""
from __future__ import annotations
import asyncio
import os
import re
import threading
from pathlib import Path
from core.http_client import UpstreamError, UpstreamHTTPError, iter_chunks
from core.http_client import client as _http
from music_api import APIException, NeteaseAPI, load_cookies
from ._aria2 import Aria2Backend, get_backend
from ._concurrency import (
CountingSemaphore,
get_file_lock,
)
from ._http import (
DL_CHUNK_SIZE,
_sniff_audio_header,
)
from ._log import (
logger,
)
from .cover import (
fetch_cover_bytes,
)
from .models import (
API_TYPE_TO_EXT,
DownloadCanceled,
DownloadException,
DownloadResult,
MusicInfo,
_RetryFromScratch,
)
from .tagging import build_tags, write_tags
# 单文件下载体积硬上限:超限即弃,防止单文件拖垮磁盘
_MAX_DOWNLOAD_BYTES = 500 * 1024 * 1024
# ============ 核心实现 ============
class MusicDownloader:
"""网易云音乐下载器
用法:
downloader = MusicDownloader(download_dir="downloads")
result = await downloader.download_music_file(music_id, quality="lossless")
"""
def __init__(
self,
download_dir: str = "downloads",
max_concurrent: int = 3,
global_max_concurrent: int = 10,
):
"""
Args:
download_dir: 下载目录
max_concurrent: 单个批量任务内的最大并发下载数
global_max_concurrent: 整个服务同时下载的文件数(含标签写入)。
所有请求共享该闸,防止大量用户同时下载导致内存/CPU 耗尽。
"""
self.download_dir = Path(download_dir)
self.download_dir.mkdir(exist_ok=True)
self.max_concurrent = max_concurrent
self.api = NeteaseAPI()
# 全局并发闸:所有请求共用,限制同时进行的下载数
# (带计数,供前端实时展示排队情况)
self._download_semaphore = CountingSemaphore(global_max_concurrent)
# 标签写入专用闸:写标签会重写整个文件(IO 密集),
# 与下载分离限流 —— 写满时只阻塞后来者,不占下载名额、不拖慢其它文件的传输
cpu_count = os.cpu_count() or 2
self._tag_semaphore = asyncio.Semaphore(max(1, min(4, cpu_count)))
# ----------- 工具方法 -----------
def _sanitize_filename(self, filename: str, max_bytes: int = 180) -> str:
"""清理文件名非法字符。
换行/回车/制表等控制字符一律剔除(否则 send_file 头部 500);
按 UTF-8 字节截断(Linux 文件系统 255 字节上限,中文按字符截 200 会超),
且从 _build_file_path 传入时保证尾部 [ID 音质] 标记不被截掉。"""
filename = re.sub(r'[\x00-\x1f\x7f]', '', filename)
filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
filename = filename.strip(' .')
# 按字节截断(在字符边界上回退,不产生半个字符)
raw = filename.encode('utf-8')
if len(raw) > max_bytes:
cut = filename.encode('utf-8')[:max_bytes]
# 回退到合法 UTF-8 边界
while cut and (cut[-1] & 0xC0) == 0x80:
cut = cut[:-1]
filename = cut.decode('utf-8', errors='ignore').rstrip(' .')
return filename or "unknown"
def _determine_file_extension(
self,
url: str,
content_type: str = "",
api_type: str = "",
) -> str:
"""确定文件扩展名
优先级: API 返回的 type > URL 后缀 > HTTP Content-Type > 默认 mp3
"""
api_type = (api_type or "").lower()
if api_type in API_TYPE_TO_EXT:
return API_TYPE_TO_EXT[api_type]
url_lower = url.lower()
if ".flac" in url_lower:
return ".flac"
if ".mp3" in url_lower:
return ".mp3"
if ".m4a" in url_lower or ".mp4" in url_lower:
return ".m4a"
content_type = (content_type or "").lower()
if "flac" in content_type:
return ".flac"
if "mpeg" in content_type or "mp3" in content_type:
return ".mp3"
if "mp4" in content_type or "m4a" in content_type:
return ".m4a"
return ".mp3"
@staticmethod
def _unlink_quiet(path: Path) -> None:
"""安静地删除文件,忽略不存在 / 权限错误"""
try:
path.unlink()
except OSError:
pass
# ----------- 信息获取 -----------
async def get_music_info(self, music_id: int, quality: str = "standard",
cookies: dict[str, str] | None = None) -> MusicInfo:
"""获取音乐详细信息(URL/详情/歌词 3 个接口并发请求,加快单曲/批量启动)"""
try:
cookies = cookies if cookies is not None else load_cookies()
# 3 个接口并发请求,单曲下载/批量预取都能明显提速
# URL/详情拿到后不再等待失败的歌词请求,已知失败不拖着等完
url_result, detail_result, lyric_result = await asyncio.gather(
self.api.get_song_url(music_id, quality, cookies),
self.api.get_song_detail(music_id),
self.api.get_lyric(music_id, cookies),
return_exceptions=True,
)
if isinstance(url_result, BaseException):
raise url_result
if isinstance(detail_result, BaseException):
raise detail_result
if isinstance(lyric_result, BaseException):
lyric_result = None # 歌词接口抖动不阻断解析/下载
if not url_result.get("data") or not url_result["data"]:
raise DownloadException(f"无法获取音乐ID {music_id} 的播放链接")
# 上游 data 数组元素可能为 null(同族)
song_data = url_result["data"][0] or {}
download_url = song_data.get("url", "") or ""
if not download_url:
raise DownloadException(f"音乐ID {music_id} 无可用的下载链接")
if not detail_result.get("songs") or not detail_result["songs"]:
raise DownloadException(f"无法获取音乐ID {music_id} 的详细信息")
song_detail = detail_result["songs"][0]
# 上游可选字段可能为 null(而非缺失),一律 (x or {}) / (x or []) 容错,
# 可选展示信息异常不再让整首有效歌曲解析失败
lyric = ((lyric_result or {}).get("lrc") or {}).get("lyric", "") or ""
tlyric = ((lyric_result or {}).get("tlyric") or {}).get("lyric", "") or ""
artists = "/".join(filter(None, ((a or {}).get("name") for a in (song_detail.get("ar") or []))))
al = song_detail.get("al") or {}
publish_time = al.get("publishTime", 0) or 0
publish_year = None
if publish_time:
import datetime
try:
publish_year = datetime.datetime.fromtimestamp(publish_time / 1000).year
except (OSError, ValueError, OverflowError):
publish_year = None
return MusicInfo(
id=music_id,
name=song_detail.get("name") or "未知歌曲",
artists=artists or "未知艺术家",
album=al.get("name") or "未知专辑",
pic_url=al.get("picUrl") or "",
duration=(song_detail.get("dt") or 0) // 1000,
track_number=song_detail.get("no", 0) or 0,
download_url=download_url,
file_type=(song_data.get("type") or "mp3").lower(), # null 容错
file_size=song_data.get("size", 0) or 0,
quality=quality,
lyric=lyric,
tlyric=tlyric,
publish_year=publish_year,
)
except APIException as e:
raise DownloadException(f"API调用失败: {e}") from e
except Exception as e:
raise DownloadException(f"获取音乐信息时发生错误: {e}") from e
# ----------- 文件路径 -----------
def _build_file_path(self, music_info: MusicInfo) -> Path:
"""根据 MusicInfo 生成下载文件路径
文件名附带歌曲 ID 与请求音质:不同歌曲同名(不同版本/重复收录)落盘互不冲突;
同一首歌不同音质也各自独立,避免低音质文件被当作高音质交付(调包)。
交付文件名由调用方另行决定,不受影响。
先截断主名(按字节),再拼接 [ID 音质] 标记——标记永不丢失,
杜绝截断后两首不同 ID 的歌落到同一路径。
"""
base = f"{music_info.artists} - {music_info.name}"
tag = f" [{music_info.id} {music_info.quality}]"
ext = self._determine_file_extension(
music_info.download_url,
api_type=music_info.file_type,
)
# 按完整路径反算文件名可用长度——旧口径只按 Linux 255 字节
# 截断文件名,未启用 LongPathsEnabled 的 Windows 上超长歌名会在 open 时
# OSError 白白重试 3 轮。预算 = 240 - 目录前缀 - 扩展名 - ".part"等余量;
# 中文名 UTF-8 字节数 ≥ 字符数,按字节截断天然保守。
# 目录过深时按比例压缩 tag 预算。注意下限保底仍可能超过 260 字节,
# 该病理场景需由部署侧控制 downloads 目录深度来规避。
budget = 240 - len(str(self.download_dir)) - len(ext) - 8
if budget < 124:
tag_budget = max(24, budget // 3)
else:
tag_budget = 60
safe_base = self._sanitize_filename(base, max_bytes=max(32, budget - tag_budget))
safe_tag = self._sanitize_filename(tag, max_bytes=tag_budget)
return self.download_dir / f"{safe_base}{safe_tag}{ext}"
# ----------- 下载 -----------
async def download_music_file(
self,
music_id: int,
quality: str = "standard",
write_meta: bool = True,
meta_fields: list[str] | None = None,
cookies: dict[str, str] | None = None,
retries: int = 2,
cancel_event: threading.Event | None = None,
music_info: MusicInfo | None = None,
progress_cb=None,
) -> DownloadResult:
"""下载音乐到本地
Args:
music_id: 音乐ID
quality: 音质
write_meta: 是否写入元数据
meta_fields: 写入字段列表,None 表示全部
cookies: 用于获取直链/歌词的 Cookie,None 表示全局账号
retries: 网络/写入失败时的重试次数(默认 2 次)
cancel_event: 取消事件,置位后立即中止下载(抛 DownloadCanceled)
music_info: 已预取的歌曲信息(跳过重复 API 调用,用于批量加速)
progress_cb: 可选进度回调 progress_cb(written_bytes, total_bytes)。
每写一个分块调用一次(含续传起点);回调内部的异常会被吞掉,
绝不影响下载主流程。调用方自行限频。
说明:
- 流式下载到 .part 临时文件,完整下载后再原子改名,半截文件不会被当作已下载
- 断点续传:失败保留 .part,重试/下次下载从已写字节续传(服务器支持 Range 时)
- per-file 锁:同名歌曲并发下载互斥,防止 .part 被交叉写入损坏
- 连接复用:core.http_client 的进程级连接池 + keep-alive
- 网络传输与标签写入分离限流,互不抢占对方资源
本函数只保留编排与重试策略:缓存校验、续传起点、传输通道、完整性判定、
元数据限流各自见对应私有方法的 docstring(此前是 205 行 / 圈复杂度 47
的单体,任何一处改动都要通读全部步骤)。
"""
if music_info is None:
music_info = await self.get_music_info(music_id, quality, cookies)
file_path = self._build_file_path(music_info)
part_path0 = file_path.with_name(file_path.name + ".part")
last_error = ""
next_wait = 0.0
for attempt in range(retries + 1):
# 重试退避等待在文件锁外执行——同名并发任务不再陪睡
if next_wait > 0:
await asyncio.sleep(next_wait)
next_wait = 0.0
# 重试轮次顶部的取消检查也要清理 .part(原实现在 try 外抛出会绕过清理)
if cancel_event is not None and cancel_event.is_set():
self._unlink_quiet(part_path0)
raise DownloadCanceled("下载已取消")
# part_path 是 file_path 的 .part 别名。必须在进入下载 try 之前绑定:
# except 分支(取消/网络错误)要清理 .part,若此处未赋值,异常处理器
# 会以 NameError 二次崩溃、掩盖原始异常
part_path = part_path0
# 同一目标文件的下载者串行化(不同文件之间仍然并行)
async with get_file_lock(file_path):
try:
# 其它任务可能刚好完成此文件,锁内复查(命中即返回,不再下载)
cached = await self._cached_result(file_path, music_info,
write_meta, meta_fields)
if cached is not None:
return cached
# 上次已下完但未改名的 .part:直接转正,不发起网络请求
promoted = await self._promote_complete_part(
part_path, file_path, music_info, write_meta, meta_fields)
if promoted is not None:
return promoted
# part_path 已在文件锁外绑定为 part_path0(同一路径)
await self._transfer(music_info, part_path,
self._resume_offset(part_path, music_info),
cancel_event, progress_cb)
# 完整性校验(已知期望大小时必须精确相等;未知期望时至少非空)
mismatch = self._size_mismatch_error(part_path, music_info)
if mismatch:
# 保留 .part 以便断点续传;下次续传从现有字节继续
last_error = mismatch
next_wait = self._backoff_wait(attempt, retries)
continue
# 完整性通过 → 嗅探 → 原子转正 → 写元数据
return await self._publish(part_path, file_path, music_info,
cancel_event, write_meta, meta_fields)
except DownloadCanceled:
# 用户明确放弃:清理半截临时文件
self._unlink_quiet(part_path)
raise
except DownloadException:
# 确定性失败(如体积超上限)不重试——原落泛型
# handler 会删残片后从 0 重拉,3 轮白拉 ~1.5GB
raise
except _RetryFromScratch:
# .part 已清理,直接进入下一轮从头下载
last_error = "续传起点失效,重新下载"
next_wait = self._backoff_wait(attempt, retries)
continue
except (UpstreamError, OSError) as e:
# 网络错误:保留 .part 支持断点续传;
# 但若任务已被取消,.part 没有任何续传价值,一并清理
if cancel_event is not None and cancel_event.is_set():
self._unlink_quiet(part_path)
raise DownloadCanceled("下载已取消") from e
last_error = f"下载请求失败: {e}"
next_wait = self._backoff_wait(attempt, retries)
continue
except Exception as e:
last_error = f"下载过程中发生错误: {e}"
next_wait = self._backoff_wait(attempt, retries)
continue
# 最后一次尝试仍失败:若任务已取消按取消收尾
if cancel_event is not None and cancel_event.is_set():
self._unlink_quiet(part_path0)
raise DownloadCanceled("下载已取消")
return DownloadResult(success=False, error_message=last_error or "下载失败")
# ----------- 下载分步实现(download_music_file 的组成件) -----------
async def _cached_result(
self,
file_path: Path,
music_info: MusicInfo,
write_meta: bool,
meta_fields: list[str] | None,
) -> DownloadResult | None:
"""锁内复查目标文件是否已是一份可用缓存;返回 None 表示需要真正下载。
缓存与本次期望不符时视为 miss 并删除重下,而不是把旧文件当本次结果交付
(跨账号音质降级 / 上游变更都会让同 ID 同音质的实际文件不同;原实现
命中即返回、从不校验尺寸)。期望未知(file_size=0)时至少拒绝 0 字节坏文件。
"""
if not file_path.exists():
return None
st_size = file_path.stat().st_size
cache_bad = ((music_info.file_size and st_size != music_info.file_size)
or (not music_info.file_size and st_size == 0))
if cache_bad:
self._unlink_quiet(file_path)
return None
# 缓存命中也尊重本次的元数据请求(重跑一次写标签,幂等)
if write_meta:
await self._write_metadata_locked(file_path, music_info, meta_fields)
return DownloadResult(
success=True,
file_path=str(file_path),
file_size=file_path.stat().st_size,
music_info=music_info,
)
async def _promote_complete_part(
self,
part_path: Path,
file_path: Path,
music_info: MusicInfo,
write_meta: bool,
meta_fields: list[str] | None,
) -> DownloadResult | None:
"""上次已下完但未改名的 .part:直接转正交付;返回 None 表示仍需下载。
尺寸要求**精确相等**——放宽为 >= 会把超尺寸残片(上游多发货/坏分片)
转正成坏文件;期望未知时不走此路(交给下载流程)。转正后立即返回,
不再发起网络请求,否则离线时会把现成的完整文件报为失败。
"""
if not part_path.exists():
return None
if not (music_info.file_size > 0
and part_path.stat().st_size == music_info.file_size):
return None
os.replace(part_path, file_path)
if write_meta:
await self._write_metadata_locked(file_path, music_info, meta_fields)
return DownloadResult(
success=True,
file_path=str(file_path),
file_size=file_path.stat().st_size,
music_info=music_info,
)
def _resume_offset(self, part_path: Path, music_info: MusicInfo) -> int:
"""断点续传起点;返回 0 表示从头下载(顺带清理没有续传价值的残片)。"""
if not part_path.exists():
return 0
existing = part_path.stat().st_size
if existing <= 0:
self._unlink_quiet(part_path) # 0 字节残片无续传价值
return 0
if music_info.file_size > 0 and existing > music_info.file_size:
# 超尺寸残片(上游多发货/坏分片)丢弃从头下——留着会被
# 「尺寸相等即转正」误交付
self._unlink_quiet(part_path)
return 0
return existing
async def _transfer(
self,
music_info: MusicInfo,
part_path: Path,
resume_from: int,
cancel_event: threading.Event | None,
progress_cb,
) -> None:
"""占全局下载名额后落盘(产物恒为 part_path),异常语义由两条通道对齐。
aria2c 可用时优先(多连接 + 原生断点续传),否则走内置 aiohttp 流式通道。
"""
# 排队用可取消等待:原阻塞式 with 使取消后仍要等全局 10 名额腾出,
# 任务卡死、退款延迟
if not await self._download_semaphore.acquire_cancellable(cancel_event, poll=0.5):
raise DownloadCanceled("下载已取消")
try:
# get_backend 首次调用要拉起 aria2c 并等 RPC 就绪(最长 8s),
# 经 to_thread 卸载,不在事件循环上原地等
aria2_backend = await asyncio.to_thread(get_backend)
if aria2_backend is not None:
# aria2c 通道:多连接 + 原生断点续传
await self._fetch_via_aria2(aria2_backend, music_info, part_path,
cancel_event, progress_cb)
else:
# 内置通道:aiohttp 流式落盘 + Range 起点校验
await self._fetch_via_http(music_info, part_path, resume_from,
cancel_event, progress_cb)
finally:
# 与 acquire_cancellable 配对释放(原 with 语义)
self._download_semaphore.release()
@staticmethod
def _size_mismatch_error(part_path: Path, music_info: MusicInfo) -> str:
"""完整性判定:已知期望大小时精确相等,未知期望时至少非空。
通过返回空串;否则返回失败原因(.part 由调用方保留以供下轮续传)。
"""
actual_size = part_path.stat().st_size
if music_info.file_size > 0:
complete = (actual_size == music_info.file_size)
else:
complete = (actual_size > 0)
if complete:
return ''
return (f"下载不完整: 期望 {music_info.file_size or '未知'} 字节, "
f"实际 {actual_size} 字节")
async def _publish(
self,
part_path: Path,
file_path: Path,
music_info: MusicInfo,
cancel_event: threading.Event | None,
write_meta: bool,
meta_fields: list[str] | None,
) -> DownloadResult:
"""完整性通过后的收尾:内容嗅探 → 原子转正 → 写入元数据。
嗅探只在期望尺寸未知("非空即通过"口径)时执行——该口径下 CDN 错误页/
风控 JSON 曾可被存成音频交付。转正走 os.replace,半截文件不会被当作已下载。
调用点必须在文件锁内(元数据也要求在锁内,其它请求才看到已完成标记的文件)。
"""
if music_info.file_size <= 0 and not _sniff_audio_header(part_path):
self._unlink_quiet(part_path)
raise DownloadException("下载内容校验失败(非音频数据)")
os.replace(part_path, file_path)
# 元数据写入移出下载闸(CPU 操作不占网络名额),
# 但仍在文件锁内执行,保证其它请求看到的是已写入标签的最终文件
if write_meta:
# 元数据段入口补取消检查点——已取消的任务不再
# 进入标签闸排队/写盘等待(有界但可达数十秒),收尾不再延迟
if cancel_event is not None and cancel_event.is_set():
raise DownloadCanceled("下载已取消")
await self._write_metadata_locked(file_path, music_info, meta_fields)
return DownloadResult(
success=True,
file_path=str(file_path),
file_size=file_path.stat().st_size,
music_info=music_info,
)
async def _write_metadata_locked(
self,
file_path: Path,
music_info: MusicInfo,
meta_fields: list[str] | None,
) -> None:
"""在标签闸内写元数据:写标签会重写整个文件(IO 密集),与下载分离
限流——写满时只阻塞后来者,不占下载名额、不拖慢其它文件的传输。"""
async with self._tag_semaphore:
await self._write_metadata(file_path, music_info, meta_fields)
@staticmethod
def _backoff_wait(attempt: int, retries: int) -> float:
"""失败后的退避秒数;已是最后一轮则不等待。锁外等待——同名并发任务不陪睡。"""
return 0.5 * (attempt + 1) if attempt < retries else 0.0
# ----------- 传输通道(二选一,产物都是 part_path) -----------
async def _fetch_via_http(
self,
music_info: MusicInfo,
part_path: Path,
resume_from: int,
cancel_event: threading.Event | None,
progress_cb,
) -> None:
"""内置流式通道:core.http_client 逐块落盘,支持 Range 断点续传。
四步:构造 Range 头 → 发请求 → 定追加/重写并校验分片起点 → 流式落盘
(含体积上限与取消)。HTTP 错误在此分类成「确定性失败」或交外层重试。
Raises:
DownloadCanceled: 取消事件置位
DownloadException: 体积超限、CDN 4xx 等确定性失败
_RetryFromScratch: 续传起点失效(残片已清,外层从头重下)
UpstreamError: 网络/5xx 等可重试失败(保留残片续传)
"""
expected_size = music_info.file_size or 0
self._safe_progress(progress_cb, resume_from, expected_size)
resp = None
try:
headers = self._range_headers(resume_from)
resp = await _http.open('GET', music_info.download_url,
headers=headers or None, timeout=60)
mode = self._resolve_write_mode(resp, resume_from, part_path)
if resp.status >= 400:
raise UpstreamHTTPError(resp.status, resp.reason or '')
await self._stream_body(resp, part_path, mode, resume_from,
expected_size, cancel_event, progress_cb)
except UpstreamHTTPError as e:
status = e.status
if status == 416:
# Range 起点无效(如 CDN 环节变化):清除旧 part,从头重新下载
self._unlink_quiet(part_path)
raise _RetryFromScratch("Range 起点失效,已从头重新下载") from e
if self._is_deterministic_client_error(status):
self._unlink_quiet(part_path)
raise DownloadException(
f"下载失败(HTTP {status}),下载链接可能已失效") from e
raise
finally:
if resp is not None:
resp.close()
@staticmethod
def _range_headers(resume_from: int) -> dict[str, str]:
"""续传时请求「起点之后的全部字节」,否则不带头。"""
if resume_from > 0:
return {"Range": f"bytes={resume_from}-"}
return {}
def _resolve_write_mode(self, resp, resume_from: int, part_path: Path) -> str:
"""落盘方式:续传且服务器确实给了 206 才追加,否则从头重写。
服务器忽略 Range 返回 200 时若继续追加,会把整段拼在残片后面;206 还须
校验分片起点(见 _verify_content_range)。
"""
if resume_from <= 0 or resp.status != 206:
return "wb"
self._verify_content_range(resp, resume_from, part_path)
return "ab"
def _verify_content_range(self, resp, resume_from: int, part_path: Path) -> None:
"""校验 206 的分片起点;无法确认起点就清残片从头下。
起点错位的分片拼起来是坏文件——代价远大于多下一次,故与「头被中间层
剥离、解析不出起点」同口径处理。
"""
content_range = (resp.headers.get("Content-Range") or "").strip()
m = re.match(r"bytes\s+(\d+)-", content_range)
if m is not None and int(m.group(1)) == resume_from:
return
self._unlink_quiet(part_path)
if m is None:
raise _RetryFromScratch("206 响应缺少可用 Content-Range,已从头重新下载")
raise _RetryFromScratch(
f"Content-Range 起点错位({m.group(1)} != {resume_from}),已从头重新下载")
async def _stream_body(
self,
resp,
part_path: Path,
mode: str,
resume_from: int,
expected_size: int,
cancel_event: threading.Event | None,
progress_cb,
) -> None:
"""逐块写入 part_path:取消检查 + 体积硬上限(500MB)+ 进度上报。
超限不能就地删除——Windows 无法删除仍被打开的文件,先置标志退出 with
再清理(原实现在打开中 unlink 被静默吞掉,超限残片留在磁盘上)。
"""
max_bytes = _MAX_DOWNLOAD_BYTES
written = resume_from if mode == "ab" else 0
over_limit = False
with open(part_path, mode) as f:
async for chunk in iter_chunks(resp, DL_CHUNK_SIZE):
if cancel_event is not None and cancel_event.is_set():
raise DownloadCanceled("下载已取消")
if not chunk:
continue
f.write(chunk)
written += len(chunk)
if written > max_bytes:
over_limit = True
break
self._safe_progress(progress_cb, written, expected_size)
if over_limit:
self._unlink_quiet(part_path) # 文件已关闭,清理可达
raise DownloadException(
f"文件超过大小上限 {max_bytes // (1024*1024)}MB,已中止")
@staticmethod
def _safe_progress(progress_cb, written: int, total: int) -> None:
"""上报一次进度;回调抛异常绝不影响下载主流程。"""
if progress_cb is None:
return
try:
progress_cb(written, total)
except Exception:
pass
@staticmethod
def _is_deterministic_client_error(status: int | None) -> bool:
"""确定性 4xx(403 防盗链 / 429 风控等):对同一失效直链重试只会加剧风控、
拖慢失败返回。408 请求超时是唯一保留可重试的 4xx。
"""
return status is not None and 400 <= status < 500 and status != 408
async def _fetch_via_aria2(
self,
backend: Aria2Backend,
music_info: MusicInfo,
part_path: Path,
cancel_event: threading.Event | None,
progress_cb,
) -> None:
"""aria2c 通道:多连接加速 + 原生断点续传。
异常语义与内置通道对齐:取消抛 DownloadCanceled、体积超限抛
DownloadException(均清残片),网络类失败抛 UpstreamNetworkError
(保留残片,由外层退避重试续传)。
"""
await backend.fetch(
music_info.download_url,
part_path,
cancel_event=cancel_event,
progress_cb=progress_cb,
max_bytes=_MAX_DOWNLOAD_BYTES,
)
# ============ 元数据写入(mutagen,纯 Python) ============
# download_music_to_memory / download_batch_async / get_download_progress
# 三个方法已删除——全服务端与验证脚本零调用(CLI 遗留),且 to_memory 不接受
# cookies 参数、真被启用会有行为差异;如需内存下载/异步批量再按当前接口重写
async def _write_metadata(
self,
file_path: Path,
music_info: MusicInfo,
fields: list[str] | None = None,
) -> bool:
"""写入音频标签与封面(mutagen);失败不影响交付,返回是否成功。
Args:
file_path: 已转正的音频文件
music_info: 歌曲信息(标题/歌手/专辑/年份/曲号/歌词/封面地址)
fields: 要写入的字段列表,None 表示全部写入
封面走共享缓存(与 ZIP 打包共用一次网络请求);各容器的落盘细节见
tagging.py——按扩展名分派到 ID3v2.3 / Vorbis comment / iTunes ilst。
写盘是阻塞 IO + 纯 Python CPU(mutagen 全量重写文件),经 to_thread 卸载。
"""
include_cover = fields is None or "cover" in fields
cover: bytes | None = None
if include_cover and music_info.pic_url:
cover = await fetch_cover_bytes(music_info.pic_url)
if not cover:
logger.warning("封面下载失败,跳过封面写入")
return await asyncio.to_thread(
write_tags, file_path, build_tags(music_info, fields), cover)