Archived
snowy-yang
183f5744d8
为什么：IDE 里成片的黄色波浪线并非风格噪声。ruff 与 pyright(basic) 都是 0 报错，但 basic 会整体漏报「变量可能未绑定」这类**可执行缺陷**；实测 standard 多报 7 处且全部为真——异常处理器引用了尚未赋值的局部变量，一旦走进 异常分支就会 NameError 二次崩溃，把原始异常彻底掩盖。 - core/db.py, core/http_client.py：3 处 @asynccontextmanager 的返回标注 AsyncIterator[X] 更正为 AsyncGenerator[X, None]（前者不含 send 语义） - core/state.py：三个模块级缓存补 TypedDict 建模。原先无标注的 dict 字面量被 推断为「所有值类型的并集」（float | None），每个读写点都退化成联合类型； 拆成 _VipCache（data 可为 None 表示未探测）与 _VipEntry（data 必定存在）后， 类型检查立刻抓出 account.py 对未探测缓存的直接解包隐患 - services/batch.py：zip_valid 提到函数头。ZIP 构建前的业务错误（参数缺失/ 无歌曲）会 early raise，而异常处理器引用了该名字 → NameError 二次崩溃 - services/podcast.py：radio_name、pct 同理补初始化 - services/downloads.py：声明返回 str 却可能给出 None，改 `or ''` 满足契约 - web/routers/download.py：id_list 提前绑定 - web/auth_state.py：解包出的 c 未被使用 - pyproject.toml：typeCheckingMode 由 basic 升到 standard 作为门禁，并移除 已被 standard 覆盖的 reportGeneralTypeIssues 门禁：ruff All checks passed · pyright(standard) 0 errors 0 warnings
385 lines
20 KiB
Python
385 lines
20 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""歌单/专辑批量打包 worker。
|
||
|
||
（自 services/runtime.py 拆出，行为与原实现一致。）
|
||
"""
|
||
|
||
import os
|
||
import time
|
||
import traceback
|
||
import zipfile
|
||
from pathlib import Path
|
||
|
||
from core.state import (
|
||
BATCH_DURATIONS,
|
||
BATCH_DURATIONS_LOCK,
|
||
DOWNLOAD_TASKS,
|
||
DOWNLOAD_TASKS_LOCK,
|
||
)
|
||
from core.utils import format_lyric as _format_lyric
|
||
from music_downloader import (
|
||
DownloadCanceled,
|
||
fetch_cover_bytes,
|
||
)
|
||
|
||
from ._shared import (
|
||
BATCH_ZIP_SEM,
|
||
_build_download_base,
|
||
)
|
||
from ._taskcore import (
|
||
_mark_task_terminal,
|
||
_settle_batch_billing,
|
||
)
|
||
from .api_service import (
|
||
api_service,
|
||
)
|
||
|
||
|
||
def _batch_zip_worker(task_id: str) -> None:
|
||
"""后台批量打包：窗口化下载 → 写 ZIP（音频免压缩），支持取消与断线暂停
|
||
|
||
- 断线暂停：设备心跳超时后，当前窗口下载完即释放 BATCH_ZIP_SEM 名额给下一位，
|
||
保存进度进入暂停等待；重新打开网站（心跳恢复）后自动回到队列继续下载
|
||
- 音频入包即删，缓解磁盘占用
|
||
"""
|
||
import tempfile as _tempfile
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
||
zip_tmp_path: str | None = None
|
||
# zip_valid 必须在函数头初始化：ZIP 构建前的业务错误（参数缺失/无歌曲等）
|
||
# 会 early raise，异常处理器若引用未绑定名会 NameError 二次崩溃 worker 线程
|
||
zip_valid = False
|
||
ok_count = 0
|
||
failed: list[str] = []
|
||
used_arcnames = set() # ZIP 内已占用文件名（同名歌曲去重，防止包内互相覆盖）
|
||
info_pool = None
|
||
info_futures = {}
|
||
try:
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
task = DOWNLOAD_TASKS[task_id]
|
||
task['status'] = 'running'
|
||
task['msg'] = '下载打包中...'
|
||
params = task['params']
|
||
cancel_ev = task['cancel']
|
||
pause_ev = task['pause']
|
||
ids = list(params['ids'])
|
||
quality = params['quality']
|
||
filename_format = params['filename_format']
|
||
write_meta = params['write_meta']
|
||
meta_fields = params['meta_fields']
|
||
lyric_format = params['lyric_format']
|
||
concurrency = params['concurrency']
|
||
cookies = params['cookies']
|
||
batch_start = time.time()
|
||
|
||
# 预取按窗口进行——原实现一次性提交全部（最多 500 首 ×3 接口）预取任务，
|
||
# 大歌单瞬间以 10 并发连发上游约 1500 次请求打风控；改为每个下载窗口只预取本窗口
|
||
# 的歌（与下载节奏对齐），_download_one 语义不变（等预取结果再下载）
|
||
from concurrent.futures import ThreadPoolExecutor as _InfoPool
|
||
info_pool = _InfoPool(max_workers=min(4, max(1, concurrency)))
|
||
|
||
def _prefetch(window_ids):
|
||
for mid in window_ids:
|
||
if mid not in info_futures:
|
||
info_futures[mid] = info_pool.submit(
|
||
api_service.downloader.get_music_info, mid, quality, cookies
|
||
)
|
||
|
||
def _download_one(mid):
|
||
"""并行下载单元：等待预取信息 → 下载 → 复用预取歌词与缓存封面（零重复请求）"""
|
||
info = info_futures[mid].result()
|
||
result = api_service.downloader.download_music_file(
|
||
mid, quality,
|
||
write_meta=write_meta,
|
||
meta_fields=meta_fields,
|
||
cookies=cookies,
|
||
cancel_event=cancel_ev,
|
||
music_info=info,
|
||
)
|
||
extra = {'lyric': '', 'cover': b''}
|
||
if result.success:
|
||
# 歌词已随 get_music_info 预取返回，无需再调一次歌词接口
|
||
extra['lyric'] = getattr(info, 'lyric', '') or ''
|
||
# 封面走共享缓存（同一专辑只拉一次网络）；取消后不再发起新请求
|
||
if not cancel_ev.is_set() and info.pic_url:
|
||
cover = fetch_cover_bytes(info.pic_url)
|
||
if cover:
|
||
extra['cover'] = cover
|
||
return result, extra
|
||
|
||
def _update_progress() -> None:
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
DOWNLOAD_TASKS[task_id]['done'] = ok_count + len(failed)
|
||
DOWNLOAD_TASKS[task_id]['failed'] = len(failed)
|
||
|
||
def _wait_if_paused() -> None:
|
||
"""暂停时等待恢复或取消（不持有 BATCH_ZIP_SEM）"""
|
||
if not (pause_ev.is_set() or task['status'] == 'paused'):
|
||
return
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
task['status'] = 'paused'
|
||
task['done'] = ok_count + len(failed)
|
||
task['msg'] = '已暂停（重新打开页面自动恢复）'
|
||
while not cancel_ev.is_set() and (pause_ev.is_set() or task['status'] == 'paused'):
|
||
time.sleep(1)
|
||
if cancel_ev.is_set():
|
||
raise DownloadCanceled("任务已取消")
|
||
pause_ev.clear()
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
task['status'] = 'running'
|
||
task['msg'] = '继续下载...'
|
||
|
||
# ZIP 构建放 downloads 卷（原实现落系统 %TEMP%——500 首无损可达 GB 级，
|
||
# 小系统盘一次大单即塞爆；单曲/播客早已指定 dir，唯批量漏了）
|
||
# F-A：临时包建在任务专属目录 tasks/<task_id>/——原放根目录会被
|
||
# 缓存清扫线程按 mtime 回收（Linux 上已打开文件也能被 unlink，Docker 部署
|
||
# 排队超 35 分钟即出现"扣费后领取 404"的悬空交付），根目录从此碰不到交付物
|
||
_task_dir_for_zip = api_service.downloads_path / 'tasks' / task_id
|
||
_task_dir_for_zip.mkdir(parents=True, exist_ok=True)
|
||
# 中央目录完整写出后（下方）才置 True，否则一律按失败全额退费
|
||
with _tempfile.NamedTemporaryFile(prefix='batch_zip_', suffix='.zip',
|
||
dir=str(_task_dir_for_zip),
|
||
delete=False) as zip_tmp:
|
||
zip_tmp_path = zip_tmp.name
|
||
# 音频本身已压缩，用 ZIP_STORED（免压缩）大幅提速；歌词等文本仍压缩
|
||
with zipfile.ZipFile(zip_tmp, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||
# 不用 with 管理下载池——取消时 with 退出会 wait=True 等在途
|
||
# future（其内部阻塞在预取 HTTP 上最长 30s），改为不等待 + 取消排队任务
|
||
pool = ThreadPoolExecutor(max_workers=concurrency)
|
||
try:
|
||
while ids:
|
||
if cancel_ev.is_set():
|
||
raise DownloadCanceled("任务已取消")
|
||
# 断线暂停：释放名额等待恢复/取消，恢复后重新进入排队
|
||
_wait_if_paused()
|
||
# 每个窗口持有 BATCH_ZIP_SEM（同一时刻全局最多 3 个批量任务在下载）
|
||
# 等待名额期间可被取消，不再无限阻塞线程与退款
|
||
if not BATCH_ZIP_SEM.acquire_cancellable(cancel_ev):
|
||
raise DownloadCanceled("任务已取消")
|
||
try:
|
||
if cancel_ev.is_set() or pause_ev.is_set() or task['status'] == 'paused':
|
||
continue # 立即释放名额，回到循环顶部处理暂停
|
||
window = ids[:concurrency]
|
||
ids = ids[concurrency:]
|
||
_prefetch(window) # 只预取本窗口
|
||
future_map = {
|
||
pool.submit(_download_one, mid): mid
|
||
for mid in window
|
||
}
|
||
for future in as_completed(future_map):
|
||
if cancel_ev.is_set():
|
||
raise DownloadCanceled("任务已取消")
|
||
mid = future_map[future]
|
||
try:
|
||
result, extra = future.result()
|
||
except DownloadCanceled:
|
||
raise
|
||
except Exception as e:
|
||
failed.append(f"{mid}({e})")
|
||
_update_progress()
|
||
continue
|
||
if not result.success:
|
||
failed.append(str(mid))
|
||
_update_progress()
|
||
continue
|
||
assert result.file_path is not None
|
||
fp = Path(result.file_path)
|
||
if not fp.exists():
|
||
# 文件可能已被同名并发任务入包删除，按失败处理而非炸掉整个任务
|
||
failed.append(str(mid))
|
||
_update_progress()
|
||
continue
|
||
mi = result.music_info
|
||
if mi is None:
|
||
# 与上面 fp.exists() 同口径：成功但缺元信息（上游详情缺失）
|
||
# 按单曲失败计，不让 mi.name 抛 AttributeError 把整批转 error
|
||
failed.append(str(mid))
|
||
_update_progress()
|
||
continue
|
||
base = _build_download_base(filename_format, {'name': mi.name, 'artist_string': mi.artists})
|
||
# 同名去重：歌单里同一首歌出现两次/同名不同版本时，
|
||
# 追加序号避免 ZIP 内重名互相覆盖（用户被扣 2 次只拿到 1 首）
|
||
# F-C：音频/歌词/封面三个 arcname 都要查重——同名同歌手但
|
||
# 音源扩展不同（flac vs mp3）时歌词/封面条目也会互相覆盖
|
||
final_name = f"{base}.{mi.file_type}"
|
||
lrc_name = f"{base}.{lyric_format}"
|
||
jpg_name = f"{base}.jpg"
|
||
k = 2
|
||
while (final_name in used_arcnames
|
||
or lrc_name in used_arcnames
|
||
or jpg_name in used_arcnames):
|
||
final_name = f"{base} ({k}).{mi.file_type}"
|
||
lrc_name = f"{base} ({k}).{lyric_format}"
|
||
jpg_name = f"{base} ({k}).jpg"
|
||
k += 1
|
||
used_arcnames.update((final_name, lrc_name, jpg_name))
|
||
# F-D：exists 与 zf.write 之间的缓存竞态（另一任务缓存 miss
|
||
# 删文件）按单曲失败计，不再让整批转 error
|
||
try:
|
||
zf.write(str(fp), arcname=final_name, compress_type=zipfile.ZIP_STORED)
|
||
except OSError as e:
|
||
failed.append(f"{mid}(缓存文件读取失败: {e})")
|
||
_update_progress()
|
||
continue
|
||
# 缓存文件是跨任务共享资源，任务不得删除；
|
||
# 磁盘回收交给监控线程的下载缓存清扫（按文件年龄）
|
||
# 歌词与封面已在并行下载线程中拉取，直接写入 ZIP（无网络等待）
|
||
lrc = extra.get('lyric', '')
|
||
if lrc:
|
||
try:
|
||
zf.writestr(lrc_name, _format_lyric(lrc, lyric_format), compress_type=zipfile.ZIP_DEFLATED)
|
||
except Exception:
|
||
pass
|
||
if extra.get('cover'):
|
||
try:
|
||
zf.writestr(jpg_name, extra['cover'], compress_type=zipfile.ZIP_STORED)
|
||
except Exception:
|
||
pass
|
||
ok_count += 1
|
||
_update_progress()
|
||
finally:
|
||
BATCH_ZIP_SEM.release()
|
||
finally:
|
||
# 取消/异常时不等待在途下载线程（其可能阻塞在预取 HTTP 上），
|
||
# 排队任务一并取消；在途线程自行收尾（.part 残片由续传/清扫兜底）
|
||
try:
|
||
pool.shutdown(wait=False, cancel_futures=True)
|
||
except TypeError:
|
||
pool.shutdown(wait=False)
|
||
# 复审 修正：zip_valid = True 必须在内层 with zipfile 退出之后
|
||
# （中央目录在 close 时才写出）——原缩进 16 在 with 内部，守卫不可达
|
||
zip_valid = True
|
||
|
||
if ok_count > 0:
|
||
with BATCH_DURATIONS_LOCK:
|
||
BATCH_DURATIONS.append(time.time() - batch_start)
|
||
|
||
# F-B：ZIP 未正常完成（磁盘满写坏中央目录等）——不发布坏包，按全额退费收敛
|
||
if not zip_valid:
|
||
if zip_tmp_path:
|
||
try:
|
||
os.remove(zip_tmp_path)
|
||
except OSError:
|
||
pass
|
||
zip_tmp_path = None
|
||
refunded = _settle_batch_billing(task_id, 0, 'error')
|
||
_mark_task_terminal(task_id, 'error',
|
||
f"ZIP 打包失败（可能磁盘空间不足），已退回 {refunded} 次" if refunded
|
||
else "ZIP 打包失败（可能磁盘空间不足），费用未扣或已退回")
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
t = DOWNLOAD_TASKS.get(task_id)
|
||
if t is not None:
|
||
t['failed'] = t.get('failed', 0) + ok_count
|
||
t['refunded'] = refunded
|
||
api_service.logger.error(f"批量ZIP构建失败未发布: {task_id} ok_count={ok_count} 退回{refunded}次")
|
||
return
|
||
|
||
# 全部失败不发布空 ZIP、不标 done —— 按失败收敛并全额退费
|
||
if ok_count == 0 and failed:
|
||
if zip_tmp_path:
|
||
try:
|
||
os.remove(zip_tmp_path)
|
||
except OSError:
|
||
pass
|
||
zip_tmp_path = None
|
||
refunded = _settle_batch_billing(task_id, 0, 'error')
|
||
_mark_task_terminal(task_id, 'error',
|
||
f"全部失败（{len(failed)} 首）" + (f"，已退回 {refunded} 次" if refunded else ''))
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
DOWNLOAD_TASKS[task_id]['failed'] = len(failed)
|
||
DOWNLOAD_TASKS[task_id]['refunded'] = refunded
|
||
api_service.logger.info(f"批量ZIP全部失败: {task_id} 失败{len(failed)} 退回{refunded}次")
|
||
return
|
||
|
||
refunded = _settle_batch_billing(task_id, ok_count)
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
t = DOWNLOAD_TASKS.get(task_id)
|
||
# 发布前锁内复查取消/失效状态——取消请求落在"检查后、拿锁前"的窗口也不得覆盖终态
|
||
if t is None or t.get('status') in ('canceled', 'error'):
|
||
raise DownloadCanceled("任务已取消")
|
||
if cancel_ev.is_set():
|
||
raise DownloadCanceled("任务已取消")
|
||
t['status'] = 'done'
|
||
t['completed_at'] = time.time()
|
||
t['done'] = ok_count + len(failed)
|
||
t['failed'] = len(failed)
|
||
t['msg'] = f"成功{ok_count} 失败{len(failed)}" + (f" 退回{refunded}次" if refunded else '')
|
||
t['refunded'] = refunded
|
||
t['result'] = {
|
||
'zip_path': zip_tmp_path,
|
||
'filename': f"批量下载_{ok_count}首.zip",
|
||
'audio_files': [], # 缓存文件不归任务所有，不进入清理清单
|
||
}
|
||
api_service.logger.info(f"批量ZIP完成: 成功{ok_count} 失败{len(failed)} 退回{refunded}次")
|
||
except DownloadCanceled:
|
||
# 部分成功后取消——已入包部分照常交付（成果守恒），未完成部分按账本退费
|
||
# F-B：ZIP 未正常完成（写坏）时不按 ok_count 交付——按全额退费收敛
|
||
delivered = ok_count if zip_valid else 0
|
||
refunded = _settle_batch_billing(task_id, delivered, 'canceled')
|
||
if delivered > 0 and zip_tmp_path:
|
||
_mark_task_terminal(task_id, 'canceled',
|
||
f"已取消（已打包 {ok_count} 首可领取）" +
|
||
(f"，退回 {refunded} 次" if refunded else ''))
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
t = DOWNLOAD_TASKS.get(task_id)
|
||
if t is not None:
|
||
t['refunded'] = refunded
|
||
t['result'] = {
|
||
'zip_path': zip_tmp_path,
|
||
'filename': f"批量下载_{ok_count}首.zip",
|
||
'audio_files': [],
|
||
}
|
||
else:
|
||
if zip_tmp_path:
|
||
try:
|
||
os.remove(zip_tmp_path)
|
||
except OSError:
|
||
pass
|
||
_mark_task_terminal(task_id, 'canceled',
|
||
'任务已取消' + (f"（已退回 {refunded} 次下载次数）" if refunded else ''))
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
t = DOWNLOAD_TASKS.get(task_id)
|
||
if t is not None:
|
||
t['refunded'] = refunded
|
||
api_service.logger.info(f"批量ZIP已取消: {task_id}（成功{ok_count} 退回{refunded}次）")
|
||
except Exception as e:
|
||
# 通用异常与取消同一套交付/结算规则：有成果则交付，按账本退未交付部分
|
||
# F-B：ZIP 未正常完成（写坏）时按全额退费，不发布坏包
|
||
# 非预期异常原文 logger 先行，用户可见文案泛化
|
||
api_service.logger.error(f"批量打包异常: {task_id} {e}\n{traceback.format_exc()}")
|
||
delivered = ok_count if zip_valid else 0
|
||
refunded = _settle_batch_billing(task_id, delivered, 'error')
|
||
if delivered > 0 and zip_tmp_path:
|
||
_mark_task_terminal(task_id, 'error',
|
||
"部分歌曲已打包完成，可领取" +
|
||
(f"，退回 {refunded} 次" if refunded else ''))
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
t = DOWNLOAD_TASKS.get(task_id)
|
||
if t is not None:
|
||
t['refunded'] = refunded
|
||
t['result'] = {
|
||
'zip_path': zip_tmp_path,
|
||
'filename': f"批量下载_{ok_count}首.zip",
|
||
'audio_files': [],
|
||
}
|
||
else:
|
||
_mark_task_terminal(task_id, 'error',
|
||
'打包失败，请稍后重试' + (f"（已退回 {refunded} 次下载次数）" if refunded else ''))
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
t = DOWNLOAD_TASKS.get(task_id)
|
||
if t is not None:
|
||
t['refunded'] = refunded
|
||
if zip_tmp_path:
|
||
try:
|
||
os.remove(zip_tmp_path)
|
||
except OSError:
|
||
pass
|
||
api_service.logger.error(f"批量ZIP任务异常: {task_id} {e}（成功{ok_count} 退回{refunded}次）\n{traceback.format_exc()}")
|
||
finally:
|
||
# 回收信息预取线程池，避免线程泄漏（未启动的预取任务一并取消）
|
||
if info_pool is not None:
|
||
try:
|
||
info_pool.shutdown(wait=False, cancel_futures=True)
|
||
except TypeError:
|
||
info_pool.shutdown(wait=False)
|