Archived
refactor: 拆分 4 个巨石模块为包,清理过时注释并删除作废文档
为什么:单文件 1100~1900 行的巨石模块让「改一处要通读全文」;同时注释里残留 大量已删台账的编号,以及「Flask/FastAPI 双栈迁移期」这类已失实的架构描述, 会误导后来者按不复存在的架构去改代码。 拆分(对外导入路径不变,行为零变更): - music_downloader.py -> music_downloader/(models/downloader/cover/_http/_log/_concurrency) - auth_system.py -> auth_system/(db/security/users/credits/ledger/mail/cards/backup...) - music_api.py -> music_api/(api/client/crypto/qr/cookies/compat/constants/errors) - services/runtime.py -> services/(api_service/_shared/_taskcore/downloads/batch/podcast/ streaming/tasks);services/runtime.py 保留为兼容门面 兼容性做法:各包用 __init__.py 以 __all__ 原样再导出全部符号,既有调用点 (auth_system.xxx / services.xxx / from music_api import Y)零改动;路径常量改为经包属性 动态读取(auth_system.DB_PATH),否则测试的 monkeypatch 会静默失效并写到生产库。 修复 9 处代码损伤(上一轮注释清理把 `()` 当「空括号残留」删掉所致;用「与拆分前逐字 等价」校验逐一定位,静态检查只会报出其中 4 处): - core/config.py ).strip() or '*' - core/state.py threading.Lock() x2 - qr_login.py datetime.now() / .strip() - music_downloader/downloader.py .lower() - services/podcast.py set() / time.time() x2 清理与删除: - 删除 部署文档.md / 功能介绍.md / README_私有包警告.txt - 清理指向已删台账与文档的过时编号注释;把「迁移期 / 两栈 / 双框架 / 已迁移」等 失实描述改写为现状描述(保留 `<-> Flask /path` 映射表等仍有价值的契约来源说明) - _mkpkg.py:公开包剔除清单去掉已不存在的文档名;私有包警告改为自包含 - pyproject.toml:补 pythonpath = ["."],使裸跑 pytest 不再 import core 失败 验证:ruff 全通过;pyright 0 errors / 0 warnings(102 文件);pytest 144 passed; 真实 uvicorn 冒烟 6 条路由符合预期且生产 data/ 指纹未变;忽略注释的逐字等价性校验通过。
This commit is contained in:
1 parent
838d635bc5
commit
6288a98ecc
89 files changed
+5806
-5112
No files matched your search
+27
-3
@@ -1,6 +1,30 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""服务层:从原 Flask 巨石中剥离的框架无关实现。
|
||||
"""服务层:从原 Flask 巨石中剥离的框架无关实现(不含任何 Flask 依赖)。
|
||||
|
||||
包含 MusicAPIService、下载/打包 worker、任务登记与监控、下载缓存清扫等。
|
||||
不含任何 Flask 依赖(路由层已全部迁至 web/routers)。
|
||||
模块划分(原先集中在 runtime.py 的 1900 行,现按职责拆分;`services.runtime`
|
||||
保留为兼容门面,原样再导出下列全部符号,既有调用点零改动):
|
||||
|
||||
- ``api_service``:MusicAPIService 与进程级实例(config / api_service)
|
||||
- ``_shared``:限流桶、请求统计、命名与共享常量
|
||||
- ``_taskcore``:任务生命周期原语(启动 / 终态 / 文件收尾 / 结算)
|
||||
- ``downloads``:单曲下载、账号探测与任务交付
|
||||
- ``batch``:歌单/专辑批量打包 worker
|
||||
- ``podcast``:播客单集下载与整台打包 worker
|
||||
- ``streaming``:同步流式响应垫片(音频分段透传)
|
||||
- ``tasks``:任务监控循环、设备心跳与缓存清扫
|
||||
"""
|
||||
|
||||
# ============ 依赖自检 ============
|
||||
# 应用依赖缺失时给出可读提示并非零退出。本层常以 NSSM 服务方式运行、控制台重定向为
|
||||
# GBK,裸 traceback 反而更难定位,故提示串保持纯 ASCII。放在包初始化里是因为它必然
|
||||
# 先于任何子模块执行——子模块各自 import 失败会先抛 traceback,那段提示就没机会出现。
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
for _dep in ('auth_system', 'podcast_api', 'music_api', 'music_downloader', 'qr_login'):
|
||||
try:
|
||||
importlib.import_module(_dep)
|
||||
except ImportError as _exc:
|
||||
print("ImportError: " + str(_exc))
|
||||
print("Please make sure all dependency modules exist and are importable.")
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,169 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""服务层共享状态与低层工具(限流桶、请求统计、命名与常量)。
|
||||
|
||||
(自 services/runtime.py 拆出,行为与原实现一致。)
|
||||
"""
|
||||
|
||||
import collections
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from core.config import (
|
||||
BATCH_ZIP_CONCURRENCY,
|
||||
)
|
||||
from core.state import RATE_BUCKETS as _RATE_BUCKETS
|
||||
from core.state import RATE_LOCK as _RATE_LOCK
|
||||
from core.state import (
|
||||
REQ_LOG,
|
||||
)
|
||||
from music_downloader import (
|
||||
CountingSemaphore,
|
||||
)
|
||||
|
||||
|
||||
def _rate_ok(key: str, limit: int, window: int) -> bool:
|
||||
"""滑动窗口限流:True=放行,False=超限"""
|
||||
now = time.time()
|
||||
with _RATE_LOCK:
|
||||
dq = _RATE_BUCKETS.get(key)
|
||||
if dq is None:
|
||||
# 桶数量防膨胀:超限时清理不活跃桶
|
||||
if len(_RATE_BUCKETS) > 8000:
|
||||
for k in [k for k, v in _RATE_BUCKETS.items() if not v or now - v[-1] > window]:
|
||||
_RATE_BUCKETS.pop(k, None)
|
||||
dq = collections.deque()
|
||||
_RATE_BUCKETS[key] = dq
|
||||
while dq and now - dq[0] > window:
|
||||
dq.popleft()
|
||||
if len(dq) >= limit:
|
||||
return False
|
||||
dq.append(now)
|
||||
return True
|
||||
|
||||
|
||||
# 旧版管理员口令已废除(2026-09-15):管理员权限仅认站点管理员会话(admin 登录),
|
||||
# 见 _verify_admin_password。管理后台(/admin/site/*)与旧版二维码接口(/admin/qr/*)
|
||||
# 均通过该会话鉴权。
|
||||
|
||||
# 全局批量打包并发信号量:限制同时构建的批量 ZIP 数量。
|
||||
# 大量用户同时批量下载时,超出的请求排队等待,避免磁盘/内存/CPU 被同时占满。
|
||||
# (下载本身的并发已由下载器内的全局信号量控制)
|
||||
# 使用带计数的信号量,供前端实时展示排队人数与预计等待时间。
|
||||
# 注:信号量依赖下载器的 CountingSemaphore,属服务层对象,随下载服务搬迁,
|
||||
# 暂留 main(并发上限 3 见 core.config.BATCH_ZIP_CONCURRENCY)。
|
||||
BATCH_ZIP_SEM = CountingSemaphore(BATCH_ZIP_CONCURRENCY)
|
||||
|
||||
|
||||
_MONITOR_ERR_TS = {'ts': 0.0, 'n': 0} # 监控线程异常限频记录(持续异常不静默也不刷屏)
|
||||
|
||||
# 监控线程周期兜底的三个节流字典——补丁时定义丢失
|
||||
# (只剩使用),NameError 被外层 except 吞成 WARNING,孤儿对账/账本清理/tasks
|
||||
# 补扫自那以后从未执行(music_api.log 实证 2064 条 WARNING)。红转绿:重启后
|
||||
# 日志零 NameError 且对账真实运行。
|
||||
_LEDGER_LAST_RECOVER = {'ts': 0.0} # 上次孤儿账本对账时间(每 60s 一次)
|
||||
|
||||
_LEDGER_LAST_CLEANUP = {'ts': 0.0} # 上次 settled 账本 90 天清理时间(每 24h 一次)
|
||||
|
||||
_TASKS_SWEEP_LAST = {'ts': 0.0} # 上次 tasks/ 孤儿目录补扫时间(每小时一次)
|
||||
|
||||
|
||||
def _sweep_rate_buckets() -> None:
|
||||
"""限流桶周期清扫(O-5):把超过 10 分钟无新请求的桶移出,防止低流量长期运行下
|
||||
缓慢累积(原实现只在"新建第 8001 个桶"时才触发清理,时机太晚)"""
|
||||
now = time.time()
|
||||
with _RATE_LOCK:
|
||||
stale = [k for k, v in _RATE_BUCKETS.items() if not v or now - v[-1] > 600]
|
||||
for k in stale:
|
||||
_RATE_BUCKETS.pop(k, None)
|
||||
|
||||
|
||||
# _safe_int / _safe_console_print 已迁出至 core.utils(此处 import 同名共享)
|
||||
|
||||
|
||||
# ============ 接口请求统计 ============
|
||||
# REQUEST_STATS / _STATS_LOCK / REQ_LOG 已迁出至 core.state(此处 import 同名共享)
|
||||
|
||||
# 只统计真实业务接口(排除首页/健康检查/统计/账号探测等内部调用)
|
||||
# 兼容别名(/Search、/Song_V1、/Playlist、/Album、/Download)一并纳入统计与限流,
|
||||
# 防止用大写别名绕过业务限流
|
||||
COUNTED_PATHS = {
|
||||
'/song', '/search', '/playlist', '/album', '/toplist', '/artist_songs',
|
||||
'/comments', '/download', '/download/batch_zip',
|
||||
'/podcast/parse', '/podcast/programs', '/podcast/search', '/podcast/hot',
|
||||
'/podcast/download', '/podcast/download/batch_zip',
|
||||
'/user/playlists', '/user/playlist', '/user/status', '/user/logout',
|
||||
'/user/qr/create', '/user/qr/check',
|
||||
# 账号探测与个人操作端点此前漏在限流名单外——
|
||||
# /account/vip 每请求最多 9 次上游探测、/user/comment 与 /user/collect 携
|
||||
# 任意 cookie 触发上游 weapi、/auth/me 可匿名无限刷,全部纳入免费 30/分桶
|
||||
'/account/vip', '/user/collect', '/user/comment', '/auth/me',
|
||||
'/admin/qr/create', '/admin/qr/check', '/admin/cookie/status',
|
||||
'/admin/cookie/delete', '/admin/logs',
|
||||
'/admin/site/play_key/get', '/admin/site/play_key/rotate',
|
||||
'/global/playlists', '/global/playlist', '/api/playlist/play',
|
||||
'/song_v1',
|
||||
}
|
||||
|
||||
_COUNTED_LOWER = {p.lower() for p in COUNTED_PATHS}
|
||||
|
||||
|
||||
ENDPOINT_NAMES = {
|
||||
'/song': '歌曲信息', '/search': '搜索', '/playlist': '歌单详情',
|
||||
'/album': '专辑详情', '/toplist': '榜单列表', '/artist_songs': '歌手歌曲',
|
||||
'/comments': '评论', '/download': '音乐下载', '/download/batch_zip': '批量ZIP',
|
||||
'/health': '健康检查', '/api/info': '服务信息', '/api/topen': '请求统计',
|
||||
'/account/vip': '账号VIP信息',
|
||||
'/podcast/parse': '播客解析', '/podcast/programs': '播客单集列表',
|
||||
'/podcast/search': '播客搜索', '/podcast/hot': '热门播客',
|
||||
'/podcast/download': '播客单集下载', '/podcast/download/batch_zip': '播客整台打包',
|
||||
'/user/playlists': '个人歌单', '/user/playlist': '个人歌单详情',
|
||||
'/user/status': '个人登录状态', '/user/logout': '个人退出',
|
||||
'/user/qr/create': '个人二维码', '/user/qr/check': '个人登录轮询',
|
||||
'/admin/qr/create': '管理员二维码', '/admin/qr/check': '管理员登录轮询',
|
||||
'/admin/cookie/status': 'Cookie状态', '/admin/cookie/delete': 'Cookie管理',
|
||||
'/admin/logs': '操作记录',
|
||||
'/global/playlists': '全局账号歌单', '/global/playlist': '全局歌单详情',
|
||||
'/api/playlist/play': '公开歌单播放API',
|
||||
}
|
||||
|
||||
|
||||
# 用户解析/下载操作记录(IP)——REQ_LOG 已迁出至 core.state(此处 import 同名共享)
|
||||
|
||||
|
||||
def _record_log(action: str, ip: str, song: str) -> None:
|
||||
"""记录用户操作(解析/下载)。歌名截断(来自上游/用户输入,不设上限会撑内存与后台页)"""
|
||||
REQ_LOG.append({
|
||||
'action': action,
|
||||
'ip': ip,
|
||||
'song': str(song or '')[:100],
|
||||
'time': time.strftime('%Y-%m-%d %H:%M:%S'),
|
||||
})
|
||||
if len(REQ_LOG) > 2000:
|
||||
del REQ_LOG[:len(REQ_LOG) - 2000]
|
||||
|
||||
|
||||
def _build_download_base(filename_format: str, music_info: dict[str, Any]) -> str:
|
||||
"""按文件名格式生成下载基础名(不含扩展名)"""
|
||||
name = music_info.get('name', '未知歌曲')
|
||||
artist = music_info.get('artist_string', '') or '未知歌手'
|
||||
|
||||
def _safe(s: str) -> str:
|
||||
# 交付文件名必须剔除控制字符——歌名含 \r\n 时交付头的
|
||||
# Content-Disposition 会抛 ValueError,任务完成已扣费但领取必 500
|
||||
# (当年只修了下载器磁盘名这条同型路径,交付名漏网)
|
||||
s = re.sub(r'[\x00-\x1f\x7f]', '', s)
|
||||
return ''.join(c for c in s if c not in r'<>:"/\|?*').strip() or 'unknown'
|
||||
|
||||
if filename_format == 'song':
|
||||
return _safe(name)
|
||||
if filename_format == 'song_artist':
|
||||
return _safe(f"{name} - {artist}")
|
||||
return _safe(f"{artist} - {name}") # artist_song(默认)
|
||||
|
||||
|
||||
# 高频轮询/流式端点:不写逐请求日志(3s 排队轮询、1.5s 进度轮询、15s 心跳、
|
||||
# Range 分段流、错误上报都会在忙碌站点把 5MB×3 的轮转日志几小时刷一圈,排障信息被冲掉)。
|
||||
# 用户主动动作(任务列表/取消)保留日志。
|
||||
_QUIET_LOG_PATHS = {'/api/queue', '/api/batch_progress', '/api/task/heartbeat',
|
||||
'/audio_stream', '/api/fe_error'}
|
||||
@@ -0,0 +1,112 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""任务生命周期原语(启动 / 终态 / 文件收尾 / 结算)。
|
||||
|
||||
(自 services/runtime.py 拆出,行为与原实现一致。)
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from typing import Any
|
||||
|
||||
import auth_system
|
||||
from core.state import (
|
||||
DOWNLOAD_TASKS,
|
||||
DOWNLOAD_TASKS_LOCK,
|
||||
)
|
||||
|
||||
from .api_service import (
|
||||
api_service,
|
||||
)
|
||||
|
||||
|
||||
def _start_task_thread(task_id: str, target, args=()) -> str | None:
|
||||
"""启动任务 worker(启动失败时收敛任务终态并按账本退费,不留僵尸 queued 记录)。
|
||||
退款只走账本(本处 settle 后装饰器看到账已关,不会重复退)。
|
||||
线程句柄存入任务字典——监控循环对 4h 超时任务若发现 worker 线程已死
|
||||
(无法自行走到终态结算),可代为按账本结算退款(settle 幂等,无双退)。"""
|
||||
try:
|
||||
th = threading.Thread(target=target, args=args, daemon=True)
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['worker_thread'] = th
|
||||
th.start()
|
||||
return None
|
||||
except Exception as e:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.pop(task_id, None)
|
||||
if t:
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 0, 'submit_failed')
|
||||
except Exception:
|
||||
pass
|
||||
api_service.logger.error(f"任务线程启动失败: {task_id} {e}")
|
||||
return f'任务启动失败:{e}'
|
||||
|
||||
|
||||
def _mark_task_terminal(task_id: str, status: str, msg: str = '') -> None:
|
||||
"""任务进入终态的统一入口(记录 completed_at,保留期从完成时刻起算,
|
||||
而不是从提交时刻起算)。
|
||||
/F-E:置 error/canceled 终态时若 worker 线程已死(无人再走正常结算),
|
||||
代为按账本结算退款——ledger_settle 幂等,worker 已自行结算过则此处为 0。
|
||||
终态先到先得——已终态(done/error/canceled)的任务
|
||||
不再被覆盖(堵住 canceled→error 的非法迁移边:取消后解析阶段的自身失败
|
||||
不得把"已取消"改写成"失败",completed_at 也不再刷新)。"""
|
||||
need_settle = False
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t and t.get('status') in ('done', 'error', 'canceled'):
|
||||
return
|
||||
if t:
|
||||
t['status'] = status
|
||||
t['msg'] = msg or t.get('msg', '')
|
||||
t['completed_at'] = time.time()
|
||||
if status in ('canceled', 'error'):
|
||||
th = t.get('worker_thread')
|
||||
if th is not None and not th.is_alive():
|
||||
need_settle = True
|
||||
# settle 移到任务锁外——ledger_settle 内部要抢 _DB_LOCK 写库
|
||||
# (busy 最长 15s),持任务锁调用会让全站进度/提交/取消/领取随库停摆;
|
||||
# settle 幂等,锁外调用无双退
|
||||
if need_settle:
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 0, status)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _task_files(task: dict[str, Any]) -> list[str]:
|
||||
"""任务当前关联的可清理文件路径(交付 ZIP/交付副本;缓存文件不在此列)"""
|
||||
result = task.get('result') or {}
|
||||
files = []
|
||||
for f in (result.get('zip_path'), result.get('file_path')):
|
||||
if f:
|
||||
files.append(f)
|
||||
return files
|
||||
|
||||
|
||||
def _settle_batch_billing(task_id: str, ok_count: int, final_status: str = 'settled') -> int:
|
||||
"""批量任务结算(走持久账本,按实际预扣-实际交付幂等退款;
|
||||
重复调用/重启后调用都只会退一次)"""
|
||||
try:
|
||||
refund = auth_system.ledger_settle(task_id, ok_count, final_status)
|
||||
except Exception:
|
||||
# 本函数多在异常处理器内被调用——结算再抛错(库忙超时)
|
||||
# 会杀死 worker 收尾路径,跳过终态标记;失败留周期对账兜底(幂等)
|
||||
api_service.logger.error(
|
||||
f"批量结算异常(留对账兜底): {task_id}\n{traceback.format_exc()}")
|
||||
return 0
|
||||
if refund:
|
||||
try:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id) or {}
|
||||
uid = int(t.get('billing_user_id') or 0)
|
||||
u = auth_system.get_user_by_id(uid) if uid else None
|
||||
uname = u['username'] if u else 'system'
|
||||
remaining = u['duration_seconds'] if u else '?'
|
||||
auth_system.add_log('success', uname, 'system', '批量退费',
|
||||
f'失败部分 {refund} 次已退回,剩余 {remaining} 次')
|
||||
except Exception:
|
||||
pass
|
||||
return refund
|
||||
@@ -0,0 +1,181 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""服务类与进程级服务实例。
|
||||
|
||||
(自 services/runtime.py 拆出,行为与原实现一致。)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from core.config import (
|
||||
Settings,
|
||||
)
|
||||
from core.utils import expand_shortlink as _expand_shortlink
|
||||
from music_api import (
|
||||
NeteaseAPI,
|
||||
load_cookies,
|
||||
)
|
||||
from music_downloader import (
|
||||
MusicDownloader,
|
||||
)
|
||||
|
||||
# 向后兼容别名:既有代码/文档沿用 APIConfig 名称
|
||||
APIConfig = Settings
|
||||
|
||||
|
||||
class APIResponse:
|
||||
"""API响应工具类"""
|
||||
|
||||
@staticmethod
|
||||
def success(data: Any = None, message: str = 'success', status_code: int = 200) -> tuple[dict[str, Any], int]:
|
||||
"""成功响应"""
|
||||
response = {
|
||||
'status': status_code,
|
||||
'success': True,
|
||||
'message': message
|
||||
}
|
||||
if data is not None:
|
||||
response['data'] = data
|
||||
return response, status_code
|
||||
|
||||
@staticmethod
|
||||
def error(message: str, status_code: int = 400, error_code: str | None = None) -> tuple[dict[str, Any], int]:
|
||||
"""错误响应"""
|
||||
response = {
|
||||
'status': status_code,
|
||||
'success': False,
|
||||
'message': message
|
||||
}
|
||||
if error_code:
|
||||
response['error_code'] = error_code
|
||||
return response, status_code
|
||||
|
||||
|
||||
class MusicAPIService:
|
||||
"""音乐API服务类"""
|
||||
|
||||
def __init__(self, config: APIConfig):
|
||||
self.config = config
|
||||
self.logger = self._setup_logger()
|
||||
self.netease_api = NeteaseAPI()
|
||||
# 下载目录传给下载器——原实现下载器固定 downloads,
|
||||
# 服务配置指向自定义目录时音乐落盘位置与配置分离
|
||||
self.downloader = MusicDownloader(download_dir=config.downloads_dir)
|
||||
|
||||
# 创建下载目录
|
||||
self.downloads_path = Path(config.downloads_dir)
|
||||
self.downloads_path.mkdir(exist_ok=True)
|
||||
|
||||
self.logger.info(f"音乐API服务初始化完成,下载目录: {self.downloads_path.absolute()}")
|
||||
|
||||
def _setup_logger(self) -> logging.Logger:
|
||||
"""设置日志记录器"""
|
||||
logger = logging.getLogger('music_api')
|
||||
logger.setLevel(getattr(logging, self.config.log_level.upper()))
|
||||
|
||||
if not logger.handlers:
|
||||
# 控制台处理器
|
||||
console_handler = logging.StreamHandler()
|
||||
console_formatter = logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
console_handler.setFormatter(console_formatter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# 文件处理器(轮转:单文件 5MB,保留 3 份,防止长期运行无限膨胀)
|
||||
try:
|
||||
from logging.handlers import RotatingFileHandler
|
||||
file_handler = RotatingFileHandler('music_api.log', maxBytes=5 * 1024 * 1024,
|
||||
backupCount=3, encoding='utf-8')
|
||||
file_formatter = logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s'
|
||||
)
|
||||
file_handler.setFormatter(file_formatter)
|
||||
logger.addHandler(file_handler)
|
||||
except Exception as e:
|
||||
logger.warning(f"无法创建日志文件: {e}")
|
||||
|
||||
return logger
|
||||
|
||||
def _get_cookies(self) -> dict[str, str]:
|
||||
"""直接从 cookie.txt 读取并解析为 dict"""
|
||||
try:
|
||||
cookies = load_cookies()
|
||||
if not cookies:
|
||||
self.logger.warning("cookie.txt 为空或不存在,部分接口(VIP 音质)将不可用")
|
||||
return cookies
|
||||
except Exception as e:
|
||||
self.logger.error(f"读取 cookie.txt 异常: {e}")
|
||||
return {}
|
||||
|
||||
def _extract_music_id(self, id_or_url: str) -> str:
|
||||
"""提取音乐ID(正规 URL 解析替代子串/删除非数字启发式)
|
||||
- 短链仅当整串就是一个 163cn.tv 短链 URL 时才请求展开(防 SSRF)
|
||||
- music.163.com 链接用 parse_qs 取 id 参数(参数顺序无关)
|
||||
"""
|
||||
raw = str(id_or_url or '').strip()
|
||||
try:
|
||||
# 短链展开:仅接受"整个输入就是 163cn.tv 链接"(子串包含不算)
|
||||
if re.match(r'^https?://(www\.)?163cn\.tv/\S+$', raw, re.IGNORECASE):
|
||||
raw = _expand_shortlink(raw)
|
||||
|
||||
# 网易云链接:正规解析 query 取 id(不再 find('id=') 误吃 userid= 尾部)
|
||||
if 'music.163.com' in raw:
|
||||
try:
|
||||
q = parse_qs(urlsplit(raw).query)
|
||||
for key in ('id', 'songid'):
|
||||
if q.get(key):
|
||||
return str(q[key][0]).strip()
|
||||
except Exception:
|
||||
pass
|
||||
# 兜底:老格式 #/song?id=xxx 的片段部分
|
||||
m = re.search(r'[?&#]id=(\d+)', raw)
|
||||
if m:
|
||||
return m.group(1)
|
||||
|
||||
# 纯数字/普通文本直接返回
|
||||
return raw
|
||||
except Exception as e:
|
||||
self.logger.error(f"提取音乐ID失败: {e}")
|
||||
return str(id_or_url or '').strip()
|
||||
|
||||
def _format_file_size(self, size_bytes) -> str:
|
||||
"""格式化文件大小(上游 size 可能为 null/垃圾值:容错返回"大小未知"而非 500)"""
|
||||
try:
|
||||
size = float(size_bytes or 0)
|
||||
except (TypeError, ValueError):
|
||||
return "大小未知"
|
||||
if size <= 0:
|
||||
return "0B"
|
||||
|
||||
units = ["B", "KB", "MB", "GB", "TB"]
|
||||
unit_index = 0
|
||||
|
||||
while size >= 1024.0 and unit_index < len(units) - 1:
|
||||
size /= 1024.0
|
||||
unit_index += 1
|
||||
|
||||
return f"{size:.2f}{units[unit_index]}"
|
||||
|
||||
def _get_quality_display_name(self, quality: str) -> str:
|
||||
"""获取音质显示名称"""
|
||||
quality_names = {
|
||||
'standard': "标准音质",
|
||||
'exhigh': "极高音质",
|
||||
'lossless': "无损音质",
|
||||
'hires': "Hi-Res音质",
|
||||
'sky': "沉浸环绕声",
|
||||
'jyeffect': "高清环绕声",
|
||||
'jymaster': "超清母带",
|
||||
'dolby': "杜比全景声"
|
||||
}
|
||||
return quality_names.get(quality, f"未知音质({quality})")
|
||||
|
||||
|
||||
# 服务实例(框架无关)
|
||||
config = APIConfig()
|
||||
|
||||
api_service = MusicAPIService(config)
|
||||
@@ -0,0 +1,381 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""歌单/专辑批量打包 worker。
|
||||
|
||||
(自 services/runtime.py 拆出,行为与原实现一致。)
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from core.state import (
|
||||
BATCH_DURATIONS,
|
||||
BATCH_DURATIONS_LOCK,
|
||||
DOWNLOAD_TASKS,
|
||||
DOWNLOAD_TASKS_LOCK,
|
||||
)
|
||||
from core.utils import format_lyric as _format_lyric
|
||||
from music_downloader import (
|
||||
DownloadCanceled,
|
||||
fetch_cover_bytes,
|
||||
)
|
||||
|
||||
from ._shared import (
|
||||
BATCH_ZIP_SEM,
|
||||
_build_download_base,
|
||||
)
|
||||
from ._taskcore import (
|
||||
_mark_task_terminal,
|
||||
_settle_batch_billing,
|
||||
)
|
||||
from .api_service import (
|
||||
api_service,
|
||||
)
|
||||
|
||||
|
||||
def _batch_zip_worker(task_id: str) -> None:
|
||||
"""后台批量打包:窗口化下载 → 写 ZIP(音频免压缩),支持取消与断线暂停
|
||||
|
||||
- 断线暂停:设备心跳超时后,当前窗口下载完即释放 BATCH_ZIP_SEM 名额给下一位,
|
||||
保存进度进入暂停等待;重新打开网站(心跳恢复)后自动回到队列继续下载
|
||||
- 音频入包即删,缓解磁盘占用
|
||||
"""
|
||||
import tempfile as _tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
zip_tmp_path: str | None = None
|
||||
ok_count = 0
|
||||
failed: list[str] = []
|
||||
used_arcnames = set() # ZIP 内已占用文件名(同名歌曲去重,防止包内互相覆盖)
|
||||
info_pool = None
|
||||
info_futures = {}
|
||||
try:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
task = DOWNLOAD_TASKS[task_id]
|
||||
task['status'] = 'running'
|
||||
task['msg'] = '下载打包中...'
|
||||
params = task['params']
|
||||
cancel_ev = task['cancel']
|
||||
pause_ev = task['pause']
|
||||
ids = list(params['ids'])
|
||||
quality = params['quality']
|
||||
filename_format = params['filename_format']
|
||||
write_meta = params['write_meta']
|
||||
meta_fields = params['meta_fields']
|
||||
lyric_format = params['lyric_format']
|
||||
concurrency = params['concurrency']
|
||||
cookies = params['cookies']
|
||||
batch_start = time.time()
|
||||
|
||||
# 预取按窗口进行——原实现一次性提交全部(最多 500 首 ×3 接口)预取任务,
|
||||
# 大歌单瞬间以 10 并发连发上游约 1500 次请求打风控;改为每个下载窗口只预取本窗口
|
||||
# 的歌(与下载节奏对齐),_download_one 语义不变(等预取结果再下载)
|
||||
from concurrent.futures import ThreadPoolExecutor as _InfoPool
|
||||
info_pool = _InfoPool(max_workers=min(4, max(1, concurrency)))
|
||||
|
||||
def _prefetch(window_ids):
|
||||
for mid in window_ids:
|
||||
if mid not in info_futures:
|
||||
info_futures[mid] = info_pool.submit(
|
||||
api_service.downloader.get_music_info, mid, quality, cookies
|
||||
)
|
||||
|
||||
def _download_one(mid):
|
||||
"""并行下载单元:等待预取信息 → 下载 → 复用预取歌词与缓存封面(零重复请求)"""
|
||||
info = info_futures[mid].result()
|
||||
result = api_service.downloader.download_music_file(
|
||||
mid, quality,
|
||||
write_meta=write_meta,
|
||||
meta_fields=meta_fields,
|
||||
cookies=cookies,
|
||||
cancel_event=cancel_ev,
|
||||
music_info=info,
|
||||
)
|
||||
extra = {'lyric': '', 'cover': b''}
|
||||
if result.success:
|
||||
# 歌词已随 get_music_info 预取返回,无需再调一次歌词接口
|
||||
extra['lyric'] = getattr(info, 'lyric', '') or ''
|
||||
# 封面走共享缓存(同一专辑只拉一次网络);取消后不再发起新请求
|
||||
if not cancel_ev.is_set() and info.pic_url:
|
||||
cover = fetch_cover_bytes(info.pic_url)
|
||||
if cover:
|
||||
extra['cover'] = cover
|
||||
return result, extra
|
||||
|
||||
def _update_progress() -> None:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
DOWNLOAD_TASKS[task_id]['done'] = ok_count + len(failed)
|
||||
DOWNLOAD_TASKS[task_id]['failed'] = len(failed)
|
||||
|
||||
def _wait_if_paused() -> None:
|
||||
"""暂停时等待恢复或取消(不持有 BATCH_ZIP_SEM)"""
|
||||
if not (pause_ev.is_set() or task['status'] == 'paused'):
|
||||
return
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
task['status'] = 'paused'
|
||||
task['done'] = ok_count + len(failed)
|
||||
task['msg'] = '已暂停(重新打开页面自动恢复)'
|
||||
while not cancel_ev.is_set() and (pause_ev.is_set() or task['status'] == 'paused'):
|
||||
time.sleep(1)
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
pause_ev.clear()
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
task['status'] = 'running'
|
||||
task['msg'] = '继续下载...'
|
||||
|
||||
# ZIP 构建放 downloads 卷(原实现落系统 %TEMP%——500 首无损可达 GB 级,
|
||||
# 小系统盘一次大单即塞爆;单曲/播客早已指定 dir,唯批量漏了)
|
||||
# F-A:临时包建在任务专属目录 tasks/<task_id>/——原放根目录会被
|
||||
# 缓存清扫线程按 mtime 回收(Linux 上已打开文件也能被 unlink,Docker 部署
|
||||
# 排队超 35 分钟即出现"扣费后领取 404"的悬空交付),根目录从此碰不到交付物
|
||||
_task_dir_for_zip = api_service.downloads_path / 'tasks' / task_id
|
||||
_task_dir_for_zip.mkdir(parents=True, exist_ok=True)
|
||||
zip_valid = False # F-B:ZIP 中央目录完整写出才置 True,否则一律按失败全额退费
|
||||
with _tempfile.NamedTemporaryFile(prefix='batch_zip_', suffix='.zip',
|
||||
dir=str(_task_dir_for_zip),
|
||||
delete=False) as zip_tmp:
|
||||
zip_tmp_path = zip_tmp.name
|
||||
# 音频本身已压缩,用 ZIP_STORED(免压缩)大幅提速;歌词等文本仍压缩
|
||||
with zipfile.ZipFile(zip_tmp, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
# 不用 with 管理下载池——取消时 with 退出会 wait=True 等在途
|
||||
# future(其内部阻塞在预取 HTTP 上最长 30s),改为不等待 + 取消排队任务
|
||||
pool = ThreadPoolExecutor(max_workers=concurrency)
|
||||
try:
|
||||
while ids:
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
# 断线暂停:释放名额等待恢复/取消,恢复后重新进入排队
|
||||
_wait_if_paused()
|
||||
# 每个窗口持有 BATCH_ZIP_SEM(同一时刻全局最多 3 个批量任务在下载)
|
||||
# 等待名额期间可被取消,不再无限阻塞线程与退款
|
||||
if not BATCH_ZIP_SEM.acquire_cancellable(cancel_ev):
|
||||
raise DownloadCanceled("任务已取消")
|
||||
try:
|
||||
if cancel_ev.is_set() or pause_ev.is_set() or task['status'] == 'paused':
|
||||
continue # 立即释放名额,回到循环顶部处理暂停
|
||||
window = ids[:concurrency]
|
||||
ids = ids[concurrency:]
|
||||
_prefetch(window) # 只预取本窗口
|
||||
future_map = {
|
||||
pool.submit(_download_one, mid): mid
|
||||
for mid in window
|
||||
}
|
||||
for future in as_completed(future_map):
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
mid = future_map[future]
|
||||
try:
|
||||
result, extra = future.result()
|
||||
except DownloadCanceled:
|
||||
raise
|
||||
except Exception as e:
|
||||
failed.append(f"{mid}({e})")
|
||||
_update_progress()
|
||||
continue
|
||||
if not result.success:
|
||||
failed.append(str(mid))
|
||||
_update_progress()
|
||||
continue
|
||||
assert result.file_path is not None
|
||||
fp = Path(result.file_path)
|
||||
if not fp.exists():
|
||||
# 文件可能已被同名并发任务入包删除,按失败处理而非炸掉整个任务
|
||||
failed.append(str(mid))
|
||||
_update_progress()
|
||||
continue
|
||||
mi = result.music_info
|
||||
if mi is None:
|
||||
# 与上面 fp.exists() 同口径:成功但缺元信息(上游详情缺失)
|
||||
# 按单曲失败计,不让 mi.name 抛 AttributeError 把整批转 error
|
||||
failed.append(str(mid))
|
||||
_update_progress()
|
||||
continue
|
||||
base = _build_download_base(filename_format, {'name': mi.name, 'artist_string': mi.artists})
|
||||
# 同名去重:歌单里同一首歌出现两次/同名不同版本时,
|
||||
# 追加序号避免 ZIP 内重名互相覆盖(用户被扣 2 次只拿到 1 首)
|
||||
# F-C:音频/歌词/封面三个 arcname 都要查重——同名同歌手但
|
||||
# 音源扩展不同(flac vs mp3)时歌词/封面条目也会互相覆盖
|
||||
final_name = f"{base}.{mi.file_type}"
|
||||
lrc_name = f"{base}.{lyric_format}"
|
||||
jpg_name = f"{base}.jpg"
|
||||
k = 2
|
||||
while (final_name in used_arcnames
|
||||
or lrc_name in used_arcnames
|
||||
or jpg_name in used_arcnames):
|
||||
final_name = f"{base} ({k}).{mi.file_type}"
|
||||
lrc_name = f"{base} ({k}).{lyric_format}"
|
||||
jpg_name = f"{base} ({k}).jpg"
|
||||
k += 1
|
||||
used_arcnames.update((final_name, lrc_name, jpg_name))
|
||||
# F-D:exists 与 zf.write 之间的缓存竞态(另一任务缓存 miss
|
||||
# 删文件)按单曲失败计,不再让整批转 error
|
||||
try:
|
||||
zf.write(str(fp), arcname=final_name, compress_type=zipfile.ZIP_STORED)
|
||||
except OSError as e:
|
||||
failed.append(f"{mid}(缓存文件读取失败: {e})")
|
||||
_update_progress()
|
||||
continue
|
||||
# 缓存文件是跨任务共享资源,任务不得删除;
|
||||
# 磁盘回收交给监控线程的下载缓存清扫(按文件年龄)
|
||||
# 歌词与封面已在并行下载线程中拉取,直接写入 ZIP(无网络等待)
|
||||
lrc = extra.get('lyric', '')
|
||||
if lrc:
|
||||
try:
|
||||
zf.writestr(lrc_name, _format_lyric(lrc, lyric_format), compress_type=zipfile.ZIP_DEFLATED)
|
||||
except Exception:
|
||||
pass
|
||||
if extra.get('cover'):
|
||||
try:
|
||||
zf.writestr(jpg_name, extra['cover'], compress_type=zipfile.ZIP_STORED)
|
||||
except Exception:
|
||||
pass
|
||||
ok_count += 1
|
||||
_update_progress()
|
||||
finally:
|
||||
BATCH_ZIP_SEM.release()
|
||||
finally:
|
||||
# 取消/异常时不等待在途下载线程(其可能阻塞在预取 HTTP 上),
|
||||
# 排队任务一并取消;在途线程自行收尾(.part 残片由续传/清扫兜底)
|
||||
try:
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
except TypeError:
|
||||
pool.shutdown(wait=False)
|
||||
# 复审 修正:zip_valid = True 必须在内层 with zipfile 退出之后
|
||||
# (中央目录在 close 时才写出)——原缩进 16 在 with 内部,守卫不可达
|
||||
zip_valid = True
|
||||
|
||||
if ok_count > 0:
|
||||
with BATCH_DURATIONS_LOCK:
|
||||
BATCH_DURATIONS.append(time.time() - batch_start)
|
||||
|
||||
# F-B:ZIP 未正常完成(磁盘满写坏中央目录等)——不发布坏包,按全额退费收敛
|
||||
if not zip_valid:
|
||||
if zip_tmp_path:
|
||||
try:
|
||||
os.remove(zip_tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
zip_tmp_path = None
|
||||
refunded = _settle_batch_billing(task_id, 0, 'error')
|
||||
_mark_task_terminal(task_id, 'error',
|
||||
f"ZIP 打包失败(可能磁盘空间不足),已退回 {refunded} 次" if refunded
|
||||
else "ZIP 打包失败(可能磁盘空间不足),费用未扣或已退回")
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['failed'] = t.get('failed', 0) + ok_count
|
||||
t['refunded'] = refunded
|
||||
api_service.logger.error(f"批量ZIP构建失败未发布: {task_id} ok_count={ok_count} 退回{refunded}次")
|
||||
return
|
||||
|
||||
# 全部失败不发布空 ZIP、不标 done —— 按失败收敛并全额退费
|
||||
if ok_count == 0 and failed:
|
||||
if zip_tmp_path:
|
||||
try:
|
||||
os.remove(zip_tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
zip_tmp_path = None
|
||||
refunded = _settle_batch_billing(task_id, 0, 'error')
|
||||
_mark_task_terminal(task_id, 'error',
|
||||
f"全部失败({len(failed)} 首)" + (f",已退回 {refunded} 次" if refunded else ''))
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
DOWNLOAD_TASKS[task_id]['failed'] = len(failed)
|
||||
DOWNLOAD_TASKS[task_id]['refunded'] = refunded
|
||||
api_service.logger.info(f"批量ZIP全部失败: {task_id} 失败{len(failed)} 退回{refunded}次")
|
||||
return
|
||||
|
||||
refunded = _settle_batch_billing(task_id, ok_count)
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
# 发布前锁内复查取消/失效状态——取消请求落在"检查后、拿锁前"的窗口也不得覆盖终态
|
||||
if t is None or t.get('status') in ('canceled', 'error'):
|
||||
raise DownloadCanceled("任务已取消")
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
t['status'] = 'done'
|
||||
t['completed_at'] = time.time()
|
||||
t['done'] = ok_count + len(failed)
|
||||
t['failed'] = len(failed)
|
||||
t['msg'] = f"成功{ok_count} 失败{len(failed)}" + (f" 退回{refunded}次" if refunded else '')
|
||||
t['refunded'] = refunded
|
||||
t['result'] = {
|
||||
'zip_path': zip_tmp_path,
|
||||
'filename': f"批量下载_{ok_count}首.zip",
|
||||
'audio_files': [], # 缓存文件不归任务所有,不进入清理清单
|
||||
}
|
||||
api_service.logger.info(f"批量ZIP完成: 成功{ok_count} 失败{len(failed)} 退回{refunded}次")
|
||||
except DownloadCanceled:
|
||||
# 部分成功后取消——已入包部分照常交付(成果守恒),未完成部分按账本退费
|
||||
# F-B:ZIP 未正常完成(写坏)时不按 ok_count 交付——按全额退费收敛
|
||||
delivered = ok_count if zip_valid else 0
|
||||
refunded = _settle_batch_billing(task_id, delivered, 'canceled')
|
||||
if delivered > 0 and zip_tmp_path:
|
||||
_mark_task_terminal(task_id, 'canceled',
|
||||
f"已取消(已打包 {ok_count} 首可领取)" +
|
||||
(f",退回 {refunded} 次" if refunded else ''))
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['refunded'] = refunded
|
||||
t['result'] = {
|
||||
'zip_path': zip_tmp_path,
|
||||
'filename': f"批量下载_{ok_count}首.zip",
|
||||
'audio_files': [],
|
||||
}
|
||||
else:
|
||||
if zip_tmp_path:
|
||||
try:
|
||||
os.remove(zip_tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
_mark_task_terminal(task_id, 'canceled',
|
||||
'任务已取消' + (f"(已退回 {refunded} 次下载次数)" if refunded else ''))
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['refunded'] = refunded
|
||||
api_service.logger.info(f"批量ZIP已取消: {task_id}(成功{ok_count} 退回{refunded}次)")
|
||||
except Exception as e:
|
||||
# 通用异常与取消同一套交付/结算规则:有成果则交付,按账本退未交付部分
|
||||
# F-B:ZIP 未正常完成(写坏)时按全额退费,不发布坏包
|
||||
# 非预期异常原文 logger 先行,用户可见文案泛化
|
||||
api_service.logger.error(f"批量打包异常: {task_id} {e}\n{traceback.format_exc()}")
|
||||
delivered = ok_count if zip_valid else 0
|
||||
refunded = _settle_batch_billing(task_id, delivered, 'error')
|
||||
if delivered > 0 and zip_tmp_path:
|
||||
_mark_task_terminal(task_id, 'error',
|
||||
"部分歌曲已打包完成,可领取" +
|
||||
(f",退回 {refunded} 次" if refunded else ''))
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['refunded'] = refunded
|
||||
t['result'] = {
|
||||
'zip_path': zip_tmp_path,
|
||||
'filename': f"批量下载_{ok_count}首.zip",
|
||||
'audio_files': [],
|
||||
}
|
||||
else:
|
||||
_mark_task_terminal(task_id, 'error',
|
||||
'打包失败,请稍后重试' + (f"(已退回 {refunded} 次下载次数)" if refunded else ''))
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['refunded'] = refunded
|
||||
if zip_tmp_path:
|
||||
try:
|
||||
os.remove(zip_tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
api_service.logger.error(f"批量ZIP任务异常: {task_id} {e}(成功{ok_count} 退回{refunded}次)\n{traceback.format_exc()}")
|
||||
finally:
|
||||
# 回收信息预取线程池,避免线程泄漏(未启动的预取任务一并取消)
|
||||
if info_pool is not None:
|
||||
try:
|
||||
info_pool.shutdown(wait=False, cancel_futures=True)
|
||||
except TypeError:
|
||||
info_pool.shutdown(wait=False)
|
||||
@@ -0,0 +1,320 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""单曲下载、账号探测与任务交付。
|
||||
|
||||
(自 services/runtime.py 拆出,行为与原实现一致。)
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import traceback
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import auth_system
|
||||
from core.config import (
|
||||
USER_COOKIE_FILE,
|
||||
)
|
||||
from core.state import (
|
||||
DOWNLOAD_TASKS,
|
||||
DOWNLOAD_TASKS_LOCK,
|
||||
)
|
||||
from core.state import GLOBAL_MUSIC_U_CACHE as _GLOBAL_MUSIC_U_CACHE
|
||||
from core.state import VIP_PROBE_LOCK as _VIP_PROBE_LOCK
|
||||
from core.utils import format_lyric as _format_lyric
|
||||
from music_api import (
|
||||
account_info,
|
||||
detect_vip_by_probe,
|
||||
)
|
||||
from music_downloader import (
|
||||
DownloadCanceled,
|
||||
DownloadException,
|
||||
fetch_cover_bytes,
|
||||
)
|
||||
from qr_login import QRLoginClient
|
||||
|
||||
from ._shared import (
|
||||
_build_download_base,
|
||||
_record_log,
|
||||
)
|
||||
from ._taskcore import (
|
||||
_mark_task_terminal,
|
||||
)
|
||||
from .api_service import (
|
||||
api_service,
|
||||
)
|
||||
|
||||
|
||||
def _parse_cookie_str(raw) -> dict[str, str]:
|
||||
"""将 'k=v;k2=v2' 格式的 Cookie 字符串解析为字典"""
|
||||
cookies: dict[str, str] = {}
|
||||
if not raw:
|
||||
return cookies
|
||||
for part in str(raw).split(';'):
|
||||
if '=' in part:
|
||||
k, v = part.strip().split('=', 1)
|
||||
if k:
|
||||
cookies[k] = v
|
||||
return cookies
|
||||
|
||||
|
||||
# _GLOBAL_MUSIC_U_CACHE 已迁出至 core.state(此处 import 同名共享)
|
||||
|
||||
|
||||
def _global_music_u() -> str:
|
||||
"""全局账号 MUSIC_U(5 分钟缓存;后台重新扫码后自动跟随新值)"""
|
||||
now = time.time()
|
||||
if _GLOBAL_MUSIC_U_CACHE['v'] is None or now - _GLOBAL_MUSIC_U_CACHE['ts'] > 300:
|
||||
try:
|
||||
ck = api_service._get_cookies()
|
||||
_GLOBAL_MUSIC_U_CACHE['v'] = (ck.get('MUSIC_U') or '').strip()
|
||||
except Exception:
|
||||
_GLOBAL_MUSIC_U_CACHE['v'] = ''
|
||||
_GLOBAL_MUSIC_U_CACHE['ts'] = now
|
||||
return _GLOBAL_MUSIC_U_CACHE['v']
|
||||
|
||||
|
||||
def _vip_name(vip_type: int) -> str:
|
||||
"""根据 vipType 返回会员类型名称(兼容新旧取值)"""
|
||||
return {
|
||||
0: '普通用户',
|
||||
1: '音乐包',
|
||||
11: '黑胶VIP',
|
||||
100: '黑胶VIP',
|
||||
10: '黑胶SVIP',
|
||||
110: '黑胶SVIP'
|
||||
}.get(vip_type, '普通用户')
|
||||
|
||||
|
||||
def _detect_vip(cookies: dict[str, str]) -> tuple[int, str, list[str]]:
|
||||
"""检测账号会员类型与可解析音质:
|
||||
优先账号接口(vipType 精确),并始终用音质探测接口实测可解析列表。
|
||||
探测单飞锁(有界等待 0.5s)——上游黑洞时并发请求不再各自串行
|
||||
8 级探测(最坏 16 分钟线程滞留),等锁超时者直接用账号接口给降级结果"""
|
||||
if not _VIP_PROBE_LOCK.acquire(timeout=0.5):
|
||||
info = account_info(cookies)
|
||||
if info:
|
||||
return info.get('vipType', 0), 'account', []
|
||||
return 0, 'probe', []
|
||||
try:
|
||||
probe = detect_vip_by_probe(cookies)
|
||||
levels_ok = probe.get('levels_ok', [])
|
||||
info = account_info(cookies)
|
||||
if info:
|
||||
return info.get('vipType', 0), 'account', levels_ok
|
||||
return probe.get('vip_type', 0), 'probe', levels_ok
|
||||
finally:
|
||||
_VIP_PROBE_LOCK.release()
|
||||
|
||||
|
||||
# ============ 管理员设置(二维码登录 / Cookie 管理) ============
|
||||
|
||||
def _admin_client() -> QRLoginClient:
|
||||
"""创建二维码登录客户端(基于 cookie.txt)"""
|
||||
return QRLoginClient()
|
||||
|
||||
|
||||
# ============ 个人歌单(游客扫码登录自己的账号) ============
|
||||
|
||||
def _user_client() -> QRLoginClient:
|
||||
"""创建个人用户二维码登录客户端(后不再落盘 user_cookie.txt,
|
||||
USER_COOKIE_FILE 仅作 QRLoginClient 构造参数遗留)"""
|
||||
return QRLoginClient(USER_COOKIE_FILE)
|
||||
|
||||
|
||||
def _task_delivery_copy(task_id: str, src: Path, filename: str) -> str:
|
||||
"""把共享缓存文件复制/硬链接为任务专属交付文件(任务成果与共享缓存分离,
|
||||
任何任务的清理都不会再删掉别的任务的成果)"""
|
||||
task_dir = api_service.downloads_path / 'tasks' / task_id
|
||||
task_dir.mkdir(parents=True, exist_ok=True)
|
||||
dst = task_dir / filename
|
||||
if dst.exists():
|
||||
return str(dst)
|
||||
try:
|
||||
os.link(str(src), str(dst)) # 同卷硬链接:零拷贝
|
||||
except OSError:
|
||||
shutil.copy2(str(src), str(dst))
|
||||
return str(dst)
|
||||
|
||||
|
||||
# _cleanup_task_dir 死代码删除——worker 直接 rmtree、监控线程直接
|
||||
# rmtree,本函数全文件零调用(审计确认)。
|
||||
|
||||
|
||||
def _single_download_worker(task_id: str) -> None:
|
||||
"""后台单曲下载:下载音频(可选 ZIP 打包),支持取消;完成后由 /download/result 交付,35 分钟保留期清理"""
|
||||
import tempfile as _tempfile
|
||||
|
||||
file_path_str: str | None = None
|
||||
zip_tmp_path: str | None = None
|
||||
task_dir_path: str | None = None
|
||||
try:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
task = DOWNLOAD_TASKS[task_id]
|
||||
task['status'] = 'running'
|
||||
task['msg'] = '下载中...'
|
||||
params = task['params']
|
||||
cancel_ev = task['cancel']
|
||||
music_id = params['music_id']
|
||||
quality = params['quality']
|
||||
filename_format = params['filename_format']
|
||||
write_meta = params['write_meta']
|
||||
meta_fields = params['meta_fields']
|
||||
lyric_format = params['lyric_format']
|
||||
need_zip = params['need_zip']
|
||||
cookies = params['cookies']
|
||||
remote_ip = params.get('remote_ip') or '-'
|
||||
|
||||
# 一次预取歌曲完整信息(URL/详情/歌词三接口并行),
|
||||
# 替代原先 name_v1 + url_v1 + 下载器内部 get_music_info 的重复串行调用
|
||||
mi = api_service.downloader.get_music_info(music_id, quality, cookies)
|
||||
music_info = {
|
||||
'id': music_id,
|
||||
'name': mi.name,
|
||||
'artist_string': mi.artists,
|
||||
'album': mi.album,
|
||||
'pic_url': mi.pic_url,
|
||||
'file_type': mi.file_type,
|
||||
'file_size': mi.file_size,
|
||||
'duration': mi.duration * 1000,
|
||||
'download_url': mi.download_url,
|
||||
}
|
||||
base = _build_download_base(filename_format, music_info)
|
||||
filename = f"{base}.{music_info['file_type']}"
|
||||
|
||||
# 字节级进度回调(限频 0.4s):写进任务 live_percent,/api/batch_progress
|
||||
# 对运行中的单曲任务优先返回它——原实现 done 按首计数,单曲进度条全程 0%
|
||||
_cb_ts = {'t': 0.0}
|
||||
|
||||
def _progress_cb(written, total_bytes):
|
||||
try:
|
||||
now = time.time()
|
||||
if now - _cb_ts['t'] < 0.4 and written < (total_bytes or 0):
|
||||
return
|
||||
_cb_ts['t'] = now
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if not t or t.get('status') != 'running':
|
||||
return
|
||||
if total_bytes:
|
||||
pct = min(99.0, written / total_bytes * 100)
|
||||
t['live_percent'] = pct
|
||||
t['msg'] = (f"下载中 {int(pct)}% · "
|
||||
f"{api_service._format_file_size(written)}/{api_service._format_file_size(total_bytes)}")
|
||||
else:
|
||||
t['msg'] = f"下载中 · 已接收 {api_service._format_file_size(written)}"
|
||||
except Exception:
|
||||
pass # 进度上报故障绝不影响下载
|
||||
|
||||
download_result = api_service.downloader.download_music_file(
|
||||
music_id, quality,
|
||||
write_meta=write_meta,
|
||||
meta_fields=meta_fields,
|
||||
cookies=cookies,
|
||||
cancel_event=cancel_ev,
|
||||
music_info=mi,
|
||||
progress_cb=_progress_cb,
|
||||
)
|
||||
if not download_result.success:
|
||||
raise DownloadException(download_result.error_message or "下载失败")
|
||||
# 下载完成后的每一阶段都可能已被取消,发布前统一校验
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
assert download_result.file_path is not None
|
||||
cache_path = Path(download_result.file_path)
|
||||
# 交付文件用任务专属副本,共享缓存不进清理清单
|
||||
file_path_str = _task_delivery_copy(task_id, cache_path, filename)
|
||||
task_dir_path = str(Path(file_path_str).parent)
|
||||
|
||||
if need_zip:
|
||||
fd, zip_tmp_path = _tempfile.mkstemp(prefix='single_zip_', suffix='.zip',
|
||||
dir=str(Path(file_path_str).parent))
|
||||
os.close(fd)
|
||||
with zipfile.ZipFile(zip_tmp_path, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.write(file_path_str, arcname=filename)
|
||||
# 歌词已在预取时一并拿到,无需再请求一次
|
||||
if mi.lyric:
|
||||
try:
|
||||
zf.writestr(f"{base}.{lyric_format}", _format_lyric(mi.lyric, lyric_format))
|
||||
except Exception:
|
||||
pass
|
||||
# 封面走共享缓存(与元数据写入共用同一次网络请求)
|
||||
cover = fetch_cover_bytes(mi.pic_url)
|
||||
if cover:
|
||||
zf.writestr(f"{base}.jpg", cover)
|
||||
info_txt = (
|
||||
f"歌曲: {music_info['name']}\n"
|
||||
f"歌手: {music_info['artist_string']}\n"
|
||||
f"专辑: {music_info['album']}\n"
|
||||
f"音质: {api_service._get_quality_display_name(quality)}\n"
|
||||
)
|
||||
zf.writestr("info.txt", info_txt)
|
||||
# 打包阶段同样以取消检查收口
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
result = {
|
||||
'file_path': file_path_str,
|
||||
'zip_path': zip_tmp_path,
|
||||
'filename': f"{base}.zip",
|
||||
'mimetype': 'application/zip',
|
||||
'audio_files': [file_path_str],
|
||||
}
|
||||
else:
|
||||
result = {
|
||||
'file_path': file_path_str,
|
||||
'filename': filename,
|
||||
'mimetype': f"audio/{music_info['file_type']}",
|
||||
'audio_files': [file_path_str],
|
||||
}
|
||||
|
||||
# 最终发布前的取消检查(末段取消不得覆盖为 done)
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is None or t.get('status') in ('canceled', 'error'):
|
||||
raise DownloadCanceled("任务已取消")
|
||||
t['status'] = 'done'
|
||||
t['completed_at'] = time.time()
|
||||
t['done'] = 1
|
||||
t['msg'] = '下载完成'
|
||||
t['result'] = result
|
||||
# 交付已落定;结算失败不再把任务改标 error(钱由重启对账兜底,宁可商家小损不坑用户)
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 1, 'delivered')
|
||||
except Exception as _e:
|
||||
api_service.logger.error(f"单曲交付结算失败(重启对账兜底): {task_id} {_e}")
|
||||
try:
|
||||
_record_log('下载', remote_ip, music_info['name'])
|
||||
except Exception:
|
||||
pass
|
||||
api_service.logger.info(f"单曲下载完成: {filename} ({task_id})")
|
||||
except DownloadCanceled:
|
||||
_mark_task_terminal(task_id, 'canceled', '任务已取消')
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 0, 'canceled')
|
||||
except Exception:
|
||||
pass
|
||||
if task_dir_path:
|
||||
shutil.rmtree(task_dir_path, ignore_errors=True)
|
||||
api_service.logger.info(f"单曲下载已取消: {task_id}")
|
||||
except DownloadException as e:
|
||||
_mark_task_terminal(task_id, 'error', str(e))
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 0, 'error')
|
||||
except Exception:
|
||||
pass
|
||||
if task_dir_path:
|
||||
shutil.rmtree(task_dir_path, ignore_errors=True)
|
||||
api_service.logger.error(f"单曲下载失败: {task_id} {e}")
|
||||
except Exception as e:
|
||||
# 非预期异常的 repr 可能含上游主机/连接细节——用户可见文案
|
||||
# 泛化为固定文案,原文 logger 先行留档(教训:先记录再泛化)
|
||||
api_service.logger.error(f"单曲下载异常: {task_id} {e}\n{traceback.format_exc()}")
|
||||
_mark_task_terminal(task_id, 'error', '下载失败,请稍后重试')
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 0, 'error')
|
||||
except Exception:
|
||||
pass
|
||||
if task_dir_path:
|
||||
shutil.rmtree(task_dir_path, ignore_errors=True)
|
||||
@@ -0,0 +1,582 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""播客单集与整台打包 worker。
|
||||
|
||||
(自 services/runtime.py 拆出,行为与原实现一致。)
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import auth_system
|
||||
import podcast_api
|
||||
from core.state import (
|
||||
DOWNLOAD_TASKS,
|
||||
DOWNLOAD_TASKS_LOCK,
|
||||
)
|
||||
from music_api import (
|
||||
APIException,
|
||||
)
|
||||
from music_downloader import (
|
||||
DownloadCanceled,
|
||||
DownloadException,
|
||||
fetch_cover_bytes,
|
||||
)
|
||||
|
||||
from ._shared import (
|
||||
BATCH_ZIP_SEM,
|
||||
_record_log,
|
||||
)
|
||||
from ._taskcore import (
|
||||
_mark_task_terminal,
|
||||
_settle_batch_billing,
|
||||
)
|
||||
from .api_service import (
|
||||
api_service,
|
||||
)
|
||||
from .streaming import (
|
||||
_stream_open,
|
||||
)
|
||||
|
||||
|
||||
def _podcast_safe_name(s: str, fallback: str = 'unknown') -> str:
|
||||
"""文件名安全化(去除非法字符)"""
|
||||
s = ''.join(c for c in str(s or '') if c not in r'<>:"/\|?*' and ord(c) >= 32).strip()
|
||||
return s or fallback
|
||||
|
||||
|
||||
def _podcast_stream_download(url: str, target: Path, total_size: int,
|
||||
cancel_ev: threading.Event,
|
||||
task_id: str = '', progress_label: str = '') -> int:
|
||||
"""流式下载播客音频到 target(先写 .part 再改名),返回实际字节数
|
||||
|
||||
自动加入全局下载信号量排队;每 0.5s 更新一次任务进度消息。
|
||||
"""
|
||||
done = 0
|
||||
last_update = 0.0
|
||||
part = target.with_name(target.name + '.part')
|
||||
# 等待全局下载名额期间可被取消(与音乐批量 BATCH_ZIP_SEM 同语义)
|
||||
sem = api_service.downloader._download_semaphore
|
||||
if not sem.acquire_cancellable(cancel_ev):
|
||||
raise DownloadCanceled("任务已取消")
|
||||
try:
|
||||
with _stream_open(url, timeout=60) as r:
|
||||
if r.status >= 400:
|
||||
raise DownloadException(f"上游返回 {r.status}")
|
||||
total = total_size or int(r.headers.get('Content-Length') or 0)
|
||||
with open(part, 'wb') as f:
|
||||
for chunk in r.iter_content(chunk_size=256 * 1024):
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
done += len(chunk)
|
||||
now = time.time()
|
||||
if now - last_update > 0.5:
|
||||
last_update = now
|
||||
if total:
|
||||
pct = min(99, int(done / total * 100))
|
||||
msg = f"{progress_label} {pct}% · {api_service._format_file_size(done)}/{api_service._format_file_size(total)}"
|
||||
else:
|
||||
msg = f"{progress_label} 已下载 {api_service._format_file_size(done)}"
|
||||
if task_id:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t and t['status'] == 'running':
|
||||
t['msg'] = msg
|
||||
# 单集任务的字节级进度(批量任务保持按集数口径)
|
||||
if t.get('type') == 'single' and total:
|
||||
t['live_percent'] = min(99.0, float(pct))
|
||||
finally:
|
||||
sem.release()
|
||||
if part.exists():
|
||||
part.replace(target)
|
||||
# 完整性校验:CDN 提前断流时 iter_content 会"正常"结束,文件静默截断
|
||||
# (API 报告的期望大小时精确相等——95% 容差会交付截断的坏文件还扣次数;
|
||||
# 期望未知时退化为非空校验,Content-Length 兜底值可能是压缩口径不做精确比对)
|
||||
if total_size and done != total_size:
|
||||
try:
|
||||
os.remove(str(target))
|
||||
except OSError:
|
||||
pass
|
||||
raise DownloadException(f"下载不完整({done}/{total_size} 字节),请重试")
|
||||
if not total_size and done <= 0:
|
||||
try:
|
||||
os.remove(str(target))
|
||||
except OSError:
|
||||
pass
|
||||
raise DownloadException("下载内容为空,请重试")
|
||||
return done
|
||||
|
||||
|
||||
def _podcast_resolve_audio(program: dict[str, Any], quality: str,
|
||||
cookies: dict[str, str]) -> dict[str, Any]:
|
||||
"""解析单集音频直链,失败抛 DownloadException"""
|
||||
song_id = program.get('main_song_id')
|
||||
if not song_id:
|
||||
raise DownloadException("该单集缺少音频信息(mainSong)")
|
||||
url_info = podcast_api.resolve_episode_url(song_id, cookies, quality)
|
||||
if not url_info or not url_info.get('url'):
|
||||
raise DownloadException("未获取到音频直链(单集可能已下架或受版权限制)")
|
||||
return url_info
|
||||
|
||||
|
||||
def _podcast_download_worker(task_id: str) -> None:
|
||||
"""后台播客单集下载:解析单集 → 音质降级取直链 → 流式下载(可选 ZIP 打包)
|
||||
|
||||
任务 type 为 'single',完成后经 /download/result 交付,与单曲下载共用交付通道。
|
||||
落盘文件进入任务专属目录且带节目 ID,跨任务永不重名;清理只动本任务目录。
|
||||
"""
|
||||
import tempfile as _tempfile
|
||||
|
||||
file_path_str: str | None = None
|
||||
zip_tmp_path: str | None = None
|
||||
task_dir: Path | None = None
|
||||
try:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
task = DOWNLOAD_TASKS[task_id]
|
||||
task['status'] = 'running'
|
||||
task['msg'] = '解析单集信息...'
|
||||
params = task['params']
|
||||
cancel_ev = task['cancel']
|
||||
program_id = params['program_id']
|
||||
quality = params['quality']
|
||||
need_zip = params['need_zip']
|
||||
cookies = params['cookies']
|
||||
remote_ip = params.get('remote_ip') or '-'
|
||||
|
||||
program = podcast_api.get_program_detail(program_id, cookies)
|
||||
url_info = _podcast_resolve_audio(program, quality, cookies)
|
||||
|
||||
serial = int(program.get('serial_num') or 0)
|
||||
name = program.get('name') or program.get('main_song_name') or '播客单集'
|
||||
prefix = f"{serial:03d} - " if serial else ''
|
||||
base = _podcast_safe_name(f"{prefix}{name}")
|
||||
ext = (url_info.get('type') or 'mp3').lower().lstrip('.')
|
||||
# 磁盘名附节目 ID + 任务目录隔离(两个任务同下载一集互不覆盖、互不删除)
|
||||
task_dir = api_service.downloads_path / 'tasks' / task_id
|
||||
task_dir.mkdir(parents=True, exist_ok=True)
|
||||
disk_name = f"{base} [{program_id}].{ext}"
|
||||
target = task_dir / disk_name
|
||||
filename = f"{base}.{ext}" # 交付名保持干净(不带 ID)
|
||||
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS[task_id]
|
||||
t['msg'] = '下载中...'
|
||||
_podcast_stream_download(url_info['url'], target, url_info.get('size', 0),
|
||||
cancel_ev, task_id, '下载中')
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
file_path_str = str(target)
|
||||
|
||||
result: dict[str, Any]
|
||||
if need_zip:
|
||||
fd, zip_tmp_path = _tempfile.mkstemp(prefix='podcast_zip_', suffix='.zip',
|
||||
dir=str(task_dir))
|
||||
os.close(fd)
|
||||
with zipfile.ZipFile(zip_tmp_path, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
zf.write(file_path_str, arcname=filename)
|
||||
cover_url = program.get('cover_url') or ''
|
||||
if cover_url:
|
||||
try:
|
||||
cover = fetch_cover_bytes(cover_url)
|
||||
if cover:
|
||||
zf.writestr(f"{base}.jpg", cover)
|
||||
except Exception:
|
||||
pass
|
||||
# info.txt 不再写直链(真实 URL 不应随交付文件外泄)
|
||||
info_txt = (
|
||||
f"单集: {name}\n"
|
||||
f"播客: {program.get('radio_name') or ''}\n"
|
||||
f"主播: {program.get('dj_name') or ''}\n"
|
||||
f"时长: {program.get('duration_text') or ''}\n"
|
||||
f"音质: {api_service._get_quality_display_name(url_info.get('level', quality))}\n"
|
||||
)
|
||||
zf.writestr("info.txt", info_txt)
|
||||
# 打包阶段后同样以取消检查收口
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
result = {
|
||||
'file_path': file_path_str,
|
||||
'zip_path': zip_tmp_path,
|
||||
'filename': f"{base}.zip",
|
||||
'mimetype': 'application/zip',
|
||||
'audio_files': [file_path_str],
|
||||
}
|
||||
else:
|
||||
result = {
|
||||
'file_path': file_path_str,
|
||||
'filename': filename,
|
||||
'mimetype': f"audio/{ext}",
|
||||
'audio_files': [file_path_str],
|
||||
}
|
||||
|
||||
# 最终发布前的取消检查(末段取消不得覆盖为 done)
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is None or t.get('status') in ('canceled', 'error'):
|
||||
raise DownloadCanceled("任务已取消")
|
||||
t['status'] = 'done'
|
||||
t['completed_at'] = time.time()
|
||||
t['done'] = 1
|
||||
t['msg'] = '下载完成'
|
||||
t['result'] = result
|
||||
# 交付已落定;结算失败不再把任务改标 error(同单曲)
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 1, 'delivered')
|
||||
except Exception as _e:
|
||||
api_service.logger.error(f"播客单集交付结算失败(重启对账兜底): {task_id} {_e}")
|
||||
try:
|
||||
_record_log('下载', remote_ip, f"[播客] {name}")
|
||||
except Exception:
|
||||
pass
|
||||
api_service.logger.info(f"播客单集下载完成: {filename} ({task_id})")
|
||||
except DownloadCanceled:
|
||||
_mark_task_terminal(task_id, 'canceled', '任务已取消')
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 0, 'canceled')
|
||||
except Exception:
|
||||
pass
|
||||
if task_dir is not None:
|
||||
shutil.rmtree(str(task_dir), ignore_errors=True)
|
||||
api_service.logger.info(f"播客单集下载已取消: {task_id}")
|
||||
except DownloadException as e:
|
||||
_mark_task_terminal(task_id, 'error', str(e))
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 0, 'error')
|
||||
except Exception:
|
||||
pass
|
||||
if task_dir is not None:
|
||||
shutil.rmtree(str(task_dir), ignore_errors=True)
|
||||
api_service.logger.error(f"播客单集下载失败: {task_id} {e}")
|
||||
except Exception as e:
|
||||
# 非预期异常原文 logger 先行,用户可见文案泛化(教训)
|
||||
api_service.logger.error(f"播客单集下载异常: {task_id} {e}\n{traceback.format_exc()}")
|
||||
_mark_task_terminal(task_id, 'error', '下载失败,请稍后重试')
|
||||
try:
|
||||
auth_system.ledger_settle(task_id, 0, 'error')
|
||||
except Exception:
|
||||
pass
|
||||
if task_dir is not None:
|
||||
shutil.rmtree(str(task_dir), ignore_errors=True)
|
||||
|
||||
|
||||
def _podcast_cleanup_files(paths: list[str | None]) -> None:
|
||||
"""清理播客任务产生的临时/残留文件"""
|
||||
for p in paths:
|
||||
if not p:
|
||||
continue
|
||||
try:
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
# 连带清理可能残留的 .part 文件
|
||||
try:
|
||||
part = Path(str(p) + '.part')
|
||||
if part.exists():
|
||||
part.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _podcast_zip_worker(task_id: str) -> None:
|
||||
"""整台播客打包:逐集 下载→入包即删,支持取消与断线暂停(复用批量任务机制)"""
|
||||
import tempfile as _tempfile
|
||||
|
||||
zip_tmp_path: str | None = None
|
||||
ok_count = 0
|
||||
failed: list[str] = []
|
||||
# zip_valid 必须在函数头初始化——空电台/无单集/补扣失败等
|
||||
# 正常业务错误在 try 内早期 raise,异常处理器引用未绑定名会二次崩溃 worker 线程
|
||||
zip_valid = False
|
||||
try:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
task = DOWNLOAD_TASKS[task_id]
|
||||
task['status'] = 'running'
|
||||
task['msg'] = '解析单集列表...'
|
||||
params = task['params']
|
||||
cancel_ev = task['cancel']
|
||||
pause_ev = task['pause']
|
||||
radio_id = params['radio_id']
|
||||
preset_ids = params.get('ids')
|
||||
limit = params['limit']
|
||||
quality = params['quality']
|
||||
cookies = params['cookies']
|
||||
remote_ip = params.get('remote_ip') or '-'
|
||||
|
||||
# 组装单集清单:整台电台(正序全量)或指定单集ID列表
|
||||
if radio_id:
|
||||
episodes = podcast_api.list_radio_all_program_ids(radio_id, cookies, max_count=limit)
|
||||
if not episodes:
|
||||
raise DownloadException("该电台暂无可下载的单集")
|
||||
else:
|
||||
episodes = []
|
||||
for pid in (preset_ids or []):
|
||||
try:
|
||||
prog = podcast_api.get_program_detail(pid, cookies)
|
||||
episodes.append({
|
||||
'id': prog['id'],
|
||||
'name': prog['name'],
|
||||
'main_song_id': prog['main_song_id'],
|
||||
'serial_num': prog['serial_num'],
|
||||
'cover_url': prog['cover_url'],
|
||||
'detail': prog,
|
||||
})
|
||||
except APIException as ae:
|
||||
failed.append(f"{pid}({ae})")
|
||||
episodes = episodes[:limit]
|
||||
if not episodes:
|
||||
raise DownloadException("没有可下载的单集")
|
||||
|
||||
radio_name = _podcast_safe_name(params.get('radio_name') or '', '播客')
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
DOWNLOAD_TASKS[task_id]['total'] = len(episodes)
|
||||
|
||||
# 按次计费补扣:整台电台提交时只预扣 1 次,此处按实际单集数补足差额。
|
||||
# 补扣失败(余额不足/数据库异常)任务必须失败并按账本退费——不允许无账本继续交付
|
||||
try:
|
||||
_buid = params.get('billing_uid')
|
||||
_extra = len(episodes) - int(params.get('billed') or 1)
|
||||
if _buid and _extra > 0:
|
||||
if not auth_system.ledger_supplement(task_id, _extra):
|
||||
raise DownloadException(
|
||||
f"补扣下载次数失败(需补扣 {_extra} 次),任务已取消,预扣部分将退回")
|
||||
if _buid:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
DOWNLOAD_TASKS[task_id]['billing_user_id'] = int(_buid)
|
||||
DOWNLOAD_TASKS[task_id]['billing_cost'] = len(episodes)
|
||||
except DownloadCanceled:
|
||||
raise
|
||||
except DownloadException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise DownloadException(f"补扣下载次数异常: {e}") from e
|
||||
|
||||
def _wait_if_paused() -> None:
|
||||
"""暂停时等待恢复或取消"""
|
||||
if not (pause_ev.is_set() or task['status'] == 'paused'):
|
||||
return
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
task['status'] = 'paused'
|
||||
task['done'] = ok_count + len(failed)
|
||||
task['msg'] = '已暂停(重新打开页面自动恢复)'
|
||||
while not cancel_ev.is_set() and (pause_ev.is_set() or task['status'] == 'paused'):
|
||||
time.sleep(1)
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
pause_ev.clear()
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
task['status'] = 'running'
|
||||
task['msg'] = '继续下载...'
|
||||
|
||||
def _update_progress(msg: str = '') -> None:
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS[task_id]
|
||||
t['done'] = ok_count + len(failed)
|
||||
t['failed'] = len(failed)
|
||||
if msg:
|
||||
t['msg'] = msg
|
||||
|
||||
# 对齐音乐批量 F-B 口径——zip_valid 仅在 ZIP with 块正常
|
||||
# 退出(中央目录完整写出)后才置 True(初始 False 已上移函数头);
|
||||
# 未 valid 一律不发布、不按 ok_count 结算
|
||||
# 同音乐批量——ZIP 构建放 downloads 卷,不落系统 %TEMP%
|
||||
# 与音乐批量打包同口径:建在任务专属目录 tasks/<task_id>/,根目录缓存清扫不再触及交付物
|
||||
_podcast_task_dir = api_service.downloads_path / 'tasks' / task_id
|
||||
_podcast_task_dir.mkdir(parents=True, exist_ok=True)
|
||||
with _tempfile.NamedTemporaryFile(prefix='podcast_batch_', suffix='.zip',
|
||||
dir=str(_podcast_task_dir),
|
||||
delete=False) as zip_tmp:
|
||||
zip_tmp_path = zip_tmp.name
|
||||
used_arcnames = set() # 包内条目名去重(同名单集/同号不同集不得互相覆盖)
|
||||
with zipfile.ZipFile(zip_tmp, 'w', zipfile.ZIP_DEFLATED) as zf:
|
||||
for idx, ep in enumerate(episodes, 1):
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
_wait_if_paused()
|
||||
# 并发对齐音乐批量:播客打包同样占用全局批量名额(BATCH_ZIP_SEM,全局 3),
|
||||
# 逐集 acquire/release——暂停等待/取消时不长期占名额;音频本身仍受
|
||||
# 全局下载信号量约束。原实现不受批量并发控制,多用户同时打包会无限并发
|
||||
if not BATCH_ZIP_SEM.acquire_cancellable(cancel_ev):
|
||||
raise DownloadCanceled("任务已取消")
|
||||
try:
|
||||
label = f"[{idx}/{len(episodes)}] {ep['name'][:24]}"
|
||||
_update_progress(f"{label} 解析中...")
|
||||
try:
|
||||
prog = ep.get('detail') or podcast_api.get_program_detail(ep['id'], cookies)
|
||||
url_info = _podcast_resolve_audio(prog, quality, cookies)
|
||||
_update_progress(f"{label} 下载中...")
|
||||
ext = (url_info.get('type') or 'mp3').lower().lstrip('.')
|
||||
serial = int(ep.get('serial_num') or 0)
|
||||
prefix = f"{serial:03d} - " if serial else ''
|
||||
base = _podcast_safe_name(f"{prefix}{ep['name']}")
|
||||
# 中间文件移入任务目录——落 downloads 根目录会被
|
||||
# 缓存清扫按 mtime 打断慢速单集(Linux 上 unlink 打开中的文件)
|
||||
tmp_audio = _podcast_task_dir / f"podcast_zip_tmp_{task_id}_{idx}.{ext}"
|
||||
try:
|
||||
_podcast_stream_download(url_info['url'], tmp_audio,
|
||||
url_info.get('size', 0), cancel_ev)
|
||||
# 重名条目自动追加序号
|
||||
arcname = f"{base}.{ext}"
|
||||
k = 2
|
||||
while arcname in used_arcnames:
|
||||
arcname = f"{base} ({k}).{ext}"
|
||||
k += 1
|
||||
used_arcnames.add(arcname)
|
||||
zf.write(str(tmp_audio), arcname=arcname, compress_type=zipfile.ZIP_STORED)
|
||||
finally:
|
||||
# finally 连带清理 .part 残片(原实现只删正文,
|
||||
# 取消/失败时残片要等 1 小时缓存清扫兜底)
|
||||
_podcast_cleanup_files([str(tmp_audio)])
|
||||
# 首集附带封面,便于识别
|
||||
if idx == 1 and ep.get('cover_url'):
|
||||
try:
|
||||
cover = fetch_cover_bytes(ep['cover_url'])
|
||||
if cover:
|
||||
zf.writestr("cover.jpg", cover, compress_type=zipfile.ZIP_STORED)
|
||||
except Exception:
|
||||
pass
|
||||
ok_count += 1
|
||||
_update_progress(f"{label} 完成")
|
||||
except DownloadCanceled:
|
||||
raise
|
||||
except Exception as e:
|
||||
failed.append(f"{ep['name']}({e})")
|
||||
_update_progress()
|
||||
finally:
|
||||
BATCH_ZIP_SEM.release()
|
||||
|
||||
# 走到这里说明 ZIP with 块正常退出、中央目录完整,才允许发布
|
||||
zip_valid = True
|
||||
|
||||
if ok_count == 0:
|
||||
raise DownloadException(f"全部单集下载失败: {'; '.join(failed[:3])}")
|
||||
|
||||
# ZIP 未正常完成(磁盘满写坏中央目录等)——不发布坏包,按全额退费收敛
|
||||
if not zip_valid:
|
||||
if zip_tmp_path:
|
||||
try:
|
||||
os.remove(zip_tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
zip_tmp_path = None
|
||||
refunded = _settle_batch_billing(task_id, 0, 'error')
|
||||
_mark_task_terminal(task_id, 'error',
|
||||
f"ZIP 打包失败(可能磁盘空间不足),已退回 {refunded} 次" if refunded
|
||||
else "ZIP 打包失败(可能磁盘空间不足),费用未扣或已退回")
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['failed'] = t.get('failed', 0) + ok_count
|
||||
t['refunded'] = refunded
|
||||
api_service.logger.error(f"播客打包构建失败未发布: {task_id} ok_count={ok_count} 退回{refunded}次")
|
||||
return
|
||||
|
||||
# 最终发布前的取消检查(末段取消不得覆盖为 done)
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
refunded = _settle_batch_billing(task_id, ok_count)
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
# 发布前锁内复查取消/失效状态(同音乐批量)
|
||||
if t is None or t.get('status') in ('canceled', 'error'):
|
||||
raise DownloadCanceled("任务已取消")
|
||||
if cancel_ev.is_set():
|
||||
raise DownloadCanceled("任务已取消")
|
||||
t['status'] = 'done'
|
||||
t['completed_at'] = time.time()
|
||||
t['done'] = ok_count + len(failed)
|
||||
t['failed'] = len(failed)
|
||||
t['msg'] = f"成功{ok_count} 失败{len(failed)}" + (f" 退回{refunded}次" if refunded else '')
|
||||
t['refunded'] = refunded
|
||||
t['result'] = {
|
||||
'zip_path': zip_tmp_path,
|
||||
'filename': f"播客_{radio_name}_{ok_count}集.zip",
|
||||
'audio_files': [],
|
||||
}
|
||||
try:
|
||||
first_name = episodes[0].get('name') if episodes else ''
|
||||
_record_log('下载', remote_ip, f"[播客打包] {radio_name} · {first_name}")
|
||||
except Exception:
|
||||
pass
|
||||
api_service.logger.info(f"播客打包完成: {radio_name} 成功{ok_count} 失败{len(failed)} ({task_id})")
|
||||
except DownloadCanceled:
|
||||
# 部分成功后取消——已入包部分照常交付,未完成部分按账本退费
|
||||
# ZIP 未正常完成(写坏)时不按 ok_count 交付——按全额退费收敛
|
||||
delivered = ok_count if zip_valid else 0
|
||||
refunded = _settle_batch_billing(task_id, delivered, 'canceled')
|
||||
if delivered > 0 and zip_tmp_path:
|
||||
_mark_task_terminal(task_id, 'canceled',
|
||||
f"已取消(已打包 {ok_count} 集可领取)" +
|
||||
(f",退回 {refunded} 次" if refunded else ''))
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['refunded'] = refunded
|
||||
t['result'] = {
|
||||
'zip_path': zip_tmp_path,
|
||||
'filename': f"播客_{radio_name or '打包'}_{ok_count}集.zip",
|
||||
'audio_files': [],
|
||||
}
|
||||
else:
|
||||
_mark_task_terminal(task_id, 'canceled',
|
||||
'任务已取消' + (f"(已退回 {refunded} 次下载次数)" if refunded else ''))
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
t = DOWNLOAD_TASKS.get(task_id)
|
||||
if t is not None:
|
||||
t['refunded'] = refunded
|
||||
if zip_tmp_path:
|
||||
try:
|
||||
os.remove(zip_tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
api_service.logger.info(f"播客打包已取消: {task_id}(成功{ok_count} 退回{refunded}次)")
|
||||
except DownloadException as e:
|
||||
delivered = ok_count if zip_valid else 0
|
||||
refunded = _settle_batch_billing(task_id, delivered)
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
_t = DOWNLOAD_TASKS.get(task_id)
|
||||
# 终态先到先得——已终态(如用户取消/超时兜底)
|
||||
# 不再被覆盖为 error(堵 canceled→error 非法迁移边)
|
||||
if _t and _t.get('status') not in ('done', 'error', 'canceled'):
|
||||
_t['status'] = 'error'
|
||||
_t['completed_at'] = time.time() # 保留期从完成时刻起算
|
||||
_t['msg'] = str(e) + (f"(已退回 {refunded} 次下载次数)" if refunded else '')
|
||||
_t['refunded'] = refunded
|
||||
elif _t:
|
||||
_t['refunded'] = refunded
|
||||
if zip_tmp_path:
|
||||
try:
|
||||
os.remove(zip_tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
api_service.logger.error(f"播客打包失败: {task_id} {e}(退回{refunded}次)")
|
||||
except Exception as e:
|
||||
# 非预期异常原文 logger 先行,用户可见文案泛化
|
||||
# (DownloadException 的文案是我们自己拼的用户可读文本,保留 str(e))
|
||||
api_service.logger.error(f"播客打包异常: {task_id} {e}\n{traceback.format_exc()}")
|
||||
delivered = ok_count if zip_valid else 0
|
||||
refunded = _settle_batch_billing(task_id, delivered)
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
_t = DOWNLOAD_TASKS.get(task_id)
|
||||
# 同上——已终态不再覆盖为 error
|
||||
if _t and _t.get('status') not in ('done', 'error', 'canceled'):
|
||||
_t['status'] = 'error'
|
||||
_t['completed_at'] = time.time() # 保留期从完成时刻起算
|
||||
_t['msg'] = '打包失败,请稍后重试' + (f"(已退回 {refunded} 次下载次数)" if refunded else '')
|
||||
_t['refunded'] = refunded
|
||||
elif _t:
|
||||
_t['refunded'] = refunded
|
||||
if zip_tmp_path:
|
||||
try:
|
||||
os.remove(zip_tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
api_service.logger.error(f"播客打包异常: {task_id} {e}(退回{refunded}次)\n{traceback.format_exc()}")
|
||||
+106
-1890
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,63 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""同步流式响应垫片(音频分段透传)。
|
||||
|
||||
(自 services/runtime.py 拆出,行为与原实现一致。)
|
||||
"""
|
||||
|
||||
|
||||
|
||||
class _SyncStreamResponse:
|
||||
"""同步流式响应包装(把 aiohttp 的异步流按需拉成同步迭代)。
|
||||
|
||||
供流式下载使用:worker 是同步线程,这里经 core.async_bridge 在后台事件循环
|
||||
逐块取数据,避免引入 requests 依赖(Flask 已删除、requests 已移出依赖)。
|
||||
"""
|
||||
|
||||
def __init__(self, url: str, timeout: float):
|
||||
from core import async_bridge
|
||||
self._bridge = async_bridge
|
||||
self._url = url
|
||||
self._timeout = timeout
|
||||
self._agent = None
|
||||
self.status = 0
|
||||
self.headers: dict[str, str] = {}
|
||||
self._resp = None
|
||||
|
||||
def __enter__(self):
|
||||
self._agent = self._bridge.run_sync(self._open())
|
||||
self.status = self._agent[0]
|
||||
self.headers = self._agent[1]
|
||||
return self
|
||||
|
||||
async def _open(self):
|
||||
from core.http_client import client
|
||||
resp = await client.open('GET', self._url, headers={
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Safari/537.36',
|
||||
'Referer': 'https://music.163.com/',
|
||||
}, timeout=self._timeout)
|
||||
return resp.status, dict(resp.headers), resp
|
||||
|
||||
def iter_content(self, chunk_size: int = 256 * 1024):
|
||||
assert self._agent is not None
|
||||
resp = self._agent[2]
|
||||
it = resp.content.iter_chunked(chunk_size)
|
||||
while True:
|
||||
try:
|
||||
chunk = self._bridge.run_sync(it.__anext__())
|
||||
except StopAsyncIteration:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
def __exit__(self, *exc):
|
||||
if self._agent is not None and self._agent[2] is not None:
|
||||
try:
|
||||
self._agent[2].close()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _stream_open(url: str, timeout: float = 60) -> _SyncStreamResponse:
|
||||
"""打开同步可迭代的流式响应(见 _SyncStreamResponse)。"""
|
||||
return _SyncStreamResponse(url, timeout)
|
||||
@@ -0,0 +1,259 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""任务监控循环与设备心跳。
|
||||
|
||||
(自 services/runtime.py 拆出,行为与原实现一致。)
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import auth_system
|
||||
from core.config import (
|
||||
CANCEL_AFTER_SECONDS,
|
||||
HEARTBEAT_INTERVAL,
|
||||
PAUSE_AFTER_SECONDS,
|
||||
RESULT_RETENTION_SECONDS,
|
||||
)
|
||||
from core.state import (
|
||||
DEVICE_LAST_SEEN,
|
||||
DEVICE_LAST_SEEN_LOCK,
|
||||
DOWNLOAD_TASKS,
|
||||
DOWNLOAD_TASKS_LOCK,
|
||||
)
|
||||
|
||||
from ._shared import (
|
||||
_LEDGER_LAST_CLEANUP,
|
||||
_LEDGER_LAST_RECOVER,
|
||||
_MONITOR_ERR_TS,
|
||||
_TASKS_SWEEP_LAST,
|
||||
_sweep_rate_buckets,
|
||||
)
|
||||
from ._taskcore import (
|
||||
_task_files,
|
||||
)
|
||||
from .api_service import (
|
||||
api_service,
|
||||
)
|
||||
|
||||
|
||||
def _quiesce_tasks_for_restore() -> None:
|
||||
"""恢复数据库前取消全部在途任务(worker 不再引用旧库用户结算)"""
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
for task in DOWNLOAD_TASKS.values():
|
||||
if task.get('status') in ('queued', 'running', 'paused'):
|
||||
task['status'] = 'canceled'
|
||||
task['msg'] = '数据库恢复中,任务已取消(费用按实际交付结算)'
|
||||
task['completed_at'] = time.time()
|
||||
try:
|
||||
task['cancel'].set()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
# 数据库恢复前静默在途任务
|
||||
auth_system.register_before_restore_hook(_quiesce_tasks_for_restore)
|
||||
|
||||
|
||||
def _sweep_download_cache(max_age_seconds: int = RESULT_RETENTION_SECONDS) -> None:
|
||||
"""下载缓存清扫(配套:批量任务不再即时删除共享缓存,改由这里按年龄回收)。
|
||||
只清扫 downloads 根目录的普通文件,不碰 tasks/ 子目录(那是任务专属交付物)。"""
|
||||
try:
|
||||
root = api_service.downloads_path
|
||||
now = time.time()
|
||||
for p in root.iterdir():
|
||||
try:
|
||||
if p.is_file() and now - p.stat().st_mtime > max_age_seconds:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _task_monitor_loop() -> None:
|
||||
"""后台监控:
|
||||
- 心跳超时(40s)→ 暂停离线设备的批量任务,释放排队名额给下一位
|
||||
- 心跳超时 > 30 分钟 → 自动取消该设备的所有未完成任务并清理文件
|
||||
- 终态任务按 completed_at 保留 35 分钟(与前端「领取」入口对齐)
|
||||
- 运行中/排队/暂停任务超过 4 小时仍未结束 → 置取消(cancel Event 一并置位)
|
||||
- 下载缓存按年龄清扫(配套,同为 35 分钟口径)
|
||||
- 顺带清理限流桶中的过期项(O-5:不再只依赖"新建桶时"的清理时机)
|
||||
"""
|
||||
while True:
|
||||
time.sleep(HEARTBEAT_INTERVAL)
|
||||
try:
|
||||
now = time.time()
|
||||
# 终态任务清理(按完成时刻;删除失败保留记录下轮重试,最多 8 次)----
|
||||
# 锁内只收集过期清单,文件删除移到锁外——GB 级任务目录
|
||||
# 的删除曾阻塞锁数十秒,期间全站进度上报/提交/取消/领取全部停摆
|
||||
finished_done: list[str] = []
|
||||
finished_retry: list[str] = []
|
||||
expired: list[tuple] = []
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
for tid, task in DOWNLOAD_TASKS.items():
|
||||
if task.get('status') not in ('done', 'error', 'canceled'):
|
||||
continue
|
||||
done_at = task.get('completed_at') or 0
|
||||
# 兼容无 completed_at 的旧记录:回退到 start
|
||||
if not done_at:
|
||||
done_at = task.get('start', 0)
|
||||
if now - done_at <= RESULT_RETENTION_SECONDS:
|
||||
continue
|
||||
expired.append((tid, _task_files(task),
|
||||
api_service.downloads_path / 'tasks' / tid))
|
||||
cleanup_results: list[tuple] = []
|
||||
for tid, files, task_dir in expired:
|
||||
all_removed = True
|
||||
for f in files:
|
||||
try:
|
||||
os.remove(f)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except OSError:
|
||||
all_removed = False # 文件暂时不可删(被占用等)
|
||||
if task_dir.exists():
|
||||
shutil.rmtree(str(task_dir), ignore_errors=True)
|
||||
if task_dir.exists():
|
||||
all_removed = False
|
||||
cleanup_results.append((tid, all_removed))
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
for tid, all_removed in cleanup_results:
|
||||
task = DOWNLOAD_TASKS.get(tid)
|
||||
if task is None:
|
||||
finished_done.append(tid) # 锁外期间已被其他路径移除
|
||||
continue
|
||||
if all_removed:
|
||||
finished_done.append(tid)
|
||||
else:
|
||||
task['_cleanup_tries'] = int(task.get('_cleanup_tries') or 0) + 1
|
||||
if task['_cleanup_tries'] >= 8:
|
||||
# 重试多次仍失败:放弃记录但大声记录日志(文件留给缓存清扫兜底)
|
||||
api_service.logger.warning(
|
||||
f"任务 {tid} 结果文件多次清理失败,已放弃(文件可能被占用)")
|
||||
finished_done.append(tid)
|
||||
else:
|
||||
finished_retry.append(tid)
|
||||
for tid in finished_done:
|
||||
DOWNLOAD_TASKS.pop(tid, None)
|
||||
if finished_done:
|
||||
api_service.logger.info(f"监控线程清理过期终态任务: {finished_done}")
|
||||
if finished_retry:
|
||||
api_service.logger.info(f"监控线程保留清理失败任务待重试: {finished_retry}")
|
||||
# ---- 长时间未结束的任务:置取消(含 cancel Event,唤醒可能的阻塞等待)----
|
||||
timeout_settle: list[str] = []
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
for tid, task in DOWNLOAD_TASKS.items():
|
||||
if (task.get('status') in ('queued', 'running', 'paused')
|
||||
and now - task.get('start', 0) > 4 * 3600):
|
||||
task['status'] = 'canceled'
|
||||
task['msg'] = '任务超时未完成,已自动取消(费用按实际交付结算)'
|
||||
task['completed_at'] = time.time()
|
||||
try:
|
||||
task['cancel'].set()
|
||||
except Exception:
|
||||
pass
|
||||
# worker 线程已死则无人再走终态结算——监控代为按账本
|
||||
# 结算退款(settle 幂等:worker 若已结算过,此处返回 0)
|
||||
th = task.get('worker_thread')
|
||||
if th is not None and not th.is_alive():
|
||||
timeout_settle.append(tid)
|
||||
api_service.logger.warning(f"任务超时自动取消: {tid}")
|
||||
# 结算移到任务锁外(同 _mark_task_terminal 口径)——
|
||||
# 库忙 15s 不再随任务锁波及全站任务端点
|
||||
for tid in timeout_settle:
|
||||
try:
|
||||
auth_system.ledger_settle(tid, 0, 'timeout')
|
||||
except Exception:
|
||||
pass
|
||||
# ---- 下载缓存清扫(保留期同交付口径 35 分钟)+ 限流桶过期项顺手清理(O-5)----
|
||||
_sweep_download_cache(RESULT_RETENTION_SECONDS)
|
||||
_sweep_rate_buckets()
|
||||
# 账本周期兜底(原仅启动执行一次)----
|
||||
# 孤儿对账:长运行进程内 open 且账龄 > 5h(严格大于任务 4h 硬超时 + 监控
|
||||
# tick 抖动——账本行 created_at 整秒截断使账龄恒 ≥ 任务龄,旧 4h 相等口径
|
||||
# 存在抢退合法 4h+ 长任务的窗口)的行每分钟周期检查退回;
|
||||
# settled 行 90 天清理改为每小时低频执行,防长运行进程下表无界增长
|
||||
try:
|
||||
# 时间戳门槛替代 tick 取模——单轮工作段超过周期时
|
||||
# 取模落点会被整段跳过,时间戳门槛只延迟不漏跑
|
||||
if now - _LEDGER_LAST_RECOVER['ts'] > 60:
|
||||
_LEDGER_LAST_RECOVER['ts'] = now
|
||||
auth_system.ledger_recover_orphans(max_age_seconds=5 * 3600)
|
||||
if now - _LEDGER_LAST_CLEANUP['ts'] > 24 * 3600:
|
||||
_LEDGER_LAST_CLEANUP['ts'] = now
|
||||
auth_system.ledger_cleanup_settled(90 * 86400)
|
||||
except Exception as e:
|
||||
api_service.logger.warning(f"账本周期兜底异常(下轮重试): {e}")
|
||||
# tasks/ 孤儿目录每小时低频补扫(>24h 且无记录)----
|
||||
try:
|
||||
if now - _TASKS_SWEEP_LAST['ts'] > 3600:
|
||||
_TASKS_SWEEP_LAST['ts'] = now
|
||||
tasks_root = api_service.downloads_path / 'tasks'
|
||||
if tasks_root.exists():
|
||||
for d in tasks_root.iterdir():
|
||||
try:
|
||||
if (d.is_dir() and d.name not in DOWNLOAD_TASKS
|
||||
and now - d.stat().st_mtime > 24 * 3600):
|
||||
shutil.rmtree(str(d), ignore_errors=True)
|
||||
api_service.logger.info(f"tasks 孤儿目录补扫移除: {d.name}")
|
||||
except FileNotFoundError:
|
||||
pass # 单条竞态不中止整轮扫描
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as e:
|
||||
api_service.logger.warning(f"tasks 孤儿目录补扫异常(下轮重试): {e}")
|
||||
# ---- 设备心跳离线处理 ----
|
||||
with DEVICE_LAST_SEEN_LOCK:
|
||||
offline = {d: now - ts for d, ts in DEVICE_LAST_SEEN.items() if now - ts > PAUSE_AFTER_SECONDS}
|
||||
if not offline:
|
||||
continue
|
||||
with DOWNLOAD_TASKS_LOCK:
|
||||
for tid, task in DOWNLOAD_TASKS.items():
|
||||
dev = task.get('device_id')
|
||||
gap = offline.get(dev) if dev else None
|
||||
if not gap:
|
||||
continue
|
||||
st = task.get('status')
|
||||
if gap > CANCEL_AFTER_SECONDS and st in ('queued', 'running', 'paused'):
|
||||
# 离线超 30 分钟:请求取消(文件删除与退款统一交给 worker 的
|
||||
# 取消收尾/终态清理;worker 已不在时由启动对账按账本兜底)
|
||||
task['status'] = 'canceled'
|
||||
task['msg'] = '设备离线超过 30 分钟,任务已自动取消'
|
||||
task['completed_at'] = time.time()
|
||||
try:
|
||||
task['cancel'].set()
|
||||
except Exception:
|
||||
pass
|
||||
api_service.logger.info(f"设备离线超 30 分钟自动请求取消: {tid}")
|
||||
elif task.get('type') == 'batch' and st in ('queued', 'running'):
|
||||
# 心跳超时:先暂停,让出排队名额给下一位
|
||||
task['status'] = 'paused'
|
||||
task['msg'] = '设备离线,任务暂停(重新打开页面自动恢复)'
|
||||
task['pause'].set()
|
||||
except Exception:
|
||||
# 持续异常不能完全静默(否则监控失能无人知晓):限频记录,每 10 分钟最多一条
|
||||
_now = time.time()
|
||||
_MONITOR_ERR_TS['n'] += 1
|
||||
if _now - _MONITOR_ERR_TS['ts'] > 600:
|
||||
api_service.logger.warning(
|
||||
f"任务监控循环异常(近 10 分钟累计 {_MONITOR_ERR_TS['n']} 次,"
|
||||
f"最后异常见堆栈): {traceback.format_exc()[-600:]}")
|
||||
_MONITOR_ERR_TS['ts'] = _now
|
||||
_MONITOR_ERR_TS['n'] = 0
|
||||
|
||||
|
||||
def _touch_device(device_id: str) -> None:
|
||||
"""记录设备心跳时间(设备表容量上限——device_id 是客户端任意串,
|
||||
登录用户可提交海量随机 ID 使字典无界增长,超限清最旧;
|
||||
入口已统一钳 64,此处兜底再钳一次防新调用点绕过)"""
|
||||
device_id = str(device_id or '')[:64]
|
||||
if not device_id:
|
||||
return
|
||||
with DEVICE_LAST_SEEN_LOCK:
|
||||
if len(DEVICE_LAST_SEEN) >= 5000:
|
||||
# 保留最近心跳的设备:按最后心跳时间排序,砍掉最旧的一半
|
||||
for d in sorted(DEVICE_LAST_SEEN, key=lambda k: DEVICE_LAST_SEEN.get(k, 0.0))[:2500]:
|
||||
DEVICE_LAST_SEEN.pop(d, None)
|
||||
DEVICE_LAST_SEEN[device_id] = time.time()
|
||||
Reference in new issue
Block a user