Archived
拆掉「同步调异步 / 异步桥同步」两类垫片的第二半(HTTP 线 B2 见前序提交)。 worker 侧此前是「起线程跑同步函数、函数体内部再用 asyncio.to_thread 把活塞给 另一条循环」——线程与协程互等,取消/暂停只能靠轮询。本轮把 worker 自身变成 协程,由 _start_task 直接挂到应用事件循环上,桥接层随之失去存在理由。 改动(按子系统): - music_downloader:downloader / _aria2 / _concurrency / cover / _http / __main__ 全链路协程化。per-file 锁改 asyncio.Lock;计数闸改 「active 计数 + await asyncio.sleep(0.5)」的可取消等待(原阻塞式 with 会让 已取消任务仍要等全局名额腾出,卡死并拖后腿退款);aria2p 的同步 RPC 仍经 to_thread,轮询改 asyncio.sleep;新增 core.http_client.iter_chunks,把**流式 读取期**的 aiohttp.ClientError 收敛成 UpstreamNetworkError(请求建立阶段早已 有这条路径,读取阶段此前没有,断流会绕过调用方的 UpstreamError 分类)。 删除 get_http_session / _SESSION_LOCAL。 models._RetryFromScratch 不再继承 requests.RequestException 而继承 Exception: 它是「续传起点失效、残片已清、从头重下」,既不是确定性失败 (DownloadException),也不是残片已保留的网络失败(UpstreamError),必须独立 成类且在 except 链中先于二者捕获。 - music_api:podcast.py 全部函数协程化(直链解析内部改 await NeteaseAPI.get_song_url);compat.py 删除——16 个死掉的同步门面早已 清空,文件只剩空壳。 - services:_taskcore / _zipcore / batch / downloads / podcast / tasks 协程化。 threading.Thread → loop.create_task(任务句柄字段 worker_thread → worker_task),ThreadPoolExecutor → asyncio.gather + asyncio.Semaphore, 终态函数与结算回调全部改 await;监控线程里「worker 已死」判定改 task.done()。 删除 services/streaming.py(同步流垫片,唯一消费者已迁到 core.http_client.open)。zipfile 实例仍非线程安全,故「下载」并行、 「入包」严格串行。 - web:路由层 130+ 处 to_thread 拆除——被包住的目标自己已是协程,再包一层只是 把协程丢进线程里跑一圈。tasks_registry.start_task 直接 create_task。 web/app.py 关闭段去掉 async_bridge.shutdown()(不再有第二条约循环)。 web/routers/download.py 的播客端点补上 3 处漏掉的 await (extract_podcast_ref / get_radio_detail)——漏掉时不报错、只静默返回协程对象, 靠 pyright 抓出来后修正。 留在 to_thread 上的都是**正当的阻塞卸载**:同步 SQLite 落库与账本结算、 mutagen 全量重写标签、zipfile 整段拷贝、shutil.rmtree、aria2p 的同步 RPC、 PBKDF2 的 10 万次迭代。ruff 对残留的本地元数据操作(exists/stat/os.replace/ 往 .part 顺序写分块)按「微秒级、卸载的调度开销大于操作本身」放行,pyproject 逐条注明理由。 取消/暂停仍由 threading.Event 承载:置位方是请求线程/监控线程,读取方是 worker 协程,只有「置位」与「轮询读取」两种操作,既不需要跨循环唤醒也不会卡住事件 循环——比 asyncio.Event 更省心(ASYNC110 同理按此豁免)。 测试:10 个文件的替身边界同步迁移——网络替身从 core.sync_http 的 get_http_session 换成 downloader/服务模块的 _http.open + **真实** iter_chunks (顺带钉住读取期异常收敛);`_run` 改为在独立线程里 asyncio.run(worker(...)); aria2 的假时钟从 time.sleep 换成 _POLL_INTERVAL=0;monitor 的线程句柄替身换成 done() 可控的轻量替身(asyncio.Task 的构造需要运行中的事件循环,而监控循环跑在 常驻线程里);podcast / 封面 / 结算包装等替身一律改 async。 门禁:ruff 全过 / pyright 0-0 / pytest 383 项全绿(0 失败)。 下一步 B2:auth_system 12 个模块改 aiosqlite 背板、web 层剩余 to_thread 收尾; B3:删 core/sync_http.py、core/async_bridge.py、core/http_errors.py(现仅 sync_http 自用)。
222 lines
8.6 KiB
Python
222 lines
8.6 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""任务状态端点。
|
||
|
||
端点:
|
||
- POST/GET /api/v1/topen ↔ Flask /api/topen(请求统计,无鉴权)
|
||
- POST/GET /api/v1/queue ↔ Flask /api/queue(排队状态,无鉴权)
|
||
- POST/GET /api/v1/batch_progress ↔ Flask /api/batch_progress(需登录,仅本人任务)
|
||
- POST /api/v1/task/cancel ↔ Flask /api/task/cancel(需登录,仅本人任务)
|
||
- POST/GET /api/v1/task/heartbeat ↔ Flask /api/task/heartbeat(需登录)
|
||
- POST/GET /api/v1/task/list ↔ Flask /api/task/list(需登录)
|
||
|
||
这些端点读写 core.state 的共享任务表(与任务执行侧同一批对象),因此
|
||
看到的是同一份任务状态。
|
||
"""
|
||
import time
|
||
from typing import Annotated, Any
|
||
|
||
from fastapi import APIRouter, Depends, Request
|
||
|
||
from core.state import (
|
||
BATCH_DURATIONS,
|
||
BATCH_DURATIONS_LOCK,
|
||
DEVICE_LAST_SEEN,
|
||
DEVICE_LAST_SEEN_LOCK,
|
||
DOWNLOAD_TASKS,
|
||
DOWNLOAD_TASKS_LOCK,
|
||
REQUEST_STATS,
|
||
STATS_LOCK,
|
||
)
|
||
from services import tasks as services_tasks
|
||
from web import ratelimit
|
||
from web.deps import SiteUser, require_login
|
||
from web.request_utils import (
|
||
request_data as _request_data,
|
||
)
|
||
from web.response import error, success
|
||
|
||
router = APIRouter()
|
||
|
||
LoginUser = Annotated[SiteUser, Depends(require_login)]
|
||
|
||
|
||
|
||
def _owned_by(task: dict[str, Any], user: SiteUser) -> bool:
|
||
return int(task.get('user_id') or 0) == int(user.id)
|
||
|
||
|
||
def _semaphores():
|
||
"""批量打包/下载信号量(懒取,与 core.state 中的同一对象共享计数)。"""
|
||
from services._shared import BATCH_ZIP_SEM
|
||
from services.api_service import api_service
|
||
return BATCH_ZIP_SEM, api_service.downloader._download_semaphore
|
||
|
||
|
||
@router.api_route('/api/topen', methods=['GET', 'POST'])
|
||
async def topen(request: Request):
|
||
"""接口请求统计(实时)——锁内快照,避免 total 与明细不一致。"""
|
||
from services._shared import ENDPOINT_NAMES
|
||
with STATS_LOCK:
|
||
snapshot = {p: dict(v) for p, v in REQUEST_STATS.items()}
|
||
endpoints = []
|
||
total = sum(v['count'] for v in snapshot.values())
|
||
for path, info in snapshot.items():
|
||
endpoints.append({
|
||
'path': path,
|
||
'name': ENDPOINT_NAMES.get(path, path),
|
||
'count': info['count'],
|
||
'percent': round(info['count'] / total * 100, 1) if total else 0,
|
||
'last': info['last'],
|
||
})
|
||
endpoints.sort(key=lambda x: x['count'], reverse=True)
|
||
return success({
|
||
'total': total,
|
||
'endpoint_count': len(endpoints),
|
||
'max_count': endpoints[0]['count'] if endpoints else 0,
|
||
'endpoints': endpoints,
|
||
}, '获取请求统计成功')
|
||
|
||
|
||
@router.api_route('/api/queue', methods=['GET', 'POST'])
|
||
async def queue(request: Request):
|
||
"""实时排队状态(排队人数 + 预计等待时间)。"""
|
||
batch_sem, dl_sem = _semaphores()
|
||
avg_batch = None
|
||
with BATCH_DURATIONS_LOCK:
|
||
if BATCH_DURATIONS:
|
||
avg_batch = sum(BATCH_DURATIONS) / len(BATCH_DURATIONS)
|
||
return success({
|
||
'batch_capacity': batch_sem.capacity,
|
||
'batch_active': batch_sem.active,
|
||
'batch_waiting': batch_sem.waiting,
|
||
'download_capacity': dl_sem.capacity,
|
||
'download_active': dl_sem.active,
|
||
'download_waiting': dl_sem.waiting,
|
||
'avg_batch_seconds': round(avg_batch, 1) if avg_batch else None,
|
||
}, '获取排队状态成功')
|
||
|
||
|
||
@router.api_route('/api/batch_progress', methods=['GET', 'POST'])
|
||
async def batch_progress(user: LoginUser, request: Request):
|
||
"""下载任务进度(已处理 X/Y + 百分比 + 预计剩余时间)。"""
|
||
data = await _request_data(request)
|
||
task_id = str(data.get('task_id') or '').strip()
|
||
if not task_id:
|
||
return error('缺少 task_id 参数')
|
||
now = time.time()
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
task = DOWNLOAD_TASKS.get(task_id)
|
||
if not task:
|
||
return error('任务不存在或已过期', 404)
|
||
if not _owned_by(task, user):
|
||
return error('无权查询该任务', 403)
|
||
|
||
total = task.get('total', 0)
|
||
done = task.get('done', 0)
|
||
live = task.get('live_percent')
|
||
if task.get('status') == 'running' and isinstance(live, (int, float)):
|
||
percent = round(min(99.0, max(0.0, float(live))), 1)
|
||
elif total:
|
||
percent = round(done / total * 100, 1)
|
||
else:
|
||
percent = 100.0 if task.get('status') in ('done',) else 0.0
|
||
elapsed = now - task.get('start', now)
|
||
eta = None
|
||
if task.get('status') == 'running' and done > 0 and elapsed > 0 and total:
|
||
rate = done / elapsed
|
||
if rate > 0:
|
||
eta = round((total - done) / rate, 1)
|
||
result = task.get('result') or {}
|
||
has_result = bool(result.get('zip_path') or result.get('file_path'))
|
||
return success({
|
||
'total': total,
|
||
'done': done,
|
||
'failed': task.get('failed', 0),
|
||
'percent': percent,
|
||
'status': task.get('status', 'running'),
|
||
'elapsed': round(elapsed, 1),
|
||
'eta': eta,
|
||
'msg': task.get('msg', ''),
|
||
'refunded': task.get('refunded', 0),
|
||
'has_result': has_result,
|
||
}, '获取任务进度成功')
|
||
|
||
|
||
@router.post('/api/task/cancel')
|
||
async def task_cancel(user: LoginUser, request: Request):
|
||
"""取消下载任务(仅任务本人的站点账号可取消)。"""
|
||
data = await _request_data(request)
|
||
task_id = str(data.get('task_id') or '').strip()
|
||
if not task_id:
|
||
return error('缺少 task_id 参数')
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
task = DOWNLOAD_TASKS.get(task_id)
|
||
if not task:
|
||
return error('任务不存在或已过期', 404)
|
||
if not _owned_by(task, user):
|
||
return error('无权取消该任务', 403)
|
||
if task.get('status') in ('done', 'error', 'canceled'):
|
||
return error(f"任务已结束({task.get('status')})", 400)
|
||
task['status'] = 'canceled'
|
||
task['msg'] = '任务已取消'
|
||
task['completed_at'] = time.time()
|
||
task['cancel'].set()
|
||
return success({'task_id': task_id, 'status': 'canceled'}, '已请求取消')
|
||
|
||
|
||
@router.api_route('/api/task/heartbeat', methods=['GET', 'POST'])
|
||
async def task_heartbeat(user: LoginUser, request: Request):
|
||
"""设备心跳:保持任务活跃;暂停中的任务自动恢复(重新进入排队)。"""
|
||
await ratelimit.guard(request, bucket='hb', limit=60, window=60)
|
||
data = await _request_data(request)
|
||
device_id = str(data.get('device_id') or '').strip()[:64]
|
||
if not device_id:
|
||
return error('缺少 device_id 参数')
|
||
services_tasks._touch_device(device_id)
|
||
resumed = 0
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
for task in DOWNLOAD_TASKS.values():
|
||
if (task.get('device_id') == device_id
|
||
and int(task.get('user_id') or 0) == int(user.id)
|
||
and task.get('status') == 'paused'):
|
||
task['status'] = 'queued'
|
||
task['msg'] = '恢复中...'
|
||
task['pause'].clear()
|
||
resumed += 1
|
||
now = time.time()
|
||
with DEVICE_LAST_SEEN_LOCK:
|
||
stale = [d for d, ts in DEVICE_LAST_SEEN.items() if now - ts > 86400]
|
||
for d in stale:
|
||
DEVICE_LAST_SEEN.pop(d, None)
|
||
return success({'resumed': resumed, 'online': True}, '心跳成功')
|
||
|
||
|
||
@router.api_route('/api/task/list', methods=['GET', 'POST'])
|
||
async def task_list(user: LoginUser, request: Request):
|
||
"""本设备未结束的任务列表(重新打开网站时恢复任务悬浮窗)。"""
|
||
data = await _request_data(request)
|
||
device_id = str(data.get('device_id') or '').strip()[:64]
|
||
if not device_id:
|
||
return error('缺少 device_id 参数')
|
||
services_tasks._touch_device(device_id)
|
||
my_tasks = []
|
||
with DOWNLOAD_TASKS_LOCK:
|
||
for tid, task in DOWNLOAD_TASKS.items():
|
||
params = task.get('params') or {}
|
||
if (task.get('device_id') == device_id
|
||
and int(task.get('user_id') or 0) == int(user.id)
|
||
and task.get('status') in ('queued', 'running', 'paused')):
|
||
my_tasks.append({
|
||
'task_id': tid,
|
||
'type': task.get('type', 'batch'),
|
||
'kind': ('podcast' if ('program_id' in params
|
||
or 'radio_name' in params
|
||
or params.get('radio_id'))
|
||
else 'music'),
|
||
'total': task.get('total', 0),
|
||
'done': task.get('done', 0),
|
||
'status': task.get('status', 'queued'),
|
||
'msg': task.get('msg', ''),
|
||
})
|
||
return success({'tasks': my_tasks}, '获取任务列表成功')
|