This repository has been archived on 2026-10-02. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
NeteaseUrl/web/routers/tasks.py
T
snowy-yang 69a50c2c31 refactor: worker 子系统全面异步化,拆掉同步转异步的桥接层
拆掉「同步调异步 / 异步桥同步」两类垫片的第二半(HTTP 线 B2 见前序提交)。
worker 侧此前是「起线程跑同步函数、函数体内部再用 asyncio.to_thread 把活塞给
另一条循环」——线程与协程互等,取消/暂停只能靠轮询。本轮把 worker 自身变成
协程,由 _start_task 直接挂到应用事件循环上,桥接层随之失去存在理由。

改动(按子系统):
- music_downloader:downloader / _aria2 / _concurrency / cover / _http /
  __main__ 全链路协程化。per-file 锁改 asyncio.Lock;计数闸改
  「active 计数 + await asyncio.sleep(0.5)」的可取消等待(原阻塞式 with 会让
  已取消任务仍要等全局名额腾出,卡死并拖后腿退款);aria2p 的同步 RPC 仍经
  to_thread,轮询改 asyncio.sleep;新增 core.http_client.iter_chunks,把**流式
  读取期**的 aiohttp.ClientError 收敛成 UpstreamNetworkError(请求建立阶段早已
  有这条路径,读取阶段此前没有,断流会绕过调用方的 UpstreamError 分类)。
  删除 get_http_session / _SESSION_LOCAL。
  models._RetryFromScratch 不再继承 requests.RequestException 而继承 Exception:
  它是「续传起点失效、残片已清、从头重下」,既不是确定性失败
  (DownloadException),也不是残片已保留的网络失败(UpstreamError),必须独立
  成类且在 except 链中先于二者捕获。
- music_api:podcast.py 全部函数协程化(直链解析内部改
  await NeteaseAPI.get_song_url);compat.py 删除——16 个死掉的同步门面早已
  清空,文件只剩空壳。
- services:_taskcore / _zipcore / batch / downloads / podcast / tasks 协程化。
  threading.Thread → loop.create_task(任务句柄字段 worker_thread →
  worker_task),ThreadPoolExecutor → asyncio.gather + asyncio.Semaphore,
  终态函数与结算回调全部改 await;监控线程里「worker 已死」判定改 task.done()。
  删除 services/streaming.py(同步流垫片,唯一消费者已迁到
  core.http_client.open)。zipfile 实例仍非线程安全,故「下载」并行、
  「入包」严格串行。
- web:路由层 130+ 处 to_thread 拆除——被包住的目标自己已是协程,再包一层只是
  把协程丢进线程里跑一圈。tasks_registry.start_task 直接 create_task。
  web/app.py 关闭段去掉 async_bridge.shutdown()(不再有第二条约循环)。
  web/routers/download.py 的播客端点补上 3 处漏掉的 await
  (extract_podcast_ref / get_radio_detail)——漏掉时不报错、只静默返回协程对象,
  靠 pyright 抓出来后修正。

留在 to_thread 上的都是**正当的阻塞卸载**:同步 SQLite 落库与账本结算、
mutagen 全量重写标签、zipfile 整段拷贝、shutil.rmtree、aria2p 的同步 RPC、
PBKDF2 的 10 万次迭代。ruff 对残留的本地元数据操作(exists/stat/os.replace/
往 .part 顺序写分块)按「微秒级、卸载的调度开销大于操作本身」放行,pyproject
逐条注明理由。

取消/暂停仍由 threading.Event 承载:置位方是请求线程/监控线程,读取方是 worker
协程,只有「置位」与「轮询读取」两种操作,既不需要跨循环唤醒也不会卡住事件
循环——比 asyncio.Event 更省心(ASYNC110 同理按此豁免)。

测试:10 个文件的替身边界同步迁移——网络替身从 core.sync_http 的
get_http_session 换成 downloader/服务模块的 _http.open + **真实** iter_chunks
(顺带钉住读取期异常收敛);`_run` 改为在独立线程里 asyncio.run(worker(...));
aria2 的假时钟从 time.sleep 换成 _POLL_INTERVAL=0;monitor 的线程句柄替身换成
done() 可控的轻量替身(asyncio.Task 的构造需要运行中的事件循环,而监控循环跑在
常驻线程里);podcast / 封面 / 结算包装等替身一律改 async。

门禁:ruff 全过 / pyright 0-0 / pytest 383 项全绿(0 失败)。

下一步 B2:auth_system 12 个模块改 aiosqlite 背板、web 层剩余 to_thread 收尾;
B3:删 core/sync_http.py、core/async_bridge.py、core/http_errors.py(现仅
sync_http 自用)。
2026-09-28 19:36:56 +08:00

222 lines
8.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# -*- coding: utf-8 -*-
"""任务状态端点。
端点:
- POST/GET /api/v1/topen ↔ Flask /api/topen(请求统计,无鉴权)
- POST/GET /api/v1/queue ↔ Flask /api/queue(排队状态,无鉴权)
- POST/GET /api/v1/batch_progress ↔ Flask /api/batch_progress(需登录,仅本人任务)
- POST /api/v1/task/cancel ↔ Flask /api/task/cancel(需登录,仅本人任务)
- POST/GET /api/v1/task/heartbeat ↔ Flask /api/task/heartbeat(需登录)
- POST/GET /api/v1/task/list ↔ Flask /api/task/list(需登录)
这些端点读写 core.state 的共享任务表(与任务执行侧同一批对象),因此
看到的是同一份任务状态。
"""
import time
from typing import Annotated, Any
from fastapi import APIRouter, Depends, Request
from core.state import (
BATCH_DURATIONS,
BATCH_DURATIONS_LOCK,
DEVICE_LAST_SEEN,
DEVICE_LAST_SEEN_LOCK,
DOWNLOAD_TASKS,
DOWNLOAD_TASKS_LOCK,
REQUEST_STATS,
STATS_LOCK,
)
from services import tasks as services_tasks
from web import ratelimit
from web.deps import SiteUser, require_login
from web.request_utils import (
request_data as _request_data,
)
from web.response import error, success
router = APIRouter()
LoginUser = Annotated[SiteUser, Depends(require_login)]
def _owned_by(task: dict[str, Any], user: SiteUser) -> bool:
return int(task.get('user_id') or 0) == int(user.id)
def _semaphores():
"""批量打包/下载信号量(懒取,与 core.state 中的同一对象共享计数)。"""
from services._shared import BATCH_ZIP_SEM
from services.api_service import api_service
return BATCH_ZIP_SEM, api_service.downloader._download_semaphore
@router.api_route('/api/topen', methods=['GET', 'POST'])
async def topen(request: Request):
"""接口请求统计(实时)——锁内快照,避免 total 与明细不一致。"""
from services._shared import ENDPOINT_NAMES
with STATS_LOCK:
snapshot = {p: dict(v) for p, v in REQUEST_STATS.items()}
endpoints = []
total = sum(v['count'] for v in snapshot.values())
for path, info in snapshot.items():
endpoints.append({
'path': path,
'name': ENDPOINT_NAMES.get(path, path),
'count': info['count'],
'percent': round(info['count'] / total * 100, 1) if total else 0,
'last': info['last'],
})
endpoints.sort(key=lambda x: x['count'], reverse=True)
return success({
'total': total,
'endpoint_count': len(endpoints),
'max_count': endpoints[0]['count'] if endpoints else 0,
'endpoints': endpoints,
}, '获取请求统计成功')
@router.api_route('/api/queue', methods=['GET', 'POST'])
async def queue(request: Request):
"""实时排队状态(排队人数 + 预计等待时间)。"""
batch_sem, dl_sem = _semaphores()
avg_batch = None
with BATCH_DURATIONS_LOCK:
if BATCH_DURATIONS:
avg_batch = sum(BATCH_DURATIONS) / len(BATCH_DURATIONS)
return success({
'batch_capacity': batch_sem.capacity,
'batch_active': batch_sem.active,
'batch_waiting': batch_sem.waiting,
'download_capacity': dl_sem.capacity,
'download_active': dl_sem.active,
'download_waiting': dl_sem.waiting,
'avg_batch_seconds': round(avg_batch, 1) if avg_batch else None,
}, '获取排队状态成功')
@router.api_route('/api/batch_progress', methods=['GET', 'POST'])
async def batch_progress(user: LoginUser, request: Request):
"""下载任务进度(已处理 X/Y + 百分比 + 预计剩余时间)。"""
data = await _request_data(request)
task_id = str(data.get('task_id') or '').strip()
if not task_id:
return error('缺少 task_id 参数')
now = time.time()
with DOWNLOAD_TASKS_LOCK:
task = DOWNLOAD_TASKS.get(task_id)
if not task:
return error('任务不存在或已过期', 404)
if not _owned_by(task, user):
return error('无权查询该任务', 403)
total = task.get('total', 0)
done = task.get('done', 0)
live = task.get('live_percent')
if task.get('status') == 'running' and isinstance(live, (int, float)):
percent = round(min(99.0, max(0.0, float(live))), 1)
elif total:
percent = round(done / total * 100, 1)
else:
percent = 100.0 if task.get('status') in ('done',) else 0.0
elapsed = now - task.get('start', now)
eta = None
if task.get('status') == 'running' and done > 0 and elapsed > 0 and total:
rate = done / elapsed
if rate > 0:
eta = round((total - done) / rate, 1)
result = task.get('result') or {}
has_result = bool(result.get('zip_path') or result.get('file_path'))
return success({
'total': total,
'done': done,
'failed': task.get('failed', 0),
'percent': percent,
'status': task.get('status', 'running'),
'elapsed': round(elapsed, 1),
'eta': eta,
'msg': task.get('msg', ''),
'refunded': task.get('refunded', 0),
'has_result': has_result,
}, '获取任务进度成功')
@router.post('/api/task/cancel')
async def task_cancel(user: LoginUser, request: Request):
"""取消下载任务(仅任务本人的站点账号可取消)。"""
data = await _request_data(request)
task_id = str(data.get('task_id') or '').strip()
if not task_id:
return error('缺少 task_id 参数')
with DOWNLOAD_TASKS_LOCK:
task = DOWNLOAD_TASKS.get(task_id)
if not task:
return error('任务不存在或已过期', 404)
if not _owned_by(task, user):
return error('无权取消该任务', 403)
if task.get('status') in ('done', 'error', 'canceled'):
return error(f"任务已结束({task.get('status')})", 400)
task['status'] = 'canceled'
task['msg'] = '任务已取消'
task['completed_at'] = time.time()
task['cancel'].set()
return success({'task_id': task_id, 'status': 'canceled'}, '已请求取消')
@router.api_route('/api/task/heartbeat', methods=['GET', 'POST'])
async def task_heartbeat(user: LoginUser, request: Request):
"""设备心跳:保持任务活跃;暂停中的任务自动恢复(重新进入排队)。"""
await ratelimit.guard(request, bucket='hb', limit=60, window=60)
data = await _request_data(request)
device_id = str(data.get('device_id') or '').strip()[:64]
if not device_id:
return error('缺少 device_id 参数')
services_tasks._touch_device(device_id)
resumed = 0
with DOWNLOAD_TASKS_LOCK:
for task in DOWNLOAD_TASKS.values():
if (task.get('device_id') == device_id
and int(task.get('user_id') or 0) == int(user.id)
and task.get('status') == 'paused'):
task['status'] = 'queued'
task['msg'] = '恢复中...'
task['pause'].clear()
resumed += 1
now = time.time()
with DEVICE_LAST_SEEN_LOCK:
stale = [d for d, ts in DEVICE_LAST_SEEN.items() if now - ts > 86400]
for d in stale:
DEVICE_LAST_SEEN.pop(d, None)
return success({'resumed': resumed, 'online': True}, '心跳成功')
@router.api_route('/api/task/list', methods=['GET', 'POST'])
async def task_list(user: LoginUser, request: Request):
"""本设备未结束的任务列表(重新打开网站时恢复任务悬浮窗)。"""
data = await _request_data(request)
device_id = str(data.get('device_id') or '').strip()[:64]
if not device_id:
return error('缺少 device_id 参数')
services_tasks._touch_device(device_id)
my_tasks = []
with DOWNLOAD_TASKS_LOCK:
for tid, task in DOWNLOAD_TASKS.items():
params = task.get('params') or {}
if (task.get('device_id') == device_id
and int(task.get('user_id') or 0) == int(user.id)
and task.get('status') in ('queued', 'running', 'paused')):
my_tasks.append({
'task_id': tid,
'type': task.get('type', 'batch'),
'kind': ('podcast' if ('program_id' in params
or 'radio_name' in params
or params.get('radio_id'))
else 'music'),
'total': task.get('total', 0),
'done': task.get('done', 0),
'status': task.get('status', 'queued'),
'msg': task.get('msg', ''),
})
return success({'tasks': my_tasks}, '获取任务列表成功')