refactor: 拆分任务监控循环的 7 组周期作业(170 行 / 圈 58 → 17 行 / 圈 3)

为什么:_task_monitor_loop 是一个 170 行 / 圈复杂度 58 / 嵌套 8 层的单函数,把
7 组互不相关的周期性作业塞在一起——终态回收、超时取消、缓存清扫、限流桶清理、
账本对账、孤儿目录补扫、心跳离线——整轮共用一个 try/except。任何一处改动都要
通读全部 7 组逻辑;上一轮恢复它的启动点时,也正是这个体量让判读很吃力。

改法:一组作业一个具名函数,另抽 _run_monitor_tick 只负责编排顺序(循环本身
只剩「sleep + try + 限频记录」三件事)。

复杂度(同口径 AST 实测,重构前后):
  _task_monitor_loop   170 行 / 圈 58 / 嵌套 8  →  17 行 / 圈 3 / 嵌套 2
  文件内最大函数        170 行                  →  66 行(_reap_finished_tasks)
  文件内 >80 行函数      1 → 0  ·  嵌套 ≥5 层   1 → 0
  代价:总行数 231 → 291(+60,全部是新的函数签名与 docstring)

等价性证据(重构的分内之事,不是附赠):
  1. AST **数值字面量多重集**比对:旧 16 种 / 新 16 种,计数零差异——确认搬运
     中没有改任何阈值(4h 硬超时 / 8 次重试 / 60s / 24h / 5h / 90 天 / 600s …)。
  2. 剔除 docstring 后的**字符串字面量**比对:旧版有的一处不少——确认用户可见
     状态文案与日志格式没被抄错。
  3. 新增 20 例行为契约测试(tests/test_monitor_jobs.py),按「调用后世界里发生
     了什么」断言,不引用实现细节,故可跨后续重构复用。

刻意保留的语义(是决定,不是疏漏):
  - 单个 tick 仍共用一个 try/except:前段作业抛异常会跳过本轮剩余作业。改为
    逐作业隔离属于**行为变更**,本提交不夹带;已用 test_tick_exception_skips_
    rest_of_tick 如实钉住现状,让那次变更有据可依、不会悄悄发生。
  - _run_periodic_ledger_jobs / _sweep_orphan_task_dirs 的局部 try/except 原样
    保留(原注释即写明「下轮重试」意图),其兜底粒度各有独立用例。
  - 代结算的前提仍是「worker_thread 字段存在且线程已死」:没有线程句柄的任务
    不退款——拿不到句柄就无法判断 worker 是否已在别处结算过,贸然退会双退。
    该约束由新增用例固化(原实现如此,此前无测试记录)。

验证:ruff 全过 · pyright 0/0 · pytest 209 passed(189→209)·
生产 data/ 未被触碰(全部用例走 tmp_path 沙箱)。
This commit is contained in:
snowy-yang committed 2026-09-26 19:04:36 +08:00
1 parent 3de75f7479
commit 65a81af94a
2 files changed
+732 -164

No files matched your search

+464
View File
@@ -0,0 +1,464 @@
# -*- coding: utf-8 -*-
"""任务监控循环 7 组周期性作业的行为契约(沙箱,不触碰生产 data/)。
为什么:``_task_monitor_loop`` 拆分前是 170 行 / 圈复杂度 57 的单函数,7 组互不
相关的作业共用一个 try/except。这种「按作业拆函数」的改动最容易引入两类损伤:
1. **异常兜底粒度被改变**——某项作业失败后,本轮其余作业还跑不跑;
2. **锁内/锁外边界被挪动**——文件删除搬回锁内会重新阻塞全站任务端点
(原注释记载「GB 级任务目录删除曾阻塞锁数十秒」)。
本文件按「调用后世界里发生了什么」逐条钉住可观察行为,断言不引用实现细节,
故可跨重构复用。
账本口径同 tests/test_zip_workers.py:一律走 ``reserve_and_open`` 真预扣。
"""
import os
import threading
import time
from typing import Any
import pytest
import auth_system as auth
import core.db as core_db
from core.config import (
CANCEL_AFTER_SECONDS,
PAUSE_AFTER_SECONDS,
RESULT_RETENTION_SECONDS,
)
from core.state import (
DEVICE_LAST_SEEN,
DEVICE_LAST_SEEN_LOCK,
DOWNLOAD_TASKS,
DOWNLOAD_TASKS_LOCK,
)
from services import tasks as services_tasks
from services._shared import (
_LEDGER_LAST_CLEANUP,
_LEDGER_LAST_RECOVER,
_MONITOR_ERR_TS,
_TASKS_SWEEP_LAST,
)
from services.api_service import api_service
PW = 'Passw0rd12'
CREDITS = 50
# 与服务的硬上限/阈值一致(测试内不引用私有常量,避免「测试跟着实现改」)
_HARD_TIMEOUT = 4 * 3600
_TASKS_ORPHAN_AGE = 25 * 3600
_LEDGER_ORPHAN_MAX_AGE = 5 * 3600
_LEDGER_SETTLED_RETENTION = 90 * 86400
class _NullLogger:
"""吞掉 api_service.logger 输出,避免测试往项目根 music_api.log 追加。"""
def debug(self, *a: Any, **k: Any) -> None: ...
def info(self, *a: Any, **k: Any) -> None: ...
def warning(self, *a: Any, **k: Any) -> None: ...
def error(self, *a: Any, **k: Any) -> None: ...
@pytest.fixture()
def sandbox(tmp_path, monkeypatch):
"""临时站点库 + 临时下载目录 + 空任务表 + 复位监控节流全局。"""
db_path = tmp_path / 'site.db'
monkeypatch.setattr(auth.db, 'DATA_DIR', tmp_path)
monkeypatch.setattr(auth.db, 'DB_PATH', db_path)
monkeypatch.setattr(auth.db, 'BACKUP_DIR', tmp_path / 'backups')
monkeypatch.setattr(auth.db, 'SECRET_KEY_PATH', tmp_path / 'secret.key')
monkeypatch.setattr(core_db, 'DATA_DIR', tmp_path)
monkeypatch.setattr(core_db, 'DB_PATH', db_path)
auth.init_db()
auth.create_user('payer', PW, email='p@example.com', credits=CREDITS)
downloads = tmp_path / 'downloads'
downloads.mkdir()
monkeypatch.setattr(api_service, 'downloads_path', downloads)
monkeypatch.setattr(api_service, 'logger', _NullLogger())
# 监控节流全局是模块级 dict,逐用例复位——否则用例间互相影响
for state in (_LEDGER_LAST_RECOVER, _LEDGER_LAST_CLEANUP, _TASKS_SWEEP_LAST):
state['ts'] = 0.0
_MONITOR_ERR_TS['ts'] = 0.0
_MONITOR_ERR_TS['n'] = 0
with DOWNLOAD_TASKS_LOCK:
DOWNLOAD_TASKS.clear()
with DEVICE_LAST_SEEN_LOCK:
DEVICE_LAST_SEEN.clear()
yield {'root': tmp_path, 'downloads': downloads}
with DOWNLOAD_TASKS_LOCK:
DOWNLOAD_TASKS.clear()
with DEVICE_LAST_SEEN_LOCK:
DEVICE_LAST_SEEN.clear()
def _add_task(tid: str, **over: Any) -> dict[str, Any]:
"""登记一条任务记录(不预扣——监控作业只读单条任务的内部字段)。"""
task: dict[str, Any] = {
'type': 'batch', 'task_id': tid, 'status': 'queued',
'start': time.time(), 'msg': '排队中...',
'cancel': threading.Event(), 'pause': threading.Event(),
'device_id': 'dev-test', 'result': None,
}
task.update(over)
with DOWNLOAD_TASKS_LOCK:
DOWNLOAD_TASKS[tid] = task
return task
def _has(tid: str) -> bool:
with DOWNLOAD_TASKS_LOCK:
return tid in DOWNLOAD_TASKS
# ============ ① 终态任务回收 ============
def test_reap_removes_expired_task_and_files(sandbox):
"""超过保留期:记录、交付文件与任务目录一并回收。"""
downloads = sandbox['downloads']
delivered = downloads / 'out.zip'
delivered.write_bytes(b'z')
task_dir = downloads / 'tasks' / 't-done'
task_dir.mkdir(parents=True)
(task_dir / 'inner.mp3').write_bytes(b'a')
_add_task('t-done', status='done',
completed_at=time.time() - RESULT_RETENTION_SECONDS - 60,
result={'zip_path': str(delivered), 'file_path': str(task_dir / 'inner.mp3')})
services_tasks._reap_finished_tasks(time.time())
assert not _has('t-done'), '过期终态任务记录应被移除'
assert not delivered.exists(), '交付文件应被删除'
assert not task_dir.exists(), '任务目录应被递归删除'
def test_reap_keeps_task_within_retention(sandbox):
"""未到保留期不动(保留期从 completed_at 起算)。"""
_add_task('t-fresh', status='done', completed_at=time.time())
services_tasks._reap_finished_tasks(time.time())
assert _has('t-fresh')
def test_reap_keeps_legacy_record_without_completed_at(sandbox):
"""无 completed_at 的旧记录回退到 start 计算(兼容既有在途记录)。"""
_add_task('t-legacy', status='done', start=time.time(), completed_at=None)
services_tasks._reap_finished_tasks(time.time())
assert _has('t-legacy'), 'start 很新 → 不应被回收'
def test_reap_ignores_non_terminal_tasks(sandbox):
"""未终态任务不归它管(哪怕 start 很久——那是超时取消的职责)。"""
_add_task('t-run', status='running', start=time.time() - 10 * _HARD_TIMEOUT)
services_tasks._reap_finished_tasks(time.time())
assert _has('t-run')
def test_reap_retries_then_gives_up(sandbox):
"""交付文件删不掉:记 _cleanup_tries 保留记录下轮重试,达上限后放弃记录。"""
# 用**目录**冒充交付文件:os.remove 必然抛 OSError
stuck = sandbox['downloads'] / 'stuck-dir'
stuck.mkdir()
_add_task('t-stuck', status='done',
completed_at=time.time() - RESULT_RETENTION_SECONDS - 60,
result={'file_path': str(stuck)})
now = time.time()
services_tasks._reap_finished_tasks(now)
assert _has('t-stuck'), '删除失败应保留记录待下轮重试'
with DOWNLOAD_TASKS_LOCK:
assert DOWNLOAD_TASKS['t-stuck']['_cleanup_tries'] == 1
for _ in range(services_tasks._CLEANUP_MAX_TRIES):
services_tasks._reap_finished_tasks(now)
assert not _has('t-stuck'), '连续失败达上限后应放弃记录(文件留给缓存清扫兜底)'
# ============ ② 超时任务取消 ============
def _finished_thread() -> threading.Thread:
"""已结束的线程替身——对应生产里 worker 崩溃/退出的情形(is_alive() 为 False)。"""
th = threading.Thread(target=lambda: None)
th.start()
th.join()
return th
def test_cancel_overtime_settles_dead_worker(sandbox, monkeypatch):
"""超硬上限且 worker 线程已死:置取消、唤醒 cancel Event、代为退款。"""
settled: list[tuple] = []
monkeypatch.setattr(auth, 'ledger_settle',
lambda tid, ok, st: settled.append((tid, ok, st)))
task = _add_task('t-slow', status='running',
start=time.time() - _HARD_TIMEOUT - 60,
worker_thread=_finished_thread())
services_tasks._cancel_overtime_tasks(time.time())
assert task['status'] == 'canceled'
assert task['cancel'].is_set(), '须唤醒 cancel Event 让阻塞中的 worker 退出'
assert settled == [('t-slow', 0, 'timeout')]
def test_cancel_overtime_without_worker_thread_does_not_settle(sandbox, monkeypatch):
"""无 worker_thread 字段的任务不代结算——代结算的前提是「线程曾存在且已死」,
没有线程句柄就无法判断 worker 是否已在别处结算过,贸然退款会双退。"""
settled: list[tuple] = []
monkeypatch.setattr(auth, 'ledger_settle',
lambda tid, ok, st: settled.append((tid, ok, st)))
task = _add_task('t-noth', status='queued',
start=time.time() - _HARD_TIMEOUT - 60)
services_tasks._cancel_overtime_tasks(time.time())
assert task['status'] == 'canceled'
assert settled == []
def test_cancel_overtime_skips_live_worker(sandbox, monkeypatch):
"""worker 仍在跑:只置取消,不代结算(交 worker 自己收敛,避免双退)。"""
settled: list[tuple] = []
monkeypatch.setattr(auth, 'ledger_settle',
lambda tid, ok, st: settled.append((tid, ok, st)))
stop = threading.Event()
alive = threading.Thread(target=stop.wait, daemon=True)
alive.start()
try:
task = _add_task('t-live', status='running',
start=time.time() - _HARD_TIMEOUT - 60, worker_thread=alive)
services_tasks._cancel_overtime_tasks(time.time())
assert task['status'] == 'canceled'
assert settled == [], 'worker 活着 → 不应代结算'
finally:
stop.set()
alive.join(timeout=5)
def test_cancel_overtime_ignores_recent(sandbox):
"""未超硬上限的任务不动。"""
task = _add_task('t-new', status='running', start=time.time() - 60)
services_tasks._cancel_overtime_tasks(time.time())
assert task['status'] == 'running'
# ============ ⑤ 账本周期兜底 ============
def test_ledger_jobs_respect_intervals(sandbox, monkeypatch):
"""时间戳门槛:到期执行一次并推进门槛,未到期不重复执行。"""
calls: list[str] = []
monkeypatch.setattr(auth, 'ledger_recover_orphans',
lambda max_age_seconds=0: calls.append(f'recover:{max_age_seconds}'))
monkeypatch.setattr(auth, 'ledger_cleanup_settled',
lambda days: calls.append(f'cleanup:{days}'))
now = time.time()
services_tasks._run_periodic_ledger_jobs(now)
assert calls == [f'recover:{_LEDGER_ORPHAN_MAX_AGE}',
f'cleanup:{_LEDGER_SETTLED_RETENTION}']
calls.clear()
services_tasks._run_periodic_ledger_jobs(now + 1)
assert calls == [], '门槛已推进,同一周期内不应重复执行'
calls.clear()
services_tasks._run_periodic_ledger_jobs(now + 61)
assert calls == [f'recover:{_LEDGER_ORPHAN_MAX_AGE}'], '仅到期的那个触发'
def test_ledger_jobs_swallow_exception(sandbox, monkeypatch):
"""账本作业抛错由自身吞掉(不能让本轮其余作业一起落空)。"""
def boom(max_age_seconds=0):
raise RuntimeError('库忙')
monkeypatch.setattr(auth, 'ledger_recover_orphans', boom)
services_tasks._run_periodic_ledger_jobs(time.time()) # 不抛即通过
# ============ ⑥ tasks/ 孤儿目录补扫 ============
def test_sweep_orphans_removes_stale_only(sandbox):
""">24h 且无任务记录的目录被回收;在册任务目录与新鲜目录保留。"""
tasks_root = sandbox['downloads'] / 'tasks'
tasks_root.mkdir(parents=True)
orphan = tasks_root / 'orphan'
orphan.mkdir()
in_registry = tasks_root / 'in-registry'
in_registry.mkdir()
fresh = tasks_root / 'fresh'
fresh.mkdir()
old = time.time() - _TASKS_ORPHAN_AGE
os.utime(orphan, (old, old))
os.utime(in_registry, (old, old))
_add_task('in-registry', status='running')
services_tasks._sweep_orphan_task_dirs(time.time())
assert not orphan.exists(), '无记录的陈旧目录应被回收'
assert in_registry.exists(), '在册任务目录不得被补扫误删'
assert fresh.exists(), '未到年龄的目录保留'
def test_sweep_orphans_honours_interval(sandbox):
"""未到补扫周期的调用直接跳过。"""
tasks_root = sandbox['downloads'] / 'tasks'
tasks_root.mkdir(parents=True)
first = tasks_root / 'orphan-a'
first.mkdir()
old = time.time() - _TASKS_ORPHAN_AGE
os.utime(first, (old, old))
now = time.time()
services_tasks._sweep_orphan_task_dirs(now)
assert not first.exists(), '首次到期应执行'
second = tasks_root / 'orphan-b'
second.mkdir()
os.utime(second, (old, old))
services_tasks._sweep_orphan_task_dirs(now + 1)
assert second.exists(), '门槛未到 → 本轮不应补扫'
# ============ ⑦ 心跳离线处理 ============
def test_offline_device_cancels_after_threshold(sandbox):
"""离线超 30 分钟:其未完成任务置取消并唤醒 cancel Event。"""
now = time.time()
with DEVICE_LAST_SEEN_LOCK:
DEVICE_LAST_SEEN['dev-off'] = now - CANCEL_AFTER_SECONDS - 60
task = _add_task('t-off', status='running', device_id='dev-off')
services_tasks._handle_offline_devices(now)
assert task['status'] == 'canceled'
assert task['cancel'].is_set()
assert '30 分钟' in task['msg']
def test_offline_device_pauses_batch_before_threshold(sandbox):
"""离线刚过暂停阈值:批量任务先暂停,让出排队名额(「断线不占坑」)。"""
now = time.time()
with DEVICE_LAST_SEEN_LOCK:
DEVICE_LAST_SEEN['dev-soft'] = now - PAUSE_AFTER_SECONDS - 1
task = _add_task('t-soft', status='running', type='batch', device_id='dev-soft')
services_tasks._handle_offline_devices(now)
assert task['status'] == 'paused'
assert task['pause'].is_set()
assert not task['cancel'].is_set(), '未到取消阈值不应发取消'
def test_offline_handler_ignores_online_device(sandbox):
"""心跳新鲜的设备:其任务不受影响。"""
now = time.time()
with DEVICE_LAST_SEEN_LOCK:
DEVICE_LAST_SEEN['dev-on'] = now
task = _add_task('t-on', status='running', device_id='dev-on')
services_tasks._handle_offline_devices(now)
assert task['status'] == 'running'
# ============ 异常限频 ============
def test_log_monitor_error_rate_limits(sandbox, monkeypatch):
"""限频:窗口内多次异常只记一条(不静默也不刷屏)。"""
events: list[str] = []
class _Recorder:
def warning(self, msg: str, *a: Any, **k: Any) -> None:
events.append(msg)
monkeypatch.setattr(api_service, 'logger', _Recorder())
for _ in range(3):
services_tasks._log_monitor_error()
assert len(events) == 1, '同一窗口内只应记录一条'
_MONITOR_ERR_TS['ts'] = time.time() - services_tasks._MONITOR_ERR_WINDOW - 1
services_tasks._log_monitor_error()
assert len(events) == 2, '窗口过期后应重新放行'
# ============ 单轮编排与兜底粒度 ============
_JOBS = (
'_reap_finished_tasks', '_cancel_overtime_tasks', '_sweep_download_cache',
'_sweep_rate_buckets', '_run_periodic_ledger_jobs', '_sweep_orphan_task_dirs',
'_handle_offline_devices',
)
def test_tick_runs_all_jobs_in_order(monkeypatch):
"""单轮按固定顺序执行全部 7 步——顺序即优先级,重构不得打乱。"""
order: list[str] = []
for name in _JOBS:
monkeypatch.setattr(
services_tasks, name, (lambda n: lambda *a, **k: order.append(n))(name))
services_tasks._run_monitor_tick(time.time())
assert order == list(_JOBS)
def test_tick_exception_skips_rest_of_tick(monkeypatch):
"""前段作业抛异常 → 本轮剩余作业跳过。
如实钉住拆分前的粒度:整个 tick 共用一个 try/except,而非逐作业隔离。
(改为逐作业隔离属行为变更,需单独评估——本测试存在的意义正是让那次
变更有据可依、不会悄悄发生。)
"""
called: list[str] = []
def boom(_now: float) -> None:
raise RuntimeError('reap 炸了')
monkeypatch.setattr(services_tasks, '_reap_finished_tasks', boom)
monkeypatch.setattr(services_tasks, '_handle_offline_devices',
lambda _now: called.append('offline'))
with pytest.raises(RuntimeError):
services_tasks._run_monitor_tick(time.time())
assert called == [], '本轮后续作业随异常一起中断(保持原语义)'
class _StopLoop(Exception):
"""从假 sleep 里抛出,把无限循环的单轮执行切断用于断言。"""
def test_loop_survives_tick_exception(monkeypatch):
"""循环本身不退出:单轮异常被限频记录后继续跑下一轮。"""
state = {'sleeps': 0, 'logged': 0}
class _FakeTime:
@staticmethod
def sleep(_seconds: float) -> None:
state['sleeps'] += 1
if state['sleeps'] >= 2:
raise _StopLoop
@staticmethod
def time() -> float:
return time.time()
def boom(_now: float) -> None:
raise RuntimeError('tick 炸了')
monkeypatch.setattr(services_tasks, 'time', _FakeTime)
monkeypatch.setattr(services_tasks, '_run_monitor_tick', boom)
monkeypatch.setattr(services_tasks, '_log_monitor_error',
lambda: state.__setitem__('logged', state['logged'] + 1))
with pytest.raises(_StopLoop):
services_tasks._task_monitor_loop()
assert state['sleeps'] == 2, '第 1 轮异常后应继续到第 2 轮'
assert state['logged'] == 1, '每轮异常记一次'