NeteaseUrl/tests/test_monitor_jobs.py

# -*- coding: utf-8 -*-
"""任务监控循环 7 组周期性作业的行为契约（沙箱，不触碰生产 data/）。

为什么：``_task_monitor_loop`` 拆分前是 170 行 / 圈复杂度 57 的单函数，7 组互不
相关的作业共用一个 try/except。这种「按作业拆函数」的改动最容易引入两类损伤：

  1. **异常兜底粒度被改变**——某项作业失败后，本轮其余作业还跑不跑；
  2. **锁内/锁外边界被挪动**——文件删除搬回锁内会重新阻塞全站任务端点
     （原注释记载「GB 级任务目录删除曾阻塞锁数十秒」）。

本文件按「调用后世界里发生了什么」逐条钉住可观察行为，断言不引用实现细节，
故可跨重构复用。

账本口径同 tests/test_zip_workers.py：一律走 ``reserve_and_open`` 真预扣。
"""
import asyncio
import os
import threading
import time
from typing import Any

import pytest

import auth_system as auth
import core.db as core_db
from core.config import (
    CANCEL_AFTER_SECONDS,
    PAUSE_AFTER_SECONDS,
    RESULT_RETENTION_SECONDS,
)
from core.state import (
    DEVICE_LAST_SEEN,
    DEVICE_LAST_SEEN_LOCK,
    DOWNLOAD_TASKS,
    DOWNLOAD_TASKS_LOCK,
)
from services import tasks as services_tasks
from services._shared import (
    _LEDGER_LAST_CLEANUP,
    _LEDGER_LAST_RECOVER,
    _MONITOR_ERR_TS,
    _TASKS_SWEEP_LAST,
)
from services.api_service import api_service

PW = 'Passw0rd12'
CREDITS = 50

# 与服务的硬上限/阈值一致（测试内不引用私有常量，避免「测试跟着实现改」）
_HARD_TIMEOUT = 4 * 3600
_TASKS_ORPHAN_AGE = 25 * 3600
_LEDGER_ORPHAN_MAX_AGE = 5 * 3600
_LEDGER_SETTLED_RETENTION = 90 * 86400


class _NullLogger:
    """吞掉 api_service.logger 输出，避免测试往项目根 music_api.log 追加。"""

    def debug(self, *a: Any, **k: Any) -> None: ...
    def info(self, *a: Any, **k: Any) -> None: ...
    def warning(self, *a: Any, **k: Any) -> None: ...
    def error(self, *a: Any, **k: Any) -> None: ...


@pytest.fixture()
def sandbox(tmp_path, monkeypatch):
    """临时站点库 + 临时下载目录 + 空任务表 + 复位监控节流全局。"""
    db_path = tmp_path / 'site.db'
    monkeypatch.setattr(auth.db, 'DATA_DIR', tmp_path)
    monkeypatch.setattr(auth.db, 'DB_PATH', db_path)
    monkeypatch.setattr(auth.db, 'BACKUP_DIR', tmp_path / 'backups')
    monkeypatch.setattr(auth.db, 'SECRET_KEY_PATH', tmp_path / 'secret.key')
    monkeypatch.setattr(core_db, 'DATA_DIR', tmp_path)
    monkeypatch.setattr(core_db, 'DB_PATH', db_path)
    asyncio.run(auth.init_db())
    asyncio.run(auth.create_user('payer', PW, email='p@example.com', credits=CREDITS))

    downloads = tmp_path / 'downloads'
    downloads.mkdir()
    monkeypatch.setattr(api_service, 'downloads_path', downloads)
    monkeypatch.setattr(api_service, 'logger', _NullLogger())

    # 监控节流全局是模块级 dict，逐用例复位——否则用例间互相影响
    for state in (_LEDGER_LAST_RECOVER, _LEDGER_LAST_CLEANUP, _TASKS_SWEEP_LAST):
        state['ts'] = 0.0
    _MONITOR_ERR_TS['ts'] = 0.0
    _MONITOR_ERR_TS['n'] = 0

    with DOWNLOAD_TASKS_LOCK:
        DOWNLOAD_TASKS.clear()
    with DEVICE_LAST_SEEN_LOCK:
        DEVICE_LAST_SEEN.clear()
    yield {'root': tmp_path, 'downloads': downloads}
    with DOWNLOAD_TASKS_LOCK:
        DOWNLOAD_TASKS.clear()
    with DEVICE_LAST_SEEN_LOCK:
        DEVICE_LAST_SEEN.clear()


def _add_task(tid: str, **over: Any) -> dict[str, Any]:
    """登记一条任务记录（不预扣——监控作业只读单条任务的内部字段）。"""
    task: dict[str, Any] = {
        'type': 'batch', 'task_id': tid, 'status': 'queued',
        'start': time.time(), 'msg': '排队中...',
        'cancel': threading.Event(), 'pause': threading.Event(),
        'device_id': 'dev-test', 'result': None,
    }
    task.update(over)
    with DOWNLOAD_TASKS_LOCK:
        DOWNLOAD_TASKS[tid] = task
    return task


def _has(tid: str) -> bool:
    with DOWNLOAD_TASKS_LOCK:
        return tid in DOWNLOAD_TASKS


# ============ ① 终态任务回收 ============

def test_reap_removes_expired_task_and_files(sandbox):
    """超过保留期：记录、交付文件与任务目录一并回收。"""
    downloads = sandbox['downloads']
    delivered = downloads / 'out.zip'
    delivered.write_bytes(b'z')
    task_dir = downloads / 'tasks' / 't-done'
    task_dir.mkdir(parents=True)
    (task_dir / 'inner.mp3').write_bytes(b'a')

    _add_task('t-done', status='done',
              completed_at=time.time() - RESULT_RETENTION_SECONDS - 60,
              result={'zip_path': str(delivered), 'file_path': str(task_dir / 'inner.mp3')})

    services_tasks._reap_finished_tasks(time.time())

    assert not _has('t-done'), '过期终态任务记录应被移除'
    assert not delivered.exists(), '交付文件应被删除'
    assert not task_dir.exists(), '任务目录应被递归删除'


def test_reap_keeps_task_within_retention(sandbox):
    """未到保留期不动（保留期从 completed_at 起算）。"""
    _add_task('t-fresh', status='done', completed_at=time.time())
    services_tasks._reap_finished_tasks(time.time())
    assert _has('t-fresh')


def test_reap_keeps_legacy_record_without_completed_at(sandbox):
    """无 completed_at 的旧记录回退到 start 计算（兼容既有在途记录）。"""
    _add_task('t-legacy', status='done', start=time.time(), completed_at=None)
    services_tasks._reap_finished_tasks(time.time())
    assert _has('t-legacy'), 'start 很新 → 不应被回收'


def test_reap_ignores_non_terminal_tasks(sandbox):
    """未终态任务不归它管（哪怕 start 很久——那是超时取消的职责）。"""
    _add_task('t-run', status='running', start=time.time() - 10 * _HARD_TIMEOUT)
    services_tasks._reap_finished_tasks(time.time())
    assert _has('t-run')


def test_reap_retries_then_gives_up(sandbox):
    """交付文件删不掉：记 _cleanup_tries 保留记录下轮重试，达上限后放弃记录。"""
    # 用**目录**冒充交付文件：os.remove 必然抛 OSError
    stuck = sandbox['downloads'] / 'stuck-dir'
    stuck.mkdir()
    _add_task('t-stuck', status='done',
              completed_at=time.time() - RESULT_RETENTION_SECONDS - 60,
              result={'file_path': str(stuck)})

    now = time.time()
    services_tasks._reap_finished_tasks(now)
    assert _has('t-stuck'), '删除失败应保留记录待下轮重试'
    with DOWNLOAD_TASKS_LOCK:
        assert DOWNLOAD_TASKS['t-stuck']['_cleanup_tries'] == 1

    for _ in range(services_tasks._CLEANUP_MAX_TRIES):
        services_tasks._reap_finished_tasks(now)
    assert not _has('t-stuck'), '连续失败达上限后应放弃记录（文件留给缓存清扫兜底）'


# ============ ② 超时任务取消 ============

class _WorkerHandle:
    """worker 句柄替身——生产里是 asyncio.Task，监控只看它的 ``done()``。

    这里用轻量替身而不是真造一个已结束的 Task：Task 的构造依赖运行中的事件循环，
    而监控循环跑在常驻线程里、与任务所在的循环无关；本文件测的是「句柄状态 →
    是否代结算」这段判断，只要 done() 可控即可。
    """

    def __init__(self, done: bool) -> None:
        self._done = done

    def done(self) -> bool:
        return self._done


def _finished_worker() -> _WorkerHandle:
    """已结束的 worker 替身——对应生产里 worker 崩溃/退出的情形（done() 为 True）。"""
    return _WorkerHandle(True)


def _live_worker() -> _WorkerHandle:
    return _WorkerHandle(False)


def test_cancel_overtime_settles_dead_worker(sandbox, monkeypatch):
    """超硬上限且 worker 已结束：置取消、唤醒 cancel Event、代为退款。"""
    settled: list[tuple] = []
    async def _settle(tid, ok, st):
        settled.append((tid, ok, st))

    monkeypatch.setattr(auth, 'ledger_settle', _settle)

    task = _add_task('t-slow', status='running',
                     start=time.time() - _HARD_TIMEOUT - 60,
                     worker_task=_finished_worker())
    services_tasks._cancel_overtime_tasks(time.time())

    assert task['status'] == 'canceled'
    assert task['cancel'].is_set(), '须唤醒 cancel Event 让阻塞中的 worker 退出'
    assert settled == [('t-slow', 0, 'timeout')]


def test_cancel_overtime_without_worker_task_does_not_settle(sandbox, monkeypatch):
    """无 worker_task 字段的任务不代结算——代结算的前提是「句柄曾存在且已结束」，
    没有句柄就无法判断 worker 是否已在别处结算过，贸然退款会双退。"""
    settled: list[tuple] = []
    async def _settle(tid, ok, st):
        settled.append((tid, ok, st))

    monkeypatch.setattr(auth, 'ledger_settle', _settle)

    task = _add_task('t-noth', status='queued',
                     start=time.time() - _HARD_TIMEOUT - 60)
    services_tasks._cancel_overtime_tasks(time.time())

    assert task['status'] == 'canceled'
    assert settled == []


def test_cancel_overtime_skips_live_worker(sandbox, monkeypatch):
    """worker 仍在跑：只置取消，不代结算（交 worker 自己收敛，避免双退）。"""
    settled: list[tuple] = []
    async def _settle(tid, ok, st):
        settled.append((tid, ok, st))

    monkeypatch.setattr(auth, 'ledger_settle', _settle)

    task = _add_task('t-live', status='running',
                     start=time.time() - _HARD_TIMEOUT - 60, worker_task=_live_worker())
    services_tasks._cancel_overtime_tasks(time.time())
    assert task['status'] == 'canceled'
    assert settled == [], 'worker 活着 → 不应代结算'


def test_cancel_overtime_ignores_recent(sandbox):
    """未超硬上限的任务不动。"""
    task = _add_task('t-new', status='running', start=time.time() - 60)
    services_tasks._cancel_overtime_tasks(time.time())
    assert task['status'] == 'running'


# ============ ⑤ 账本周期兜底 ============

def test_ledger_jobs_respect_intervals(sandbox, monkeypatch):
    """时间戳门槛：到期执行一次并推进门槛，未到期不重复执行。"""
    calls: list[str] = []
    async def _recover(max_age_seconds=0):
        calls.append(f'recover:{max_age_seconds}')

    async def _cleanup(days):
        calls.append(f'cleanup:{days}')

    monkeypatch.setattr(auth, 'ledger_recover_orphans', _recover)
    monkeypatch.setattr(auth, 'ledger_cleanup_settled', _cleanup)

    now = time.time()
    services_tasks._run_periodic_ledger_jobs(now)
    assert calls == [f'recover:{_LEDGER_ORPHAN_MAX_AGE}',
                     f'cleanup:{_LEDGER_SETTLED_RETENTION}']

    calls.clear()
    services_tasks._run_periodic_ledger_jobs(now + 1)
    assert calls == [], '门槛已推进，同一周期内不应重复执行'

    calls.clear()
    services_tasks._run_periodic_ledger_jobs(now + 61)
    assert calls == [f'recover:{_LEDGER_ORPHAN_MAX_AGE}'], '仅到期的那个触发'


def test_ledger_jobs_swallow_exception(sandbox, monkeypatch):
    """账本作业抛错由自身吞掉（不能让本轮其余作业一起落空）。"""
    async def boom(max_age_seconds=0):
        raise RuntimeError('库忙')

    monkeypatch.setattr(auth, 'ledger_recover_orphans', boom)
    services_tasks._run_periodic_ledger_jobs(time.time())   # 不抛即通过


# ============ ⑥ tasks/ 孤儿目录补扫 ============

def test_sweep_orphans_removes_stale_only(sandbox):
    """>24h 且无任务记录的目录被回收；在册任务目录与新鲜目录保留。"""
    tasks_root = sandbox['downloads'] / 'tasks'
    tasks_root.mkdir(parents=True)
    orphan = tasks_root / 'orphan'
    orphan.mkdir()
    in_registry = tasks_root / 'in-registry'
    in_registry.mkdir()
    fresh = tasks_root / 'fresh'
    fresh.mkdir()

    old = time.time() - _TASKS_ORPHAN_AGE
    os.utime(orphan, (old, old))
    os.utime(in_registry, (old, old))

    _add_task('in-registry', status='running')

    services_tasks._sweep_orphan_task_dirs(time.time())

    assert not orphan.exists(), '无记录的陈旧目录应被回收'
    assert in_registry.exists(), '在册任务目录不得被补扫误删'
    assert fresh.exists(), '未到年龄的目录保留'


def test_sweep_orphans_honours_interval(sandbox):
    """未到补扫周期的调用直接跳过。"""
    tasks_root = sandbox['downloads'] / 'tasks'
    tasks_root.mkdir(parents=True)
    first = tasks_root / 'orphan-a'
    first.mkdir()
    old = time.time() - _TASKS_ORPHAN_AGE
    os.utime(first, (old, old))

    now = time.time()
    services_tasks._sweep_orphan_task_dirs(now)
    assert not first.exists(), '首次到期应执行'

    second = tasks_root / 'orphan-b'
    second.mkdir()
    os.utime(second, (old, old))
    services_tasks._sweep_orphan_task_dirs(now + 1)
    assert second.exists(), '门槛未到 → 本轮不应补扫'


# ============ ⑦ 心跳离线处理 ============

def test_offline_device_cancels_after_threshold(sandbox):
    """离线超 30 分钟：其未完成任务置取消并唤醒 cancel Event。"""
    now = time.time()
    with DEVICE_LAST_SEEN_LOCK:
        DEVICE_LAST_SEEN['dev-off'] = now - CANCEL_AFTER_SECONDS - 60
    task = _add_task('t-off', status='running', device_id='dev-off')

    services_tasks._handle_offline_devices(now)

    assert task['status'] == 'canceled'
    assert task['cancel'].is_set()
    assert '30 分钟' in task['msg']


def test_offline_device_pauses_batch_before_threshold(sandbox):
    """离线刚过暂停阈值：批量任务先暂停，让出排队名额（「断线不占坑」）。"""
    now = time.time()
    with DEVICE_LAST_SEEN_LOCK:
        DEVICE_LAST_SEEN['dev-soft'] = now - PAUSE_AFTER_SECONDS - 1
    task = _add_task('t-soft', status='running', type='batch', device_id='dev-soft')

    services_tasks._handle_offline_devices(now)

    assert task['status'] == 'paused'
    assert task['pause'].is_set()
    assert not task['cancel'].is_set(), '未到取消阈值不应发取消'


def test_offline_handler_ignores_online_device(sandbox):
    """心跳新鲜的设备：其任务不受影响。"""
    now = time.time()
    with DEVICE_LAST_SEEN_LOCK:
        DEVICE_LAST_SEEN['dev-on'] = now
    task = _add_task('t-on', status='running', device_id='dev-on')

    services_tasks._handle_offline_devices(now)

    assert task['status'] == 'running'


# ============ 异常限频 ============

def test_log_monitor_error_rate_limits(sandbox, monkeypatch):
    """限频：窗口内多次异常只记一条（不静默也不刷屏）。"""
    events: list[str] = []

    class _Recorder:
        def warning(self, msg: str, *a: Any, **k: Any) -> None:
            events.append(msg)

    monkeypatch.setattr(api_service, 'logger', _Recorder())

    for _ in range(3):
        services_tasks._log_monitor_error()
    assert len(events) == 1, '同一窗口内只应记录一条'

    _MONITOR_ERR_TS['ts'] = time.time() - services_tasks._MONITOR_ERR_WINDOW - 1
    services_tasks._log_monitor_error()
    assert len(events) == 2, '窗口过期后应重新放行'


# ============ 单轮编排与兜底粒度 ============

_JOBS = (
    '_reap_finished_tasks', '_cancel_overtime_tasks', '_sweep_download_cache',
    '_sweep_rate_buckets', '_run_periodic_ledger_jobs', '_sweep_orphan_task_dirs',
    '_handle_offline_devices',
)


def test_tick_runs_all_jobs_in_order(monkeypatch):
    """单轮按固定顺序执行全部 7 步——顺序即优先级，重构不得打乱。"""
    order: list[str] = []
    for name in _JOBS:
        monkeypatch.setattr(
            services_tasks, name, (lambda n: lambda *a, **k: order.append(n))(name))

    services_tasks._run_monitor_tick(time.time())

    assert order == list(_JOBS)


def test_tick_exception_skips_rest_of_tick(monkeypatch):
    """前段作业抛异常 → 本轮剩余作业跳过。

    如实钉住拆分前的粒度：整个 tick 共用一个 try/except，而非逐作业隔离。
    （改为逐作业隔离属行为变更，需单独评估——本测试存在的意义正是让那次
    变更有据可依、不会悄悄发生。）
    """
    called: list[str] = []

    def boom(_now: float) -> None:
        raise RuntimeError('reap 炸了')

    monkeypatch.setattr(services_tasks, '_reap_finished_tasks', boom)
    monkeypatch.setattr(services_tasks, '_handle_offline_devices',
                        lambda _now: called.append('offline'))

    with pytest.raises(RuntimeError):
        services_tasks._run_monitor_tick(time.time())
    assert called == [], '本轮后续作业随异常一起中断（保持原语义）'


class _StopLoop(Exception):
    """从假 sleep 里抛出，把无限循环的单轮执行切断用于断言。"""


def test_loop_survives_tick_exception(monkeypatch):
    """循环本身不退出：单轮异常被限频记录后继续跑下一轮。"""
    state = {'sleeps': 0, 'logged': 0}

    class _FakeTime:
        @staticmethod
        def sleep(_seconds: float) -> None:
            state['sleeps'] += 1
            if state['sleeps'] >= 2:
                raise _StopLoop

        @staticmethod
        def time() -> float:
            return time.time()

    def boom(_now: float) -> None:
        raise RuntimeError('tick 炸了')

    monkeypatch.setattr(services_tasks, 'time', _FakeTime)
    monkeypatch.setattr(services_tasks, '_run_monitor_tick', boom)
    monkeypatch.setattr(services_tasks, '_log_monitor_error',
                        lambda: state.__setitem__('logged', state['logged'] + 1))

    with pytest.raises(_StopLoop):
        services_tasks._task_monitor_loop()

    assert state['sleeps'] == 2, '第 1 轮异常后应继续到第 2 轮'
    assert state['logged'] == 1, '每轮异常记一次'