# -*- coding: utf-8 -*- """任务监控循环与设备心跳。 （自 services/runtime.py 拆出，行为与原实现一致。） """ import os import shutil import time import traceback import auth_system from core.config import ( CANCEL_AFTER_SECONDS, HEARTBEAT_INTERVAL, PAUSE_AFTER_SECONDS, RESULT_RETENTION_SECONDS, ) from core.state import ( DEVICE_LAST_SEEN, DEVICE_LAST_SEEN_LOCK, DOWNLOAD_TASKS, DOWNLOAD_TASKS_LOCK, ) from ._shared import ( _LEDGER_LAST_CLEANUP, _LEDGER_LAST_RECOVER, _MONITOR_ERR_TS, _TASKS_SWEEP_LAST, _sweep_rate_buckets, ) from ._taskcore import ( _task_files, ) from .api_service import ( api_service, ) def _quiesce_tasks_for_restore() -> None: """恢复数据库前取消全部在途任务（worker 不再引用旧库用户结算）""" with DOWNLOAD_TASKS_LOCK: for task in DOWNLOAD_TASKS.values(): if task.get('status') in ('queued', 'running', 'paused'): task['status'] = 'canceled' task['msg'] = '数据库恢复中，任务已取消（费用按实际交付结算）' task['completed_at'] = time.time() try: task['cancel'].set() except Exception: pass # 数据库恢复前静默在途任务 auth_system.register_before_restore_hook(_quiesce_tasks_for_restore) def _sweep_download_cache(max_age_seconds: int = RESULT_RETENTION_SECONDS) -> None: """下载缓存清扫（配套：批量任务不再即时删除共享缓存，改由这里按年龄回收）。 只清扫 downloads 根目录的普通文件，不碰 tasks/ 子目录（那是任务专属交付物）。""" try: root = api_service.downloads_path now = time.time() for p in root.iterdir(): try: if p.is_file() and now - p.stat().st_mtime > max_age_seconds: p.unlink() except OSError: pass except Exception: pass def _task_monitor_loop() -> None: """后台监控： - 心跳超时（40s）→ 暂停离线设备的批量任务，释放排队名额给下一位 - 心跳超时 > 30 分钟 → 自动取消该设备的所有未完成任务并清理文件 - 终态任务按 completed_at 保留 35 分钟（与前端「领取」入口对齐） - 运行中/排队/暂停任务超过 4 小时仍未结束 → 置取消（cancel Event 一并置位） - 下载缓存按年龄清扫（配套，同为 35 分钟口径） - 顺带清理限流桶中的过期项（O-5：不再只依赖"新建桶时"的清理时机） """ while True: time.sleep(HEARTBEAT_INTERVAL) try: now = time.time() # 终态任务清理（按完成时刻；删除失败保留记录下轮重试，最多 8 次）---- # 锁内只收集过期清单，文件删除移到锁外——GB 级任务目录 # 的删除曾阻塞锁数十秒，期间全站进度上报/提交/取消/领取全部停摆 finished_done: list[str] = [] finished_retry: list[str] = [] expired: list[tuple] = [] with DOWNLOAD_TASKS_LOCK: for tid, task in DOWNLOAD_TASKS.items(): if task.get('status') not in ('done', 'error', 'canceled'): continue done_at = task.get('completed_at') or 0 # 兼容无 completed_at 的旧记录：回退到 start if not done_at: done_at = task.get('start', 0) if now - done_at <= RESULT_RETENTION_SECONDS: continue expired.append((tid, _task_files(task), api_service.downloads_path / 'tasks' / tid)) cleanup_results: list[tuple] = [] for tid, files, task_dir in expired: all_removed = True for f in files: try: os.remove(f) except FileNotFoundError: pass except OSError: all_removed = False # 文件暂时不可删（被占用等） if task_dir.exists(): shutil.rmtree(str(task_dir), ignore_errors=True) if task_dir.exists(): all_removed = False cleanup_results.append((tid, all_removed)) with DOWNLOAD_TASKS_LOCK: for tid, all_removed in cleanup_results: task = DOWNLOAD_TASKS.get(tid) if task is None: finished_done.append(tid) # 锁外期间已被其他路径移除 continue if all_removed: finished_done.append(tid) else: task['_cleanup_tries'] = int(task.get('_cleanup_tries') or 0) + 1 if task['_cleanup_tries'] >= 8: # 重试多次仍失败：放弃记录但大声记录日志（文件留给缓存清扫兜底） api_service.logger.warning( f"任务 {tid} 结果文件多次清理失败，已放弃（文件可能被占用）") finished_done.append(tid) else: finished_retry.append(tid) for tid in finished_done: DOWNLOAD_TASKS.pop(tid, None) if finished_done: api_service.logger.info(f"监控线程清理过期终态任务: {finished_done}") if finished_retry: api_service.logger.info(f"监控线程保留清理失败任务待重试: {finished_retry}") # ---- 长时间未结束的任务：置取消（含 cancel Event，唤醒可能的阻塞等待）---- timeout_settle: list[str] = [] with DOWNLOAD_TASKS_LOCK: for tid, task in DOWNLOAD_TASKS.items(): if (task.get('status') in ('queued', 'running', 'paused') and now - task.get('start', 0) > 4 * 3600): task['status'] = 'canceled' task['msg'] = '任务超时未完成，已自动取消（费用按实际交付结算）' task['completed_at'] = time.time() try: task['cancel'].set() except Exception: pass # worker 线程已死则无人再走终态结算——监控代为按账本 # 结算退款（settle 幂等：worker 若已结算过，此处返回 0） th = task.get('worker_thread') if th is not None and not th.is_alive(): timeout_settle.append(tid) api_service.logger.warning(f"任务超时自动取消: {tid}") # 结算移到任务锁外（同 _mark_task_terminal 口径）—— # 库忙 15s 不再随任务锁波及全站任务端点 for tid in timeout_settle: try: auth_system.ledger_settle(tid, 0, 'timeout') except Exception: pass # ---- 下载缓存清扫（保留期同交付口径 35 分钟）+ 限流桶过期项顺手清理（O-5）---- _sweep_download_cache(RESULT_RETENTION_SECONDS) _sweep_rate_buckets() # 账本周期兜底（原仅启动执行一次）---- # 孤儿对账：长运行进程内 open 且账龄 > 5h（严格大于任务 4h 硬超时 + 监控 # tick 抖动——账本行 created_at 整秒截断使账龄恒 ≥ 任务龄，旧 4h 相等口径 # 存在抢退合法 4h+ 长任务的窗口）的行每分钟周期检查退回； # settled 行 90 天清理改为每小时低频执行，防长运行进程下表无界增长 try: # 时间戳门槛替代 tick 取模——单轮工作段超过周期时 # 取模落点会被整段跳过，时间戳门槛只延迟不漏跑 if now - _LEDGER_LAST_RECOVER['ts'] > 60: _LEDGER_LAST_RECOVER['ts'] = now auth_system.ledger_recover_orphans(max_age_seconds=5 * 3600) if now - _LEDGER_LAST_CLEANUP['ts'] > 24 * 3600: _LEDGER_LAST_CLEANUP['ts'] = now auth_system.ledger_cleanup_settled(90 * 86400) except Exception as e: api_service.logger.warning(f"账本周期兜底异常（下轮重试）: {e}") # tasks/ 孤儿目录每小时低频补扫（>24h 且无记录）---- try: if now - _TASKS_SWEEP_LAST['ts'] > 3600: _TASKS_SWEEP_LAST['ts'] = now tasks_root = api_service.downloads_path / 'tasks' if tasks_root.exists(): for d in tasks_root.iterdir(): try: if (d.is_dir() and d.name not in DOWNLOAD_TASKS and now - d.stat().st_mtime > 24 * 3600): shutil.rmtree(str(d), ignore_errors=True) api_service.logger.info(f"tasks 孤儿目录补扫移除: {d.name}") except FileNotFoundError: pass # 单条竞态不中止整轮扫描 except OSError: pass except Exception as e: api_service.logger.warning(f"tasks 孤儿目录补扫异常（下轮重试）: {e}") # ---- 设备心跳离线处理 ---- with DEVICE_LAST_SEEN_LOCK: offline = {d: now - ts for d, ts in DEVICE_LAST_SEEN.items() if now - ts > PAUSE_AFTER_SECONDS} if not offline: continue with DOWNLOAD_TASKS_LOCK: for tid, task in DOWNLOAD_TASKS.items(): dev = task.get('device_id') gap = offline.get(dev) if dev else None if not gap: continue st = task.get('status') if gap > CANCEL_AFTER_SECONDS and st in ('queued', 'running', 'paused'): # 离线超 30 分钟：请求取消（文件删除与退款统一交给 worker 的 # 取消收尾/终态清理；worker 已不在时由启动对账按账本兜底） task['status'] = 'canceled' task['msg'] = '设备离线超过 30 分钟，任务已自动取消' task['completed_at'] = time.time() try: task['cancel'].set() except Exception: pass api_service.logger.info(f"设备离线超 30 分钟自动请求取消: {tid}") elif task.get('type') == 'batch' and st in ('queued', 'running'): # 心跳超时：先暂停，让出排队名额给下一位 task['status'] = 'paused' task['msg'] = '设备离线，任务暂停（重新打开页面自动恢复）' task['pause'].set() except Exception: # 持续异常不能完全静默（否则监控失能无人知晓）：限频记录，每 10 分钟最多一条 _now = time.time() _MONITOR_ERR_TS['n'] += 1 if _now - _MONITOR_ERR_TS['ts'] > 600: api_service.logger.warning( f"任务监控循环异常（近 10 分钟累计 {_MONITOR_ERR_TS['n']} 次，" f"最后异常见堆栈）: {traceback.format_exc()[-600:]}") _MONITOR_ERR_TS['ts'] = _now _MONITOR_ERR_TS['n'] = 0 def _touch_device(device_id: str) -> None: """记录设备心跳时间（设备表容量上限——device_id 是客户端任意串， 登录用户可提交海量随机 ID 使字典无界增长，超限清最旧； 入口已统一钳 64，此处兜底再钳一次防新调用点绕过）""" device_id = str(device_id or '')[:64] if not device_id: return with DEVICE_LAST_SEEN_LOCK: if len(DEVICE_LAST_SEEN) >= 5000: # 保留最近心跳的设备：按最后心跳时间排序，砍掉最旧的一半 for d in sorted(DEVICE_LAST_SEEN, key=lambda k: DEVICE_LAST_SEEN.get(k, 0.0))[:2500]: DEVICE_LAST_SEEN.pop(d, None) DEVICE_LAST_SEEN[device_id] = time.time()