85 lines
3.1 KiB
Python
85 lines
3.1 KiB
Python
"""后台高负载告警任务。"""
|
||
import asyncio
|
||
import logging
|
||
import time
|
||
|
||
import monitor
|
||
|
||
log = logging.getLogger("alerter")
|
||
|
||
|
||
class Alerter:
|
||
def __init__(self, ob, cfg: dict):
|
||
self.ob = ob
|
||
self.cfg = cfg
|
||
self.interval = cfg.get("interval", 10)
|
||
self.threshold = cfg.get("cpu_threshold", 85.0)
|
||
self.consecutive_need = cfg.get("consecutive", 2)
|
||
self.cooldown = cfg.get("cooldown", 600)
|
||
self.top_n = cfg.get("top_n", 5)
|
||
self.groups = cfg.get("groups", []) or []
|
||
self.friends = cfg.get("friends", []) or []
|
||
|
||
self._consecutive = 0
|
||
self._alerting = False # 当前是否处于高负载状态
|
||
self._last_alert_ts = 0.0
|
||
self._task = None
|
||
|
||
def start(self):
|
||
self._task = asyncio.get_running_loop().create_task(self._loop())
|
||
|
||
async def stop(self):
|
||
if self._task:
|
||
self._task.cancel()
|
||
|
||
async def _loop(self):
|
||
log.info("告警监控已启动: 阈值 %.0f%%, 连续 %d 次, 间隔 %ds, 冷却 %ds",
|
||
self.threshold, self.consecutive_need, self.interval, self.cooldown)
|
||
while True:
|
||
try:
|
||
await asyncio.sleep(self.interval)
|
||
if not self.ob.connected or (not self.groups and not self.friends):
|
||
continue
|
||
cpu = monitor.get_cpu()
|
||
await self._check(cpu["percent"])
|
||
except asyncio.CancelledError:
|
||
break
|
||
except Exception:
|
||
log.exception("告警循环出错")
|
||
|
||
async def _check(self, cpu_percent: float):
|
||
now = time.monotonic()
|
||
if cpu_percent >= self.threshold:
|
||
self._consecutive += 1
|
||
if (self._consecutive >= self.consecutive_need
|
||
and (not self._alerting or now - self._last_alert_ts >= self.cooldown)):
|
||
self._alerting = True
|
||
self._last_alert_ts = now
|
||
await self._send_alert(cpu_percent)
|
||
else:
|
||
self._consecutive = 0
|
||
if self._alerting:
|
||
self._alerting = False
|
||
msg = (f"✅ [恢复] 服务器 CPU 已回落到 {cpu_percent:.1f}%，"
|
||
f"负载恢复正常。")
|
||
log.info(msg)
|
||
await self.ob.broadcast(msg, self.groups, self.friends)
|
||
|
||
async def _send_alert(self, cpu_percent: float):
|
||
top = monitor.get_top_processes(self.top_n)
|
||
cpu = monitor.get_cpu()
|
||
load = cpu["load"]
|
||
lines = [
|
||
f"⚠️ [高负载告警] CPU 持续超过 {self.threshold:.0f}%，当前 {cpu_percent:.1f}%",
|
||
f"负载: {load[0]:.2f} / {load[1]:.2f} / {load[2]:.2f} 核心: {cpu['cores']}",
|
||
"",
|
||
f"TOP{self.top_n} 高负载进程:",
|
||
]
|
||
for i, p in enumerate(top, 1):
|
||
lines.append(
|
||
f"{i}. {p['name']} (PID {p['pid']}) CPU {p['cpu']}% 内存 {p['mem_mb']}MB"
|
||
)
|
||
msg = "\n".join(lines)
|
||
log.warning("触发高负载告警: %.1f%%", cpu_percent)
|
||
await self.ob.broadcast(msg, self.groups, self.friends)
|