100 lines
3.7 KiB
Python
100 lines
3.7 KiB
Python
"""后台高负载告警任务。"""
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
|
|
import monitor
|
|
import templates
|
|
|
|
log = logging.getLogger("alerter")
|
|
|
|
|
|
class Alerter:
|
|
def __init__(self, ob, cfg: dict):
|
|
self.ob = ob
|
|
self.cfg = cfg
|
|
self.interval = cfg.get("interval", 10)
|
|
self.threshold = cfg.get("cpu_threshold", 85.0)
|
|
self.consecutive_need = cfg.get("consecutive", 2)
|
|
self.cooldown = cfg.get("cooldown", 600)
|
|
self.top_n = cfg.get("top_n", 5)
|
|
self.groups = cfg.get("groups", []) or []
|
|
self.friends = cfg.get("friends", []) or []
|
|
|
|
self._consecutive = 0
|
|
self._alerting = False # 当前是否处于高负载状态
|
|
self._last_alert_ts = 0.0
|
|
self._task = None
|
|
|
|
def apply_config(self, cfg: dict):
|
|
"""热重载告警参数（保留告警状态）。"""
|
|
self.interval = cfg.get("interval", self.interval)
|
|
new_threshold = cfg.get("cpu_threshold", self.threshold)
|
|
if new_threshold != self.threshold:
|
|
log.info("告警阈值变更: %.0f%% -> %.0f%%", self.threshold, new_threshold)
|
|
self.threshold = new_threshold
|
|
self.consecutive_need = cfg.get("consecutive", self.consecutive_need)
|
|
self.cooldown = cfg.get("cooldown", self.cooldown)
|
|
self.top_n = cfg.get("top_n", self.top_n)
|
|
self.groups = cfg.get("groups", self.groups) or []
|
|
self.friends = cfg.get("friends", self.friends) or []
|
|
|
|
def start(self):
|
|
self._task = asyncio.get_running_loop().create_task(self._loop())
|
|
|
|
async def stop(self):
|
|
if self._task:
|
|
self._task.cancel()
|
|
|
|
async def _loop(self):
|
|
log.info("告警监控已启动: 阈值 %.0f%%, 连续 %d 次, 间隔 %ds, 冷却 %ds",
|
|
self.threshold, self.consecutive_need, self.interval, self.cooldown)
|
|
while True:
|
|
try:
|
|
await asyncio.sleep(self.interval)
|
|
if not self.ob.connected or (not self.groups and not self.friends):
|
|
continue
|
|
cpu = monitor.get_cpu()
|
|
await self._check(cpu["percent"])
|
|
except asyncio.CancelledError:
|
|
break
|
|
except Exception:
|
|
log.exception("告警循环出错")
|
|
|
|
async def _check(self, cpu_percent: float):
|
|
now = time.monotonic()
|
|
if cpu_percent >= self.threshold:
|
|
self._consecutive += 1
|
|
if (self._consecutive >= self.consecutive_need
|
|
and (not self._alerting or now - self._last_alert_ts >= self.cooldown)):
|
|
self._alerting = True
|
|
self._last_alert_ts = now
|
|
await self._send_alert(cpu_percent)
|
|
else:
|
|
self._consecutive = 0
|
|
if self._alerting:
|
|
self._alerting = False
|
|
msg = templates.render("recovered", cpu=f"{cpu_percent:.1f}")
|
|
log.info(msg)
|
|
await self.ob.broadcast(msg, self.groups, self.friends)
|
|
|
|
async def _send_alert(self, cpu_percent: float):
|
|
top = monitor.get_top_processes(self.top_n)
|
|
cpu = monitor.get_cpu()
|
|
load = cpu["load"]
|
|
top_list = "\n".join(
|
|
f"{i}. {p['name']} (PID {p['pid']}) CPU {p['cpu']}% 内存 {p['mem_mb']}MB"
|
|
for i, p in enumerate(top, 1)
|
|
)
|
|
msg = templates.render(
|
|
"alert",
|
|
threshold=f"{self.threshold:.0f}",
|
|
cpu=f"{cpu_percent:.1f}",
|
|
cores=cpu["cores"],
|
|
load=f"{load[0]:.2f} / {load[1]:.2f} / {load[2]:.2f}",
|
|
top_n=self.top_n,
|
|
top_list=top_list or "无数据",
|
|
)
|
|
log.warning("触发高负载告警: %.1f%%", cpu_percent)
|
|
await self.ob.broadcast(msg, self.groups, self.friends)
|