diff --git a/alerter.py b/alerter.py index 59df141..f0c2f4d 100644 --- a/alerter.py +++ b/alerter.py @@ -1,4 +1,4 @@ -"""后台高负载告警任务。""" +"""后台高负载告警任务:CPU/内存超阈值告警 + 定时状态推送。""" import asyncio import logging import time @@ -9,33 +9,43 @@ import templates log = logging.getLogger("alerter") +def _top1_str(p: dict) -> str: + return f"{p['name']} (PID {p['pid']}) CPU {p['cpu']}% 内存 {p['mem_mb']}MB" + + class Alerter: def __init__(self, ob, cfg: dict): self.ob = ob - self.cfg = cfg - self.interval = cfg.get("interval", 10) - self.threshold = cfg.get("cpu_threshold", 85.0) - self.consecutive_need = cfg.get("consecutive", 2) - self.cooldown = cfg.get("cooldown", 600) - self.top_n = cfg.get("top_n", 5) - self.groups = cfg.get("groups", []) or [] - self.friends = cfg.get("friends", []) or [] + self.groups: list = [] + self.friends: list = [] + self.apply_config(cfg, initial=True) - self._consecutive = 0 - self._alerting = False # 当前是否处于高负载状态 - self._last_alert_ts = 0.0 + self._cpu_hits = 0 + self._mem_hits = 0 + self._cpu_alerting = False + self._mem_alerting = False + self._cpu_last_ts = 0.0 + self._mem_last_ts = 0.0 self._task = None - def apply_config(self, cfg: dict): + def apply_config(self, cfg: dict, initial: bool = False): """热重载告警参数(保留告警状态)。""" - self.interval = cfg.get("interval", self.interval) - new_threshold = cfg.get("cpu_threshold", self.threshold) - if new_threshold != self.threshold: - log.info("告警阈值变更: %.0f%% -> %.0f%%", self.threshold, new_threshold) - self.threshold = new_threshold - self.consecutive_need = cfg.get("consecutive", self.consecutive_need) - self.cooldown = cfg.get("cooldown", self.cooldown) - self.top_n = cfg.get("top_n", self.top_n) + if initial: + self.interval = cfg.get("interval", 10) + self.threshold = cfg.get("cpu_threshold", 85.0) + self.consecutive_need = cfg.get("consecutive", 2) + self.cooldown = cfg.get("cooldown", 600) + self.top_n = cfg.get("top_n", 5) + else: + self.interval = cfg.get("interval", self.interval) + new_threshold = cfg.get("cpu_threshold", self.threshold) + if new_threshold != self.threshold: + log.info("CPU告警阈值变更: %.0f%% -> %.0f%%", self.threshold, new_threshold) + self.threshold = new_threshold + self.consecutive_need = cfg.get("consecutive", self.consecutive_need) + self.cooldown = cfg.get("cooldown", self.cooldown) + self.top_n = cfg.get("top_n", self.top_n) + self.memory_threshold = cfg.get("memory_threshold", self.__dict__.get("memory_threshold", 90.0)) self.groups = cfg.get("groups", self.groups) or [] self.friends = cfg.get("friends", self.friends) or [] @@ -47,38 +57,57 @@ class Alerter: self._task.cancel() async def _loop(self): - log.info("告警监控已启动: 阈值 %.0f%%, 连续 %d 次, 间隔 %ds, 冷却 %ds", - self.threshold, self.consecutive_need, self.interval, self.cooldown) + log.info("告警监控已启动: CPU阈值 %.0f%%, 内存阈值 %.0f%%, 连续 %d 次, 间隔 %ds, 冷却 %ds", + self.threshold, self.memory_threshold, self.consecutive_need, + self.interval, self.cooldown) while True: try: await asyncio.sleep(self.interval) if not self.ob.connected or (not self.groups and not self.friends): continue cpu = monitor.get_cpu() - await self._check(cpu["percent"]) + mem = monitor.get_memory() + await self._check(cpu["percent"], mem["percent"]) except asyncio.CancelledError: break except Exception: log.exception("告警循环出错") - async def _check(self, cpu_percent: float): + async def _check(self, cpu_percent: float, mem_percent: float): now = time.monotonic() + # CPU 告警 if cpu_percent >= self.threshold: - self._consecutive += 1 - if (self._consecutive >= self.consecutive_need - and (not self._alerting or now - self._last_alert_ts >= self.cooldown)): - self._alerting = True - self._last_alert_ts = now - await self._send_alert(cpu_percent) + self._cpu_hits += 1 + if (self._cpu_hits >= self.consecutive_need + and (not self._cpu_alerting or now - self._cpu_last_ts >= self.cooldown)): + self._cpu_alerting = True + self._cpu_last_ts = now + await self._send_cpu_alert(cpu_percent) else: - self._consecutive = 0 - if self._alerting: - self._alerting = False + self._cpu_hits = 0 + if self._cpu_alerting: + self._cpu_alerting = False msg = templates.render("recovered", cpu=f"{cpu_percent:.1f}") log.info(msg) await self.ob.broadcast(msg, self.groups, self.friends) - async def _send_alert(self, cpu_percent: float): + # 内存告警 + if mem_percent >= self.memory_threshold: + self._mem_hits += 1 + if (self._mem_hits >= self.consecutive_need + and (not self._mem_alerting or now - self._mem_last_ts >= self.cooldown)): + self._mem_alerting = True + self._mem_last_ts = now + await self._send_mem_alert(mem_percent) + else: + self._mem_hits = 0 + if self._mem_alerting: + self._mem_alerting = False + msg = templates.render("recovered_mem", mem=f"{mem_percent:.1f}") + log.info(msg) + await self.ob.broadcast(msg, self.groups, self.friends) + + async def _send_cpu_alert(self, cpu_percent: float): top = monitor.get_top_processes(self.top_n) cpu = monitor.get_cpu() load = cpu["load"] @@ -86,6 +115,7 @@ class Alerter: f"{i}. {p['name']} (PID {p['pid']}) CPU {p['cpu']}% 内存 {p['mem_mb']}MB" for i, p in enumerate(top, 1) ) + top1 = top[0] if top else {"name": "无数据", "pid": "-", "cpu": 0, "mem_mb": 0} msg = templates.render( "alert", threshold=f"{self.threshold:.0f}", @@ -94,6 +124,65 @@ class Alerter: load=f"{load[0]:.2f} / {load[1]:.2f} / {load[2]:.2f}", top_n=self.top_n, top_list=top_list or "无数据", + top1=_top1_str(top1), ) - log.warning("触发高负载告警: %.1f%%", cpu_percent) + log.warning("触发CPU高负载告警: %.1f%%", cpu_percent) await self.ob.broadcast(msg, self.groups, self.friends) + + async def _send_mem_alert(self, mem_percent: float): + m = monitor.get_memory() + top1 = monitor.get_top_memory(1) + top1 = top1[0] if top1 else {"name": "无数据", "pid": "-", "cpu": 0, "mem_mb": 0} + msg = templates.render( + "alert_mem", + threshold=f"{self.memory_threshold:.0f}", + mem=f"{mem_percent:.1f}", + mem_detail=f"{m['used_gb']}/{m['total_gb']}GB (Swap {m['swap_percent']}%)", + top1=_top1_str(top1), + ) + log.warning("触发内存高占用告警: %.1f%%", mem_percent) + await self.ob.broadcast(msg, self.groups, self.friends) + + +class Reporter: + """定时向配置目标推送状态报告。""" + + def __init__(self, ob, cfg: dict): + self.ob = ob + self.enabled = cfg.get("enabled", False) + self.interval_minutes = cfg.get("interval_minutes", 60) + self.groups = cfg.get("groups", []) or [] + self.friends = cfg.get("friends", []) or [] + self._task = None + + def start(self): + if not self.enabled: + log.info("定时报告未启用") + return + self._task = asyncio.get_running_loop().create_task(self._loop()) + + async def stop(self): + if self._task: + self._task.cancel() + + async def _loop(self): + log.info("定时报告已启动: 每 %d 分钟推送一次", self.interval_minutes) + while True: + await asyncio.sleep(self.interval_minutes * 60) + try: + if not self.ob.connected or (not self.groups and not self.friends): + continue + status = bot_status() + msg = templates.render("report", status=status) + await self.ob.broadcast(msg, self.groups, self.friends) + log.info("已推送定时报告") + except asyncio.CancelledError: + break + except Exception: + log.exception("定时报告推送出错") + + +def bot_status() -> str: + """状态文本(避免 bot <-> alerter 循环导入,延迟导入 bot)。""" + import bot + return bot.fmt_status(monitor.get_status()) diff --git a/bot.py b/bot.py index ba971f6..53238b1 100644 --- a/bot.py +++ b/bot.py @@ -6,7 +6,7 @@ from pathlib import Path import monitor import templates -from alerter import Alerter +from alerter import Alerter, Reporter from onebot import OneBotServer from updater import Updater @@ -205,6 +205,8 @@ class Bot: await self.ob.start(self.on_event) alerter = Alerter(self.ob, self.cfg.get("alert", {})) alerter.start() + reporter = Reporter(self.ob, self.cfg.get("report", {})) + reporter.start() updater = Updater(self.ob, self.cfg.get("update", {})) if self.cfg.get("update", {}).get("enabled", False): updater.start() @@ -217,6 +219,7 @@ class Bot: finally: watcher.cancel() await updater.stop() + await reporter.stop() await alerter.stop() await self.ob.stop() diff --git a/config.example.toml b/config.example.toml index b9ad1d0..f70c2cb 100644 --- a/config.example.toml +++ b/config.example.toml @@ -10,6 +10,8 @@ access_token = "" [alert] # CPU 告警阈值(%) cpu_threshold = 85.0 +# 内存告警阈值(%),设为 0 关闭内存告警 +memory_threshold = 90.0 # 需要连续超过阈值多少个采样周期才告警 consecutive = 2 # 采样间隔(秒) @@ -22,6 +24,16 @@ top_n = 5 groups = [123456789] friends = [] +[report] +# 定时推送状态报告 +enabled = false +# 推送间隔(分钟) +interval_minutes = 60 +# 推送目标(留空则用 [alert] 的目标) +groups = [] +friends = [] + + [update] # Git 热更新:定期检查远端是否有新版本,更新前先推送通知,再拉取并自动重启 enabled = true diff --git a/monitor.py b/monitor.py index 4f6358c..6f309d7 100644 --- a/monitor.py +++ b/monitor.py @@ -229,6 +229,25 @@ def get_top_processes(n: int = 10) -> list: return procs[:n] +def get_top_memory(n: int = 10) -> list: + """按内存占用排序的 TOP N 进程(无需 CPU 基准采样)。""" + procs = [] + for p in psutil.process_iter(["pid", "name"]): + try: + if p.info["pid"] in (0, 4): + continue + procs.append({ + "pid": p.info["pid"], + "name": (p.info["name"] or "?")[:24], + "cpu": 0.0, + "mem_mb": round(p.memory_info().rss / 1024**2, 1), + }) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue + procs.sort(key=lambda x: x["mem_mb"], reverse=True) + return procs[:n] + + def get_uptime() -> float: try: with open("/proc/uptime") as f: diff --git a/templates/alert.txt b/templates/alert.txt index 6a95d68..6235f21 100644 --- a/templates/alert.txt +++ b/templates/alert.txt @@ -3,3 +3,5 @@ TOP{top_n} 高负载进程: {top_list} + +占用最大: {top1} diff --git a/templates/alert_mem.txt b/templates/alert_mem.txt new file mode 100644 index 0000000..7e1de18 --- /dev/null +++ b/templates/alert_mem.txt @@ -0,0 +1,4 @@ +[内存告警] 内存使用率持续超过 {threshold}%,当前 {mem}% +内存: {mem_detail} + +占用内存最大的进程: {top1} diff --git a/templates/recovered_mem.txt b/templates/recovered_mem.txt new file mode 100644 index 0000000..9dc67b8 --- /dev/null +++ b/templates/recovered_mem.txt @@ -0,0 +1 @@ +[恢复] 服务器内存使用率已回落到 {mem}%,恢复正常。 diff --git a/templates/report.txt b/templates/report.txt new file mode 100644 index 0000000..ab27fd2 --- /dev/null +++ b/templates/report.txt @@ -0,0 +1,2 @@ +[定时报告] 服务器状态 +{status}