feat: memory alerts with top-memory process, top offender in alerts, scheduled status reports

alerter.py

@@ -1,4 +1,4 @@
-"""后台高负载告警任务。"""
+"""后台高负载告警任务：CPU/内存超阈值告警 + 定时状态推送。"""
 import asyncio
 import logging
 import time
@@ -9,33 +9,43 @@ import templates
 log = logging.getLogger("alerter")
 
 
+def _top1_str(p: dict) -> str:
+    return f"{p['name']} (PID {p['pid']})  CPU {p['cpu']}%  内存 {p['mem_mb']}MB"
+
+
 class Alerter:
     def __init__(self, ob, cfg: dict):
         self.ob = ob
-        self.cfg = cfg
-        self.interval = cfg.get("interval", 10)
-        self.threshold = cfg.get("cpu_threshold", 85.0)
-        self.consecutive_need = cfg.get("consecutive", 2)
-        self.cooldown = cfg.get("cooldown", 600)
-        self.top_n = cfg.get("top_n", 5)
-        self.groups = cfg.get("groups", []) or []
-        self.friends = cfg.get("friends", []) or []
+        self.groups: list = []
+        self.friends: list = []
+        self.apply_config(cfg, initial=True)
 
-        self._consecutive = 0
-        self._alerting = False          # 当前是否处于高负载状态
-        self._last_alert_ts = 0.0
+        self._cpu_hits = 0
+        self._mem_hits = 0
+        self._cpu_alerting = False
+        self._mem_alerting = False
+        self._cpu_last_ts = 0.0
+        self._mem_last_ts = 0.0
         self._task = None
 
-    def apply_config(self, cfg: dict):
+    def apply_config(self, cfg: dict, initial: bool = False):
         """热重载告警参数（保留告警状态）。"""
-        self.interval = cfg.get("interval", self.interval)
-        new_threshold = cfg.get("cpu_threshold", self.threshold)
-        if new_threshold != self.threshold:
-            log.info("告警阈值变更: %.0f%% -> %.0f%%", self.threshold, new_threshold)
-            self.threshold = new_threshold
-        self.consecutive_need = cfg.get("consecutive", self.consecutive_need)
-        self.cooldown = cfg.get("cooldown", self.cooldown)
-        self.top_n = cfg.get("top_n", self.top_n)
+        if initial:
+            self.interval = cfg.get("interval", 10)
+            self.threshold = cfg.get("cpu_threshold", 85.0)
+            self.consecutive_need = cfg.get("consecutive", 2)
+            self.cooldown = cfg.get("cooldown", 600)
+            self.top_n = cfg.get("top_n", 5)
+        else:
+            self.interval = cfg.get("interval", self.interval)
+            new_threshold = cfg.get("cpu_threshold", self.threshold)
+            if new_threshold != self.threshold:
+                log.info("CPU告警阈值变更: %.0f%% -> %.0f%%", self.threshold, new_threshold)
+                self.threshold = new_threshold
+            self.consecutive_need = cfg.get("consecutive", self.consecutive_need)
+            self.cooldown = cfg.get("cooldown", self.cooldown)
+            self.top_n = cfg.get("top_n", self.top_n)
+        self.memory_threshold = cfg.get("memory_threshold", self.__dict__.get("memory_threshold", 90.0))
         self.groups = cfg.get("groups", self.groups) or []
         self.friends = cfg.get("friends", self.friends) or []
 
@@ -47,38 +57,57 @@ class Alerter:
             self._task.cancel()
 
     async def _loop(self):
-        log.info("告警监控已启动: 阈值 %.0f%%, 连续 %d 次, 间隔 %ds, 冷却 %ds",
-                 self.threshold, self.consecutive_need, self.interval, self.cooldown)
+        log.info("告警监控已启动: CPU阈值 %.0f%%, 内存阈值 %.0f%%, 连续 %d 次, 间隔 %ds, 冷却 %ds",
+                 self.threshold, self.memory_threshold, self.consecutive_need,
+                 self.interval, self.cooldown)
         while True:
             try:
                 await asyncio.sleep(self.interval)
                 if not self.ob.connected or (not self.groups and not self.friends):
                     continue
                 cpu = monitor.get_cpu()
-                await self._check(cpu["percent"])
+                mem = monitor.get_memory()
+                await self._check(cpu["percent"], mem["percent"])
             except asyncio.CancelledError:
                 break
             except Exception:
                 log.exception("告警循环出错")
 
-    async def _check(self, cpu_percent: float):
+    async def _check(self, cpu_percent: float, mem_percent: float):
         now = time.monotonic()
+        # CPU 告警
         if cpu_percent >= self.threshold:
-            self._consecutive += 1
-            if (self._consecutive >= self.consecutive_need
-                    and (not self._alerting or now - self._last_alert_ts >= self.cooldown)):
-                self._alerting = True
-                self._last_alert_ts = now
-                await self._send_alert(cpu_percent)
+            self._cpu_hits += 1
+            if (self._cpu_hits >= self.consecutive_need
+                    and (not self._cpu_alerting or now - self._cpu_last_ts >= self.cooldown)):
+                self._cpu_alerting = True
+                self._cpu_last_ts = now
+                await self._send_cpu_alert(cpu_percent)
         else:
-            self._consecutive = 0
-            if self._alerting:
-                self._alerting = False
+            self._cpu_hits = 0
+            if self._cpu_alerting:
+                self._cpu_alerting = False
                 msg = templates.render("recovered", cpu=f"{cpu_percent:.1f}")
                 log.info(msg)
                 await self.ob.broadcast(msg, self.groups, self.friends)
 
-    async def _send_alert(self, cpu_percent: float):
+        # 内存告警
+        if mem_percent >= self.memory_threshold:
+            self._mem_hits += 1
+            if (self._mem_hits >= self.consecutive_need
+                    and (not self._mem_alerting or now - self._mem_last_ts >= self.cooldown)):
+                self._mem_alerting = True
+                self._mem_last_ts = now
+                await self._send_mem_alert(mem_percent)
+        else:
+            self._mem_hits = 0
+            if self._mem_alerting:
+                self._mem_alerting = False
+                msg = templates.render("recovered_mem", mem=f"{mem_percent:.1f}")
+                log.info(msg)
+                await self.ob.broadcast(msg, self.groups, self.friends)
+
+    async def _send_cpu_alert(self, cpu_percent: float):
         top = monitor.get_top_processes(self.top_n)
         cpu = monitor.get_cpu()
         load = cpu["load"]
@@ -86,6 +115,7 @@ class Alerter:
             f"{i}. {p['name']} (PID {p['pid']})  CPU {p['cpu']}%  内存 {p['mem_mb']}MB"
             for i, p in enumerate(top, 1)
         )
+        top1 = top[0] if top else {"name": "无数据", "pid": "-", "cpu": 0, "mem_mb": 0}
         msg = templates.render(
             "alert",
             threshold=f"{self.threshold:.0f}",
@@ -94,6 +124,65 @@ class Alerter:
             load=f"{load[0]:.2f} / {load[1]:.2f} / {load[2]:.2f}",
             top_n=self.top_n,
             top_list=top_list or "无数据",
+            top1=_top1_str(top1),
         )
-        log.warning("触发高负载告警: %.1f%%", cpu_percent)
+        log.warning("触发CPU高负载告警: %.1f%%", cpu_percent)
         await self.ob.broadcast(msg, self.groups, self.friends)
+
+    async def _send_mem_alert(self, mem_percent: float):
+        m = monitor.get_memory()
+        top1 = monitor.get_top_memory(1)
+        top1 = top1[0] if top1 else {"name": "无数据", "pid": "-", "cpu": 0, "mem_mb": 0}
+        msg = templates.render(
+            "alert_mem",
+            threshold=f"{self.memory_threshold:.0f}",
+            mem=f"{mem_percent:.1f}",
+            mem_detail=f"{m['used_gb']}/{m['total_gb']}GB (Swap {m['swap_percent']}%)",
+            top1=_top1_str(top1),
+        )
+        log.warning("触发内存高占用告警: %.1f%%", mem_percent)
+        await self.ob.broadcast(msg, self.groups, self.friends)
+
+
+class Reporter:
+    """定时向配置目标推送状态报告。"""
+
+    def __init__(self, ob, cfg: dict):
+        self.ob = ob
+        self.enabled = cfg.get("enabled", False)
+        self.interval_minutes = cfg.get("interval_minutes", 60)
+        self.groups = cfg.get("groups", []) or []
+        self.friends = cfg.get("friends", []) or []
+        self._task = None
+
+    def start(self):
+        if not self.enabled:
+            log.info("定时报告未启用")
+            return
+        self._task = asyncio.get_running_loop().create_task(self._loop())
+
+    async def stop(self):
+        if self._task:
+            self._task.cancel()
+
+    async def _loop(self):
+        log.info("定时报告已启动: 每 %d 分钟推送一次", self.interval_minutes)
+        while True:
+            await asyncio.sleep(self.interval_minutes * 60)
+            try:
+                if not self.ob.connected or (not self.groups and not self.friends):
+                    continue
+                status = bot_status()
+                msg = templates.render("report", status=status)
+                await self.ob.broadcast(msg, self.groups, self.friends)
+                log.info("已推送定时报告")
+            except asyncio.CancelledError:
+                break
+            except Exception:
+                log.exception("定时报告推送出错")
+
+
+def bot_status() -> str:
+    """状态文本（避免 bot <-> alerter 循环导入，延迟导入 bot）。"""
+    import bot
+    return bot.fmt_status(monitor.get_status())

bot.py

@@ -6,7 +6,7 @@ from pathlib import Path
 
 import monitor
 import templates
-from alerter import Alerter
+from alerter import Alerter, Reporter
 from onebot import OneBotServer
 from updater import Updater
 
@@ -205,6 +205,8 @@ class Bot:
         await self.ob.start(self.on_event)
         alerter = Alerter(self.ob, self.cfg.get("alert", {}))
         alerter.start()
+        reporter = Reporter(self.ob, self.cfg.get("report", {}))
+        reporter.start()
         updater = Updater(self.ob, self.cfg.get("update", {}))
         if self.cfg.get("update", {}).get("enabled", False):
             updater.start()
@@ -217,6 +219,7 @@ class Bot:
         finally:
             watcher.cancel()
             await updater.stop()
+            await reporter.stop()
             await alerter.stop()
             await self.ob.stop()
 

config.example.toml

@@ -10,6 +10,8 @@ access_token = ""
 [alert]
 # CPU 告警阈值(%)
 cpu_threshold = 85.0
+# 内存告警阈值(%)，设为 0 关闭内存告警
+memory_threshold = 90.0
 # 需要连续超过阈值多少个采样周期才告警
 consecutive = 2
 # 采样间隔(秒)
@@ -22,6 +24,16 @@ top_n = 5
 groups = [123456789]
 friends = []
 
+[report]
+# 定时推送状态报告
+enabled = false
+# 推送间隔(分钟)
+interval_minutes = 60
+# 推送目标（留空则用 [alert] 的目标）
+groups = []
+friends = []
+
+
 [update]
 # Git 热更新：定期检查远端是否有新版本，更新前先推送通知，再拉取并自动重启
 enabled = true

monitor.py

@@ -229,6 +229,25 @@ def get_top_processes(n: int = 10) -> list:
     return procs[:n]
 
 
+def get_top_memory(n: int = 10) -> list:
+    """按内存占用排序的 TOP N 进程（无需 CPU 基准采样）。"""
+    procs = []
+    for p in psutil.process_iter(["pid", "name"]):
+        try:
+            if p.info["pid"] in (0, 4):
+                continue
+            procs.append({
+                "pid": p.info["pid"],
+                "name": (p.info["name"] or "?")[:24],
+                "cpu": 0.0,
+                "mem_mb": round(p.memory_info().rss / 1024**2, 1),
+            })
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            continue
+    procs.sort(key=lambda x: x["mem_mb"], reverse=True)
+    return procs[:n]
+
+
 def get_uptime() -> float:
     try:
         with open("/proc/uptime") as f:

templates/alert.txt

@@ -3,3 +3,5 @@
 
 TOP{top_n} 高负载进程:
 {top_list}
+
+占用最大: {top1}

templates/alert_mem.txt

@@ -0,0 +1,4 @@
+[内存告警] 内存使用率持续超过 {threshold}%，当前 {mem}%
+内存: {mem_detail}
+
+占用内存最大的进程: {top1}

templates/recovered_mem.txt

@@ -0,0 +1 @@
+[恢复] 服务器内存使用率已回落到 {mem}%，恢复正常。

templates/report.txt

@@ -0,0 +1,2 @@
+[定时报告] 服务器状态
+{status}