自动补测、推送审计/退避重试、监控指标与监控页面

graylog_ops/behavior.py

@@ -134,15 +134,28 @@ def asn_info(ip, root):
     return {"asn": "", "org": "未知", "shared": "unknown"}
 
 
-RANGE_OVERRIDE = None
+_RANGE_LOCAL = threading.local()
+
+
+def set_range(from_ts, to_ts):
+    """Set an absolute analysis range for the current thread only."""
+    _RANGE_LOCAL.value = {"from": from_ts, "to": to_ts}
+
+
+def clear_range():
+    _RANGE_LOCAL.value = None
+
+
+def current_range():
+    return getattr(_RANGE_LOCAL, "value", None)
 
 
 def pivot(client, series, row_groups, column_groups=None, query="*",
           limit=300, timeout=180000):
-    if RANGE_OVERRIDE:
+    override = current_range()
+    if override:
         timerange = {"type": "absolute",
-                     "from": RANGE_OVERRIDE["from"],
-                     "to": RANGE_OVERRIDE["to"]}
+                     "from": override["from"], "to": override["to"]}
     else:
         timerange = {"type": "relative", "range": WINDOW}
     body = {"queries": [{
@@ -367,6 +380,9 @@ class BehaviorAnalyzer:
         self.notifier = notifier
         self.last_run = None
         self.last_error = None
+        self.last_duration = None
+        self.last_count = 0
+        self.error_count = 0
         self.runs = 0
         self.asn = None
         try:
@@ -411,9 +427,10 @@ class BehaviorAnalyzer:
                 pass
             try:
                 extra = {}
-                if RANGE_OVERRIDE:
-                    extra = {"from_ts": RANGE_OVERRIDE["from"],
-                             "to_ts": RANGE_OVERRIDE["to"]}
+                override = current_range()
+                if override:
+                    extra = {"from_ts": override["from"],
+                             "to_ts": override["to"]}
                 hist = self.client.histogram(query=q, range_seconds=self.window,
                                              interval="5m",
                                              filter_str=self.filter_str, **extra)
@@ -623,7 +640,14 @@ class BehaviorAnalyzer:
             self.store.sync_fused()
             self.last_run = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
             self.last_error = None
+            self.last_duration = round(time.time() - started, 2)
+            self.last_count = len(items)
             self.runs += 1
+            self.store.set_meta("behavior_last_success", self.last_run)
+            self.store.set_meta("behavior_last_duration", self.last_duration)
+            self.store.set_meta("behavior_last_count", self.last_count)
+            self.store.set_meta("behavior_last_error", "")
+            self.store.incr_meta("behavior_runs_total")
             print("[%s] 行为分析: %d 个违规 IP，%d 个建议封禁，耗时 %.1fs"
                   % (self.last_run, len(items),
                      sum(1 for item in items if item["verdict"] == "建议封禁"),
@@ -633,6 +657,9 @@ class BehaviorAnalyzer:
             return items
         except Exception as exc:  # keep the service alive
             self.last_error = str(exc)
+            self.error_count += 1
+            self.store.set_meta("behavior_last_error", str(exc))
+            self.store.incr_meta("behavior_errors_total")
             print("[行为分析失败] %s" % exc, flush=True)
             return []
 
@@ -694,15 +721,46 @@ def notify_fused(store, notifier, root=None):
         if link:
             lines.append("[查看详情](%s)" % link)
         lines.append("")
+    ips = [row["ip"] for row in due[:20]]
     try:
         notifier.post("\n".join(lines).strip())
     except Exception as exc:
-        # 失败时不刷屏：每 5 分钟最多提示一次；下一轮会自动重试
+        store.mark_push_failed(ips, str(exc), stamp=now)
+        fails = store.incr_meta("push_consecutive_failures")
+        store.set_meta("push_last_error", str(exc))
+        store.set_meta("push_last_error_at", now)
+        # 失败日志限流：每 5 分钟最多提示一次
         global _PUSH_ERROR_AT
-        now = time.time()
-        if now - _PUSH_ERROR_AT > 300:
-            print("推送失败（每 5 分钟最多提示一次）: %s" % exc, flush=True)
-            _PUSH_ERROR_AT = now
+        now_epoch = time.time()
+        if now_epoch - _PUSH_ERROR_AT > 300:
+            print("推送失败（重试 #%d，每 5 分钟最多提示一次）: %s"
+                  % (fails, exc), flush=True)
+            _PUSH_ERROR_AT = now_epoch
+        if fails == 3 or (fails >= 3 and fails % 10 == 0):
+            print("[推送告警] 连续 %d 次推送失败，请检查 webhook / 网络" % fails,
+                  flush=True)
+            _send_alert("钉钉推送连续 %d 次失败：%s" % (fails, exc))
         return 0
-    store.mark_pushed([row["ip"] for row in due[:20]], now)
-    return len(due[:20])
+    store.mark_pushed(ips, now, channel=getattr(notifier, "kind", "dingtalk"))
+    store.set_meta("push_consecutive_failures", 0)
+    store.set_meta("push_last_error", "")
+    return len(ips)
+
+
+def _send_alert(text):
+    """Optional independent alert webhook (NOTIFY_ALERT_URL)."""
+    import os
+    url = (os.environ.get("NOTIFY_ALERT_URL") or "").strip()
+    if not url:
+        return
+    try:
+        from .notify import Notifier
+        alert = Notifier({
+            "type": os.environ.get("NOTIFY_ALERT_TYPE", "dingtalk"),
+            "url": url,
+            "secret": os.environ.get("NOTIFY_ALERT_SECRET", ""),
+            "title": "推送告警",
+        })
+        alert.post(text, title="推送告警")
+    except Exception as exc:
+        print("告警发送失败: %s" % exc, flush=True)