自动补测、推送审计/退避重试、监控指标与监控页面

graylog_ops/watch.py

@@ -21,6 +21,7 @@ Design
 """
 
 import csv
+import datetime
 import html
 import io
 import ipaddress
@@ -362,9 +363,31 @@ class Store:
                        fused_reason TEXT, updated_at TEXT
                    )"""
             )
+            for column in ("push_fail_count INTEGER DEFAULT 0",
+                           "push_next_at TEXT"):
+                try:
+                    self.conn.execute(
+                        "ALTER TABLE ip_fused ADD COLUMN %s" % column)
+                except sqlite3.OperationalError:
+                    pass
             self.conn.execute(
                 "CREATE INDEX IF NOT EXISTS idx_ip_fused_first "
                 "ON ip_fused(first_banned_at DESC, ip)")
+            self.conn.execute(
+                """CREATE TABLE IF NOT EXISTS meta (
+                       key TEXT PRIMARY KEY, value TEXT
+                   )"""
+            )
+            self.conn.execute(
+                """CREATE TABLE IF NOT EXISTS notifications (
+                       id INTEGER PRIMARY KEY AUTOINCREMENT,
+                       ip TEXT, channel TEXT, status TEXT, attempt INTEGER,
+                       error TEXT, created_at TEXT
+                   )"""
+            )
+            self.conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_notifications_created "
+                "ON notifications(created_at DESC)")
             self.conn.execute(
                 """CREATE TABLE IF NOT EXISTS events (
                        id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -746,6 +769,85 @@ class Store:
                 self.conn.execute("DELETE FROM ip_fused WHERE ip = ?", (ip,))
             return len(wanted)
 
+    # ------------------------------------------------------------- meta
+    def get_meta(self, key, default=None):
+        with self.lock:
+            row = self.conn.execute(
+                "SELECT value FROM meta WHERE key=?", (key,)).fetchone()
+        return row["value"] if row else default
+
+    def set_meta(self, key, value):
+        with self.lock, self.conn:
+            self.conn.execute(
+                "INSERT INTO meta(key, value) VALUES(?,?) "
+                "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
+                (key, str(value)))
+
+    def incr_meta(self, key, amount=1):
+        with self.lock, self.conn:
+            self.conn.execute(
+                "INSERT INTO meta(key, value) VALUES(?,?) "
+                "ON CONFLICT(key) DO UPDATE SET value=CAST("
+                "CAST(value AS INTEGER)+? AS TEXT)",
+                (key, str(amount), int(amount)))
+            row = self.conn.execute(
+                "SELECT value FROM meta WHERE key=?", (key,)).fetchone()
+        return int(row["value"] or 0)
+
+    def meta_dict(self):
+        with self.lock:
+            rows = self.conn.execute("SELECT key, value FROM meta").fetchall()
+        return {row["key"]: row["value"] for row in rows}
+
+    # ------------------------------------------------------- notifications
+    def log_notification(self, ip, channel, status, attempt, error=None,
+                        stamp=None):
+        stamp = stamp or now_iso()
+        with self.lock, self.conn:
+            self.conn.execute(
+                "INSERT INTO notifications(ip, channel, status, attempt, error, "
+                "created_at) VALUES(?,?,?,?,?,?)",
+                (ip, channel, status, int(attempt or 0), error, stamp))
+
+    def mark_push_failed(self, ips, error, stamp=None, base=60, cap=3600):
+        stamp = stamp or now_iso()
+        base_time = time.time()
+        with self.lock, self.conn:
+            for ip in ips:
+                row = self.conn.execute(
+                    "SELECT COALESCE(push_fail_count,0) AS n FROM ip_fused "
+                    "WHERE ip=?", (ip,)).fetchone()
+                fails = int(row["n"] if row else 0) + 1
+                delay = min(cap, base * (2 ** (fails - 1)))
+                next_at = time.strftime(
+                    "%Y-%m-%dT%H:%M:%SZ",
+                    time.gmtime(base_time + delay))
+                self.conn.execute(
+                    "UPDATE ip_fused SET push_fail_count=?, push_next_at=? "
+                    "WHERE ip=?", (fails, next_at, ip))
+                self.conn.execute(
+                    "INSERT INTO notifications(ip, channel, status, attempt, "
+                    "error, created_at) VALUES(?,?,?,?,?,?)",
+                    (ip, "dingtalk", "failed", fails, error, stamp))
+        return len(ips)
+
+    def notification_stats(self, hours=24):
+        since = time.strftime("%Y-%m-%dT%H:%M:%SZ",
+                              time.gmtime(time.time() - hours * 3600))
+        with self.lock:
+            rows = self.conn.execute(
+                "SELECT status, COUNT(*) AS n FROM notifications "
+                "WHERE created_at >= ? GROUP BY status", (since,)).fetchall()
+            recent = self.conn.execute(
+                "SELECT * FROM notifications ORDER BY id DESC LIMIT 30").fetchall()
+        counts = {row["status"]: int(row["n"]) for row in rows}
+        sent = counts.get("sent", 0)
+        failed = counts.get("failed", 0)
+        total = sent + failed
+        return {"sent": sent, "failed": failed,
+                "success_ratio": round(sent / total, 4) if total else None,
+                "recent": [dict(row) for row in recent]}
+
     def due_pushes(self, active_hours=24, cooldown_hours=24):
         now = time.time()
         active_cut = time.strftime("%Y-%m-%dT%H:%M:%SZ",
@@ -767,8 +869,9 @@ class Store:
                      LEFT JOIN behavior_profiles b ON b.ip = f.ip
                     WHERE f.last_activity_at >= ?
                       AND (f.last_pushed_at IS NULL OR f.last_pushed_at <= ?)
+                      AND (f.push_next_at IS NULL OR f.push_next_at <= ?)
                     ORDER BY f.first_banned_at DESC LIMIT 50""",
-                (active_cut, push_cut)).fetchall()
+                (active_cut, push_cut, now_iso())).fetchall()
         items = []
         for row in rows:
             item = dict(row)
@@ -785,12 +888,18 @@ class Store:
             items.append(item)
         return items
 
-    def mark_pushed(self, ips, stamp):
+    def mark_pushed(self, ips, stamp, channel="dingtalk"):
         with self.lock, self.conn:
             for ip in ips:
                 self.conn.execute(
-                    "UPDATE ip_fused SET last_pushed_at=?, push_count=push_count+1 "
-                    "WHERE ip=?", (stamp, ip))
+                    "UPDATE ip_fused SET last_pushed_at=?, "
+                    "push_count=push_count+1, push_fail_count=0, "
+                    "push_next_at=NULL WHERE ip=?", (stamp, ip))
+                self.conn.execute(
+                    "INSERT INTO notifications(ip, channel, status, attempt, "
+                    "error, created_at) VALUES(?,?,?,?,?,?)",
+                    (ip, channel, "sent", 1, None, stamp))
+        self.incr_meta("push_sent_total", len(ips))
 
     def stats(self):
         with self.lock:
@@ -946,6 +1055,8 @@ class Detector:
         self.last_run = None
         self.last_success = None
         self.last_error = None
+        self.last_duration = None
+        self.error_count = 0
         self.runs = 0
 
     def _terms(self, query, window_cap=None):
@@ -1232,7 +1343,13 @@ class Detector:
 
         self.last_run = stamp
         self.last_success = time.time()
+        self.last_duration = round(time.time() - started, 2)
+        self.last_error = None
         self.runs += 1
+        self.store.set_meta("detect_last_success", stamp)
+        self.store.set_meta("detect_last_duration", self.last_duration)
+        self.store.set_meta("detect_last_error", "")
+        self.store.incr_meta("detect_rounds_total")
         return {
             "window": self.window,
             "request_seconds": round(time.time() - started, 2),
@@ -1779,6 +1896,32 @@ a:hover{text-decoration:underline}
 .notes div{font-size:12.5px;color:var(--ink-2);background:var(--surface-2);
   border:1px solid var(--line);border-radius:9px;padding:9px 12px;line-height:1.55}
 .hint{color:var(--muted);font-size:12px;margin-top:11px;line-height:1.7}
+.monitor-head{display:flex;align-items:center;gap:14px;margin:6px 0 18px}
+.monitor-title{font-size:21px;font-weight:600;color:var(--ink);margin-right:auto}
+.mgrid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));
+  gap:12px;margin-bottom:18px}
+@media (max-width:1100px){.mgrid{grid-template-columns:repeat(2,minmax(0,1fr))}}
+@media (max-width:700px){.mgrid{grid-template-columns:1fr}}
+.mcard{background:var(--surface);border:1px solid var(--line);
+  border-radius:14px;padding:15px 17px;box-shadow:0 1px 2px rgba(36,33,25,.03)}
+.mcard h3{font-size:11.5px;color:var(--muted);font-weight:600;
+  letter-spacing:.06em;margin:0 0 7px}
+.mbig{font-family:"Iowan Old Style",Palatino,Georgia,serif;font-size:27px;
+  color:var(--ink);line-height:1.15;margin-bottom:6px}
+.mbig.mok{color:#3E8E63}.mbig.mbad{color:#C0503A}.mbig.mwarn{color:var(--amber)}
+.mrow{display:flex;justify-content:space-between;gap:10px;font-size:12px;
+  color:var(--ink-2);margin-top:5px}
+.mrow span:last-child{color:var(--muted);text-align:right}
+.mtablewrap{overflow:auto;border:1px solid var(--line);border-radius:14px;
+  background:var(--surface);box-shadow:0 1px 2px rgba(36,33,25,.03)}
+.mtable{width:100%;border-collapse:collapse;font-size:12.5px}
+.mtable th{color:var(--muted);font-size:11.5px;font-weight:600;text-align:left;
+  padding:10px 14px;border-bottom:1px solid var(--line);background:var(--surface-2)}
+.mtable td{padding:9px 14px;border-bottom:1px solid #EFE9DD;color:var(--ink-2)}
+.mtable tr:last-child td{border-bottom:0}
+.nbadge{display:inline-block;padding:2px 9px;border-radius:99px;font-size:11px}
+.nbadge.ok{background:var(--green-soft,#E4EFE7);color:#2E6B4F}
+.nbadge.bad{background:var(--red-soft);color:#93402D}
 .pager{display:flex;align-items:center;justify-content:center;gap:18px;
   margin:18px 0 2px;color:var(--muted);font-size:12.5px;flex-wrap:wrap}
 .pg-total b{color:var(--ink);font-weight:600}
@@ -1895,6 +2038,7 @@ a:hover{text-decoration:underline}
   </section>
 
   <section id="detailView" hidden></section>
+  <section id="monitorView" hidden></section>
 </main>
 
 <div class="toast" id="toast"></div>
@@ -1956,7 +2100,7 @@ const RULE_TITLES = {
   bad_ua: '扫描器 UA', bot_ua: '爬虫 UA', not_found: '404 探测'
 };
 const state = { q: '', items: [], page: 1, pageSize: 20, total: 0, pages: 1,
-                 timer: null, inDetail: false, detailSamples: [] };
+                 timer: null, inDetail: false, inMonitor: false, detailSamples: [] };
 
 const modal = $('#modal');
 function openModal(title, text) {
@@ -2020,8 +2164,9 @@ async function loadStats() {
       <div class="stat cool"><div class="n">${fmt(s.fused_both)}</div><div class="k">双来源</div></div>
       <div class="stat"><div class="n">${fmt(s.fused_behavior_only)}</div><div class="k">行为新发现</div></div>`;
     $('#meta').innerHTML = `最近更新 <b>${localTime(s.updated_at)}</b><br>
+      <a href="#/monitor">监控</a> ·
       <a href="${BASE}/api/ips" target="_blank">JSON</a> ·
-      <a href="${BASE}/api/stats" target="_blank">stats</a>`;
+      <a href="${BASE}/api/metrics" target="_blank">metrics</a>`;
   } catch (err) {
     $('#meta').textContent = '统计加载失败: ' + err.message;
   }
@@ -2281,6 +2426,8 @@ function advice(item) {
 
 async function showDetail(ip) {
   state.inDetail = true;
+  state.inMonitor = false;
+  $('#monitorView').hidden = true;
   $('#listView').hidden = true;
   const view = $('#detailView');
   view.hidden = false;
@@ -2458,7 +2605,9 @@ async function showDetail(ip) {
 
 function showList() {
   state.inDetail = false;
+  state.inMonitor = false;
   document.title = '恶意 IP 清单';
+  $('#monitorView').hidden = true;
   $('#detailView').hidden = true;
   $('#detailView').innerHTML = '';
   $('#listView').hidden = false;
@@ -2467,7 +2616,101 @@ function showList() {
   loadList();
 }
 
+
+function monitorCard(title, value, rows, valueClass) {
+  const body = rows.map(row =>
+    `<div class="mrow"><span>${esc(row[0])}</span><span>${esc(row[1])}</span></div>`
+  ).join('');
+  return `<div class="mcard"><h3>${esc(title)}</h3>
+    <div class="mbig ${valueClass || ''}">${esc(value)}</div>${body}</div>`;
+}
+
+function monitorHtml(m) {
+  const d = m.detect, b = m.behavior, p = m.push, l = m.list, bf = m.backfill;
+  const stateText = m.graylog === 'ok' ? '正常' : m.graylog === 'error' ? '失败'
+                  : m.graylog === 'stale' ? '过期' : '未知';
+  const stateCls = m.graylog === 'ok' ? 'mok' : m.graylog === 'error' ? 'mbad'
+                 : 'mwarn';
+  const ratio = p.success_ratio_24h == null ? '-' :
+                (p.success_ratio_24h * 100).toFixed(1) + '%';
+  const cards = [
+    monitorCard('Graylog 连接', stateText, [
+      ['最近成功', d.last_success ? localTime(d.last_success) : '-'],
+      ['距上次成功', d.gap_seconds == null ? '-' : Math.round(d.gap_seconds) + ' 秒'],
+      ['最近失败', d.last_error ? d.last_error.slice(0, 24) : '无'],
+    ], stateCls),
+    monitorCard('特征检测', fmt(d.rounds) + ' 轮', [
+      ['失败次数', fmt(d.errors)],
+      ['最近耗时', d.last_duration ? d.last_duration + ' 秒' : '-'],
+      ['检测间隔', '60 秒'],
+    ]),
+    monitorCard('行为分析', fmt(b.runs) + ' 次', [
+      ['最近耗时', b.last_duration ? b.last_duration + ' 秒' : '-'],
+      ['最近违规', fmt(b.last_count)],
+      ['最近运行', b.last_success ? localTime(b.last_success) : '-'],
+    ], b.last_error ? 'mbad' : ''),
+    monitorCard('推送（24h）', ratio, [
+      ['成功 / 失败', fmt(p.sent_24h) + ' / ' + fmt(p.failed_24h)],
+      ['连续失败', fmt(p.consecutive_failures)],
+      ['最近错误', p.last_error ? p.last_error.slice(0, 24) : '无'],
+    ], (p.consecutive_failures > 0) ? 'mbad' : 'mok'),
+    monitorCard('名单', fmt(l.fused_total) + ' 个', [
+      ['今日新增', fmt(l.fused_new_24h)],
+      ['双来源', fmt(l.fused_both)],
+      ['行为新发现', fmt(l.fused_behavior_only)],
+    ]),
+    monitorCard('自动补测', fmt(bf.runs) + ' 次', [
+      ['上次补测', bf.last_at ? localTime(bf.last_at) : '无'],
+      ['补测区间', bf.last_from ? (localTime(bf.last_from) + ' → ' + localTime(bf.last_to)) : '-'],
+      ['上次新增', fmt(bf.last_added)],
+    ], bf.last_error ? 'mbad' : ''),
+  ].join('');
+  const rows = (m.recent_notifications || []).map(row => {
+    const cls = row.status === 'sent' ? 'ok' : 'bad';
+    return `<tr><td class="muted">${localTime(row.created_at)}</td>
+      <td>${esc(row.ip)}</td>
+      <td><span class="nbadge ${cls}">${row.status === 'sent' ? '成功' : '失败'}</span></td>
+      <td class="muted">${esc(row.error || '-')}</td></tr>`;
+  }).join('');
+  return `<div class="monitor-head">
+      <a class="back" href="#/">← 返回名单</a>
+      <div class="monitor-title">运行监控</div>
+      <span class="muted">每 30 秒刷新</span>
+    </div>
+    <div class="mgrid">${cards}</div>
+    <div class="mtablewrap">
+      <table class="mtable">
+        <thead><tr><th>时间</th><th>IP</th><th>结果</th><th>错误</th></tr></thead>
+        <tbody>${rows || '<tr><td colspan="4" class="empty">暂无推送记录</td></tr>'}</tbody>
+      </table>
+    </div>`;
+}
+
+async function showMonitor() {
+  state.inMonitor = true;
+  state.inDetail = false;
+  document.title = '运行监控';
+  $('#listView').hidden = true;
+  $('#detailView').hidden = true;
+  $('#monitorView').hidden = false;
+  loadStatus();
+  loadStats();
+  await loadMonitor();
+}
+
+async function loadMonitor() {
+  const view = $('#monitorView');
+  try {
+    const data = await jget(BASE + '/api/monitor');
+    view.innerHTML = monitorHtml(data);
+  } catch (err) {
+    view.innerHTML = `<a class="back" href="#/">← 返回名单</a>
+      <div class="empty">加载失败：${esc(err.message)}</div>`;
+  }
+}
+
 function route() {
+  if (location.hash === '#/monitor') { showMonitor(); return; }
   const match = location.hash.match(/^#\/ip\/(.+)$/);
   if (match) showDetail(decodeURIComponent(match[1]));
   else showList();
@@ -2524,7 +2767,8 @@ function startTimer() {
   clearInterval(state.timer);
   state.timer = setInterval(() => {
     loadStatus();
-    if (!state.inDetail) { loadStats(); loadList(); }
+    if (state.inMonitor) { loadMonitor(); }
+    else if (!state.inDetail) { loadStats(); loadList(); }
   }, 30000);
 }
 window.addEventListener('hashchange', route);
@@ -2584,6 +2828,133 @@ def make_handler(store, window, interval, detector=None, path_prefix=""):
                 return self._send(200, {"ok": True})
             if path == "/api/stats":
                 return self._send(200, store.stats())
+            if path in ("/metrics", "/api/monitor"):
+                stats = store.stats()
+                meta = store.meta_dict()
+                notif = store.notification_stats(24)
+                now_epoch = time.time()
+
+                def stamp_epoch(value):
+                    try:
+                        return datetime.strptime(
+                            value, "%Y-%m-%dT%H:%M:%SZ").replace(
+                                tzinfo=timezone.utc).timestamp()
+                    except (TypeError, ValueError):
+                        return None
+
+                last_success = stamp_epoch(meta.get("detect_last_success"))
+                last_behavior = stamp_epoch(meta.get("behavior_last_success"))
+                age = round(now_epoch - last_success, 1) if last_success else None
+                behavior_age = (round(now_epoch - last_behavior, 1)
+                                if last_behavior else None)
+                if age is None:
+                    graylog = "unknown"
+                elif meta.get("detect_last_error") and age > interval * 2:
+                    graylog = "error"
+                elif age <= max(120, interval * 2.5):
+                    graylog = "ok"
+                else:
+                    graylog = "stale"
+
+                payload = {
+                    "time": now_iso(),
+                    "graylog": graylog,
+                    "detect": {
+                        "rounds": int(meta.get("detect_rounds_total") or 0),
+                        "errors": int(meta.get("detect_errors_total") or 0),
+                        "last_success": meta.get("detect_last_success"),
+                        "gap_seconds": age,
+                        "last_duration": float(meta.get("detect_last_duration") or 0),
+                        "last_error": meta.get("detect_last_error") or "",
+                    },
+                    "behavior": {
+                        "runs": int(meta.get("behavior_runs_total") or 0),
+                        "errors": int(meta.get("behavior_errors_total") or 0),
+                        "last_success": meta.get("behavior_last_success"),
+                        "age_seconds": behavior_age,
+                        "last_duration": float(meta.get("behavior_last_duration") or 0),
+                        "last_count": int(meta.get("behavior_last_count") or 0),
+                        "last_error": meta.get("behavior_last_error") or "",
+                    },
+                    "push": {
+                        "sent_total": int(meta.get("push_sent_total") or 0),
+                        "failed_total": int(meta.get("push_failed_total") or 0),
+                        "sent_24h": notif["sent"],
+                        "failed_24h": notif["failed"],
+                        "success_ratio_24h": notif["success_ratio"],
+                        "consecutive_failures": int(
+                            meta.get("push_consecutive_failures") or 0),
+                        "last_error": meta.get("push_last_error") or "",
+                        "last_error_at": meta.get("push_last_error_at") or "",
+                    },
+                    "list": {
+                        "fused_total": stats.get("fused_total"),
+                        "fused_new_24h": stats.get("fused_new_24h"),
+                        "fused_both": stats.get("fused_both"),
+                        "fused_behavior_only": stats.get("fused_behavior_only"),
+                        "updated_at": stats.get("updated_at"),
+                    },
+                    "backfill": {
+                        "runs": int(meta.get("backfill_runs_total") or 0),
+                        "last_at": meta.get("backfill_last_at") or "",
+                        "last_from": meta.get("backfill_last_from") or "",
+                        "last_to": meta.get("backfill_last_to") or "",
+                        "last_added": int(meta.get("backfill_last_added") or 0),
+                        "last_error": meta.get("backfill_last_error") or "",
+                    },
+                    "recent_notifications": notif["recent"],
+                }
+                if path == "/api/monitor":
+                    return self._send(200, payload)
+
+                lines = [
+                    "# HELP ipsec_detect_rounds_total Successful detection rounds",
+                    "# TYPE ipsec_detect_rounds_total counter",
+                    "ipsec_detect_rounds_total %d" % payload["detect"]["rounds"],
+                    "# TYPE ipsec_detect_errors_total counter",
+                    "ipsec_detect_errors_total %d" % payload["detect"]["errors"],
+                    "# TYPE ipsec_detect_last_success_timestamp gauge",
+                    "ipsec_detect_last_success_timestamp %s"
+                    % (last_success or 0),
+                    "# TYPE ipsec_detect_gap_seconds gauge",
+                    "ipsec_detect_gap_seconds %s" % (age if age is not None else -1),
+                    "# TYPE ipsec_detect_last_duration_seconds gauge",
+                    "ipsec_detect_last_duration_seconds %s"
+                    % payload["detect"]["last_duration"],
+                    "# TYPE ipsec_behavior_runs_total counter",
+                    "ipsec_behavior_runs_total %d" % payload["behavior"]["runs"],
+                    "# TYPE ipsec_behavior_errors_total counter",
+                    "ipsec_behavior_errors_total %d" % payload["behavior"]["errors"],
+                    "# TYPE ipsec_behavior_last_duration_seconds gauge",
+                    "ipsec_behavior_last_duration_seconds %s"
+                    % payload["behavior"]["last_duration"],
+                    "# TYPE ipsec_behavior_last_violations gauge",
+                    "ipsec_behavior_last_violations %d"
+                    % payload["behavior"]["last_count"],
+                    "# TYPE ipsec_fused_total gauge",
+                    "ipsec_fused_total %s" % (payload["list"]["fused_total"] or 0),
+                    "# TYPE ipsec_fused_new_24h gauge",
+                    "ipsec_fused_new_24h %s" % (payload["list"]["fused_new_24h"] or 0),
+                    "# TYPE ipsec_fused_both gauge",
+                    "ipsec_fused_both %s" % (payload["list"]["fused_both"] or 0),
+                    "# TYPE ipsec_push_sent_total counter",
+                    "ipsec_push_sent_total %d" % payload["push"]["sent_total"],
+                    "# TYPE ipsec_push_failed_total counter",
+                    "ipsec_push_failed_total %d" % payload["push"]["failed_total"],
+                    "# TYPE ipsec_push_consecutive_failures gauge",
+                    "ipsec_push_consecutive_failures %d"
+                    % payload["push"]["consecutive_failures"],
+                    "# TYPE ipsec_push_success_ratio_24h gauge",
+                    "ipsec_push_success_ratio_24h %s"
+                    % (payload["push"]["success_ratio_24h"]
+                       if payload["push"]["success_ratio_24h"] is not None else -1),
+                    "# TYPE ipsec_graylog_up gauge",
+                    "ipsec_graylog_up %d" % (1 if graylog == "ok" else 0),
+                    "",
+                ]
+                return self._send(200, "\n".join(lines),
+                                  content_type="text/plain; version=0.0.4; charset=utf-8")
+
             if path == "/api/status":
                 status = {"graylog": "unknown", "last_run": None,
                           "last_error": None, "age_seconds": None}
@@ -2838,6 +3209,47 @@ def run(ctx, args):
     # Fused block list: existing feature detections first (historical rows keep
     # their original discovery time), then the behaviour thread keeps adding.
     store.sync_fused()
+    filter_str = ("streams:" + ctx.stream_id) if ctx.stream_id else None
+
+    def auto_backfill_worker(from_dt, to_dt):
+        from .backfill import auto_backfill_gap
+        summary = auto_backfill_gap(
+            store, ctx.client, fields, filter_str, from_dt, to_dt,
+            os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+        print("[自动补测] %s" % json.dumps(summary, ensure_ascii=False),
+              flush=True)
+
+    auto_enabled = not getattr(args, "no_auto_backfill", False)
+    threshold = int(getattr(args, "backfill_threshold", 1800) or 0)
+    max_gap = int(getattr(args, "backfill_max_gap", 7 * 86400) or 0)
+    if (auto_enabled and threshold > 0
+            and not getattr(args, "serve_only", False)):
+        from .backfill import parse_iso
+        last_meta = store.get_meta("detect_last_success")
+        if not last_meta:
+            row = store.conn.execute(
+                "SELECT MAX(updated_at) AS t FROM ip_stats").fetchone()
+            last_meta = row["t"] if row and row["t"] else None
+        last_dt = parse_iso(last_meta) if last_meta else None
+        if last_dt is not None:
+            now_dt = datetime.now(timezone.utc)
+            gap = (now_dt - last_dt).total_seconds()
+            if gap > max_gap:
+                print("检测缺口 %.1f 小时，超过自动补测上限 %d 小时，已跳过"
+                      % (gap / 3600.0, max_gap // 3600), flush=True)
+            elif gap > threshold:
+                print("检测缺口 %.1f 小时，后台自动补测 %s → %s"
+                      % (gap / 3600.0, last_dt.strftime("%m-%d %H:%M"),
+                         now_dt.astimezone().strftime("%m-%d %H:%M")),
+                      flush=True)
+                threading.Thread(target=auto_backfill_worker,
+                                 args=(last_dt, now_dt), daemon=True).start()
+            else:
+                print("检测连续（上次成功 %s），无需补测"
+                      % last_dt.strftime("%m-%d %H:%M"), flush=True)
+        else:
+            print("没有历史检测记录，跳过自动补测", flush=True)
+
     behavior_analyzer = None
     if not getattr(args, "no_behavior", False):
         behavior_analyzer = BehaviorAnalyzer(
@@ -2915,9 +3327,15 @@ def run(ctx, args):
                     print("  封禁名单: %d 个 -> %s" % (len(chosen), args.export), flush=True)
             except GraylogError as exc:
                 detector.last_error = str(exc)
+                detector.error_count += 1
+                detector.store.set_meta("detect_last_error", str(exc))
+                detector.store.incr_meta("detect_errors_total")
                 print("[%s] 检测失败: %s" % (now_iso(), exc), flush=True)
             except Exception as exc:
                 detector.last_error = str(exc)
+                detector.error_count += 1
+                detector.store.set_meta("detect_last_error", str(exc))
+                detector.store.incr_meta("detect_errors_total")
                 print("[%s] 检测异常: %s" % (now_iso(), exc), flush=True)
             time.sleep(max(5, int(args.interval)))
     except KeyboardInterrupt: