自动补测、推送审计/退避重试、监控指标与监控页面

graylog_ops/backfill.py

@@ -0,0 +1,224 @@
+"""Backfill detection for a historical time window.
+
+Used by the startup auto-backfill thread and by tools/backfill_*.py.
+"""
+import datetime
+import ipaddress
+import time
+
+from . import behavior, watch
+from .asn import AsnDB
+from .geo import GeoIP
+
+
+def parse_iso(value):
+    try:
+        return datetime.datetime.fromisoformat(
+            str(value).replace("Z", "+00:00")).astimezone(datetime.timezone.utc)
+    except (TypeError, ValueError):
+        return None
+
+
+def iso(stamp):
+    return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+
+def _suppress(store, ips):
+    if not ips:
+        return
+    stamp = watch.now_iso()
+    with store.lock, store.conn:
+        store.conn.executemany(
+            "UPDATE ip_fused SET last_pushed_at=?, push_fail_count=0, "
+            "push_next_at=NULL WHERE ip=?", [(stamp, ip) for ip in ips])
+
+
+def feature_backfill(client, store, fields, filter_str, from_dt, to_dt,
+                     geo=None, asn=None, size=10000):
+    """Signature-rule backfill with recursive time slicing."""
+    ip_field = fields["client_ip"]
+    span = int((to_dt - from_dt).total_seconds())
+    counts, rule_totals = {}, {}
+
+    def collect(query, start, end):
+        result = client.terms(ip_field, query=query, size=size,
+                              from_ts=iso(start), to_ts=iso(end),
+                              filter_str=filter_str)
+        terms = result.get("terms") or []
+        if len(terms) >= size and (end - start).total_seconds() > 300:
+            middle = start + (end - start) / 2
+            return collect(query, start, middle) + collect(query, middle, end)
+        return [(str(item.get("term") or ""), int(item.get("count") or 0))
+                for item in terms if item.get("term")]
+
+    for rule in watch.RULE_DEFS:
+        if rule.get("weight", 0) <= 0 and rule["name"] != "not_found":
+            continue
+        try:
+            pairs = collect(rule["query"](fields), from_dt, to_dt)
+        except Exception:
+            continue
+        merged = {}
+        for ip, count in pairs:
+            merged[ip] = merged.get(ip, 0) + count
+        rule_totals[rule["name"]] = sum(merged.values())
+        for ip, count in merged.items():
+            counts.setdefault(ip, {})[rule["name"]] = count
+
+    now_iso = watch.now_iso()
+    candidates = []
+    for ip, hits in counts.items():
+        try:
+            if not ipaddress.ip_address(ip).is_global:
+                continue
+        except ValueError:
+            continue
+        total = sum(hits.values())
+        tier, score, reasons = watch.classify(total, hits, window_scale=span / 900.0)
+        if tier != watch.TIER_MALICIOUS:
+            continue
+        candidates.append({"ip": ip, "hits": hits, "total": total,
+                           "score": score, "reasons": reasons})
+
+    candidates.sort(key=lambda item: (-item["score"], -item["total"], item["ip"]))
+    touched = []
+    created = upgraded = extended = 0
+    for item in candidates:
+        ip = item["ip"]
+        first = last = None
+        try:
+            asc = client.search_absolute(query='%s:"%s"' % (ip_field, ip),
+                                         from_ts=iso(from_dt), to_ts=iso(to_dt),
+                                         limit=1, sort="timestamp:asc",
+                                         fields=["timestamp"], filter_str=filter_str)
+            desc = client.search_absolute(query='%s:"%s"' % (ip_field, ip),
+                                          from_ts=iso(from_dt), to_ts=iso(to_dt),
+                                          limit=1, sort="timestamp:desc",
+                                          fields=["timestamp"], filter_str=filter_str)
+            first = (asc or [{}])[0].get("timestamp")
+            last = (desc or [{}])[0].get("timestamp")
+            first = iso(parse_iso(first)) if parse_iso(first) else None
+            last = iso(parse_iso(last)) if parse_iso(last) else None
+        except Exception:
+            pass
+        first = first or iso(from_dt)
+        last = last or iso(to_dt)
+        record = {
+            "ip": ip, "tier": watch.TIER_MALICIOUS, "score": item["score"],
+            "total": item["total"], "rules": item["hits"],
+            "reasons": item["reasons"], "first_seen": first, "last_seen": last,
+            "last_malicious_at": last,
+            "risk": "high" if item["score"] >= 85 else "medium",
+            "window": span,
+            "geo": geo.lookup(ip) if geo else {},
+            "asn": asn.lookup(ip) if asn else {},
+        }
+        existing = store.get(ip)
+        if existing is None:
+            store.upsert(record, now_iso)
+            created += 1
+            touched.append(ip)
+        elif existing.get("tier") == watch.TIER_MALICIOUS:
+            with store.lock, store.conn:
+                store.conn.execute(
+                    "UPDATE ip_stats SET first_seen=?, last_seen=?, "
+                    "last_malicious_at=?, ever_malicious=1 WHERE ip=?",
+                    (min(first, existing.get("first_seen") or first),
+                     max(last, existing.get("last_seen") or last),
+                     max(last, existing.get("last_malicious_at") or last), ip))
+            extended += 1
+            touched.append(ip)
+        elif float(existing.get("score") or 0) < item["score"]:
+            store.upsert(record, now_iso)
+            upgraded += 1
+            touched.append(ip)
+        else:
+            extended += 1
+            touched.append(ip)
+
+    if touched:
+        store.sync_fused()
+        _suppress(store, touched)
+        with store.lock, store.conn:
+            store.conn.executemany(
+                """UPDATE ip_fused SET first_banned_at = (
+                       SELECT s.first_seen FROM ip_stats s WHERE s.ip = ip_fused.ip)
+                   WHERE ip=? AND EXISTS (
+                       SELECT 1 FROM ip_stats s WHERE s.ip = ip_fused.ip
+                          AND s.first_seen < ip_fused.first_banned_at)""",
+                [(ip,) for ip in touched])
+
+    return {"candidates": len(candidates), "created": created,
+            "upgraded": upgraded, "extended": extended,
+            "rule_totals": rule_totals,
+            "ips": [item["ip"] for item in candidates]}
+
+
+def behavior_backfill(client, store, fields, filter_str, from_dt, to_dt,
+                      root, geo=None, asn=None):
+    """Run the independent behaviour analyzer over an absolute range."""
+    behavior.set_range(iso(from_dt), iso(to_dt))
+    try:
+        analyzer = behavior.BehaviorAnalyzer(client, store, fields, filter_str,
+                                             root=root, notifier=None)
+        items = analyzer.analyze()
+    finally:
+        behavior.clear_range()
+    for item in items:
+        try:
+            last = client.search_absolute(
+                query='%s:"%s"' % (fields["client_ip"], item["ip"]),
+                from_ts=iso(from_dt), to_ts=iso(to_dt), limit=1,
+                sort="timestamp:desc", fields=["timestamp"],
+                filter_str=filter_str)
+            stamp = (last or [{}])[0].get("timestamp")
+            parsed = parse_iso(stamp)
+            if parsed:
+                item["last_ts"] = parsed.strftime("%Y-%m-%dT%H:%M:%SZ")
+        except Exception:
+            pass
+    flagged = [item for item in items if item.get("verdict") == "建议封禁"]
+    store.upsert_historical_profiles(items)
+    before = {row["ip"] for row in store.conn.execute("SELECT ip FROM ip_fused")}
+    store.sync_fused()
+    added = [item["ip"] for item in flagged if item["ip"] not in before]
+    _suppress(store, added)
+    return {"violations": len(items), "flagged": len(flagged),
+            "newly_fused": len(added), "new_ips": sorted(added)}
+
+
+def auto_backfill_gap(store, client, fields, filter_str, from_dt, to_dt,
+                      root, behavior_enabled=True):
+    started = time.time()
+    geo = asn = None
+    try:
+        import os
+        geo_path = os.path.join(root, "data", "geo", "geo.db")
+        asn_path = os.path.join(root, "data", "geo", "asn.db")
+        if os.path.exists(geo_path):
+            geo = GeoIP(geo_path)
+        if os.path.exists(asn_path):
+            asn = AsnDB(asn_path)
+    except Exception:
+        pass
+    summary = {"from": iso(from_dt), "to": iso(to_dt)}
+    try:
+        summary["feature"] = feature_backfill(
+            client, store, fields, filter_str, from_dt, to_dt, geo=geo, asn=asn)
+        gap_hours = (to_dt - from_dt).total_seconds() / 3600.0
+        if behavior_enabled and 1 <= gap_hours <= 48:
+            summary["behavior"] = behavior_backfill(
+                client, store, fields, filter_str, from_dt, to_dt, root,
+                geo=geo, asn=asn)
+        store.incr_meta("backfill_runs_total")
+        store.set_meta("backfill_last_at", watch.now_iso())
+        store.set_meta("backfill_last_from", iso(from_dt))
+        store.set_meta("backfill_last_to", iso(to_dt))
+        store.set_meta("backfill_last_added",
+                       len(summary["feature"].get("ips") or []))
+        store.set_meta("backfill_last_error", "")
+    except Exception as exc:
+        summary["error"] = str(exc)
+        store.set_meta("backfill_last_error", str(exc))
+    summary["seconds"] = round(time.time() - started, 1)
+    return summary