补测：行为检测支持历史区间；特征扫描自动时间切片；历史画像保留

tools/backfill_behavior.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Backfill behaviour detection for a historical range (service-off gap).
+
+Runs the independent behaviour analyzer against an absolute time range and
+stores flagged profiles with historical=1 so later live runs keep them.
+"""
+import argparse
+import datetime
+import json
+import os
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from graylog_ops import behavior, watch  # noqa: E402
+from graylog_ops.client import GraylogClient  # noqa: E402
+from graylog_ops.http import HttpClient  # noqa: E402
+from graylog_ops.judge import build_fields, load_profile  # noqa: E402
+
+STREAM_ID = "6ac83059a6878e26256fcacc"
+
+
+def parse_local(value):
+    text = value.strip()
+    if text.endswith("Z"):
+        stamp = datetime.datetime.fromisoformat(text.replace("Z", "+00:00"))
+    elif "T" in text and ("+" in text[10:] or text.endswith("00")):
+        stamp = datetime.datetime.fromisoformat(text)
+    else:
+        stamp = datetime.datetime.strptime(text, "%Y-%m-%d %H:%M").astimezone()
+    if stamp.tzinfo is None:
+        stamp = stamp.astimezone()
+    return stamp.astimezone(datetime.timezone.utc)
+
+
+def utc_iso(stamp):
+    return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--from", dest="from_ts", required=True)
+    parser.add_argument("--to", dest="to_ts", required=True)
+    parser.add_argument("--db", default=os.path.join(ROOT, "data", "ips.db"))
+    parser.add_argument("--profile", default=os.path.join(ROOT, "profiles", "f5.json"))
+    parser.add_argument("--config", default=os.path.join(ROOT, "graylog-ops.json"))
+    parser.add_argument("--stream", default=STREAM_ID)
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    from_dt, to_dt = parse_local(args.from_ts), parse_local(args.to_ts)
+    if to_dt <= from_dt:
+        raise SystemExit("结束时间必须晚于开始时间")
+
+    cfg = json.load(open(args.config, encoding="utf-8"))
+    client = GraylogClient(HttpClient(cfg["url"], username=cfg["token"],
+                                      password="token", timeout=120, retries=2))
+    fields = build_fields(load_profile(args.profile))
+    filter_str = "streams:" + args.stream
+
+    behavior.RANGE_OVERRIDE = {"from": utc_iso(from_dt), "to": utc_iso(to_dt)}
+    store = watch.Store(args.db)
+    analyzer = behavior.BehaviorAnalyzer(client, store, fields, filter_str,
+                                         root=ROOT, notifier=None)
+    try:
+        items = analyzer.analyze()
+    finally:
+        behavior.RANGE_OVERRIDE = None
+
+    flagged = [item for item in items if item.get("verdict") == "建议封禁"]
+    for item in items:
+        try:
+            last = client.search_absolute(
+                query='%s:"%s"' % (fields["client_ip"], item["ip"]),
+                from_ts=utc_iso(from_dt), to_ts=utc_iso(to_dt),
+                limit=1, sort="timestamp:desc", fields=["timestamp"],
+                filter_str=filter_str)
+            stamp = (last or [{}])[0].get("timestamp")
+            if stamp:
+                item["last_ts"] = datetime.datetime.fromisoformat(
+                    str(stamp).replace("Z", "+00:00")).astimezone(
+                        datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        except Exception as exc:
+            print("时间查询 %s 失败: %s" % (item["ip"], exc), file=sys.stderr)
+
+    print("行为违规 %d 个，建议封禁 %d 个" % (len(items), len(flagged)),
+          file=sys.stderr)
+    for item in sorted(flagged, key=lambda x: -x["score"])[:20]:
+        print("  %-16s 评分 %-5s %-10s %s" % (
+            item["ip"], item["score"], item.get("asn_brand") or "",
+            item.get("reason_text") or ""), file=sys.stderr)
+
+    if args.dry_run:
+        print(json.dumps({"violations": len(items),
+                          "flagged": len(flagged),
+                          "ips": [i["ip"] for i in flagged]},
+                         ensure_ascii=False, indent=2))
+        store.close()
+        return 0
+
+    store.upsert_historical_profiles(items)
+    before = {row["ip"] for row in store.conn.execute(
+        "SELECT ip FROM ip_fused")}
+    store.sync_fused()
+    added = [i["ip"] for i in flagged
+             if i["ip"] not in before]
+    stamp = watch.now_iso()
+    if added:
+        with store.lock, store.conn:
+            store.conn.executemany(
+                "UPDATE ip_fused SET last_pushed_at=? WHERE ip=?",
+                [(stamp, ip) for ip in added])
+    print(json.dumps({
+        "range": [utc_iso(from_dt), utc_iso(to_dt)],
+        "violations": len(items),
+        "flagged": len(flagged),
+        "newly_fused": len(added),
+        "new_ips": sorted(added),
+    }, ensure_ascii=False, indent=2))
+    store.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())