补测：行为检测支持历史区间；特征扫描自动时间切片；历史画像保留

tools/backfill_behavior.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Backfill behaviour detection for a historical range (service-off gap).
+
+Runs the independent behaviour analyzer against an absolute time range and
+stores flagged profiles with historical=1 so later live runs keep them.
+"""
+import argparse
+import datetime
+import json
+import os
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, ROOT)
+
+from graylog_ops import behavior, watch  # noqa: E402
+from graylog_ops.client import GraylogClient  # noqa: E402
+from graylog_ops.http import HttpClient  # noqa: E402
+from graylog_ops.judge import build_fields, load_profile  # noqa: E402
+
+STREAM_ID = "6ac83059a6878e26256fcacc"
+
+
+def parse_local(value):
+    text = value.strip()
+    if text.endswith("Z"):
+        stamp = datetime.datetime.fromisoformat(text.replace("Z", "+00:00"))
+    elif "T" in text and ("+" in text[10:] or text.endswith("00")):
+        stamp = datetime.datetime.fromisoformat(text)
+    else:
+        stamp = datetime.datetime.strptime(text, "%Y-%m-%d %H:%M").astimezone()
+    if stamp.tzinfo is None:
+        stamp = stamp.astimezone()
+    return stamp.astimezone(datetime.timezone.utc)
+
+
+def utc_iso(stamp):
+    return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--from", dest="from_ts", required=True)
+    parser.add_argument("--to", dest="to_ts", required=True)
+    parser.add_argument("--db", default=os.path.join(ROOT, "data", "ips.db"))
+    parser.add_argument("--profile", default=os.path.join(ROOT, "profiles", "f5.json"))
+    parser.add_argument("--config", default=os.path.join(ROOT, "graylog-ops.json"))
+    parser.add_argument("--stream", default=STREAM_ID)
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    from_dt, to_dt = parse_local(args.from_ts), parse_local(args.to_ts)
+    if to_dt <= from_dt:
+        raise SystemExit("结束时间必须晚于开始时间")
+
+    cfg = json.load(open(args.config, encoding="utf-8"))
+    client = GraylogClient(HttpClient(cfg["url"], username=cfg["token"],
+                                      password="token", timeout=120, retries=2))
+    fields = build_fields(load_profile(args.profile))
+    filter_str = "streams:" + args.stream
+
+    behavior.RANGE_OVERRIDE = {"from": utc_iso(from_dt), "to": utc_iso(to_dt)}
+    store = watch.Store(args.db)
+    analyzer = behavior.BehaviorAnalyzer(client, store, fields, filter_str,
+                                         root=ROOT, notifier=None)
+    try:
+        items = analyzer.analyze()
+    finally:
+        behavior.RANGE_OVERRIDE = None
+
+    flagged = [item for item in items if item.get("verdict") == "建议封禁"]
+    for item in items:
+        try:
+            last = client.search_absolute(
+                query='%s:"%s"' % (fields["client_ip"], item["ip"]),
+                from_ts=utc_iso(from_dt), to_ts=utc_iso(to_dt),
+                limit=1, sort="timestamp:desc", fields=["timestamp"],
+                filter_str=filter_str)
+            stamp = (last or [{}])[0].get("timestamp")
+            if stamp:
+                item["last_ts"] = datetime.datetime.fromisoformat(
+                    str(stamp).replace("Z", "+00:00")).astimezone(
+                        datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        except Exception as exc:
+            print("时间查询 %s 失败: %s" % (item["ip"], exc), file=sys.stderr)
+
+    print("行为违规 %d 个，建议封禁 %d 个" % (len(items), len(flagged)),
+          file=sys.stderr)
+    for item in sorted(flagged, key=lambda x: -x["score"])[:20]:
+        print("  %-16s 评分 %-5s %-10s %s" % (
+            item["ip"], item["score"], item.get("asn_brand") or "",
+            item.get("reason_text") or ""), file=sys.stderr)
+
+    if args.dry_run:
+        print(json.dumps({"violations": len(items),
+                          "flagged": len(flagged),
+                          "ips": [i["ip"] for i in flagged]},
+                         ensure_ascii=False, indent=2))
+        store.close()
+        return 0
+
+    store.upsert_historical_profiles(items)
+    before = {row["ip"] for row in store.conn.execute(
+        "SELECT ip FROM ip_fused")}
+    store.sync_fused()
+    added = [i["ip"] for i in flagged
+             if i["ip"] not in before]
+    stamp = watch.now_iso()
+    if added:
+        with store.lock, store.conn:
+            store.conn.executemany(
+                "UPDATE ip_fused SET last_pushed_at=? WHERE ip=?",
+                [(stamp, ip) for ip in added])
+    print(json.dumps({
+        "range": [utc_iso(from_dt), utc_iso(to_dt)],
+        "violations": len(items),
+        "flagged": len(flagged),
+        "newly_fused": len(added),
+        "new_ips": sorted(added),
+    }, ensure_ascii=False, indent=2))
+    store.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/backfill_gap.py

@@ -67,6 +67,8 @@ def main():
     parser.add_argument("--profile", default=os.path.join(ROOT, "profiles", "f5.json"))
     parser.add_argument("--config", default=os.path.join(ROOT, "graylog-ops.json"))
     parser.add_argument("--stream", default=STREAM_ID)
+    parser.add_argument("--size", type=int, default=10000,
+                        help="单次聚合最多取多少个 IP（超过会自动按时间切片）")
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
 
@@ -84,25 +86,35 @@ def main():
 
     counts = {}
     rule_totals = {}
+
+    def collect(rule, query, start, end):
+        """Query one time slice; split recursively when the terms cap is hit."""
+        result = client.terms(ip_field, query=query, size=args.size,
+                              from_ts=utc_iso(start), to_ts=utc_iso(end),
+                              filter_str=filter_str)
+        terms = result.get("terms") or []
+        if len(terms) >= args.size and (end - start).total_seconds() > 300:
+            middle = start + (end - start) / 2
+            return collect(rule, query, start, middle) + collect(rule, query, middle, end)
+        return [(str(item.get("term") or ""), int(item.get("count") or 0))
+                for item in terms if item.get("term")]
+
     for rule in watch.RULE_DEFS:
         if rule.get("weight", 0) <= 0 and rule["name"] != "not_found":
             continue
         try:
             query = rule["query"](fields)
-            result = client.terms(ip_field, query=query, size=2000,
-                                  from_ts=utc_iso(from_dt), to_ts=utc_iso(to_dt),
-                                  filter_str=filter_str)
+            pairs = collect(rule, query, from_dt, to_dt)
         except Exception as exc:  # keep going; one broken rule must not stop us
             print("规则 %s 查询失败: %s" % (rule["name"], exc), file=sys.stderr)
             continue
-        terms = result.get("terms") or []
-        rule_totals[rule["name"]] = sum(int(item.get("count") or 0) for item in terms)
-        for item in terms:
-            ip = str(item.get("term") or "")
-            if not ip:
-                continue
-            counts.setdefault(ip, {})[rule["name"]] = int(item.get("count") or 0)
-        print("规则 %-14s %6d 个 IP" % (rule["name"], len(terms)), file=sys.stderr)
+        merged = {}
+        for ip, count in pairs:
+            merged[ip] = merged.get(ip, 0) + count
+        rule_totals[rule["name"]] = sum(merged.values())
+        for ip, count in merged.items():
+            counts.setdefault(ip, {})[rule["name"]] = count
+        print("规则 %-14s %6d 个 IP" % (rule["name"], len(merged)), file=sys.stderr)
 
     now_iso = watch.now_iso()
     store = watch.Store(args.db)