补测：行为检测支持历史区间；特征扫描自动时间切片；历史画像保留

tools/backfill_gap.py

@@ -67,6 +67,8 @@ def main():
     parser.add_argument("--profile", default=os.path.join(ROOT, "profiles", "f5.json"))
     parser.add_argument("--config", default=os.path.join(ROOT, "graylog-ops.json"))
     parser.add_argument("--stream", default=STREAM_ID)
+    parser.add_argument("--size", type=int, default=10000,
+                        help="单次聚合最多取多少个 IP（超过会自动按时间切片）")
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
 
@@ -84,25 +86,35 @@ def main():
 
     counts = {}
     rule_totals = {}
+
+    def collect(rule, query, start, end):
+        """Query one time slice; split recursively when the terms cap is hit."""
+        result = client.terms(ip_field, query=query, size=args.size,
+                              from_ts=utc_iso(start), to_ts=utc_iso(end),
+                              filter_str=filter_str)
+        terms = result.get("terms") or []
+        if len(terms) >= args.size and (end - start).total_seconds() > 300:
+            middle = start + (end - start) / 2
+            return collect(rule, query, start, middle) + collect(rule, query, middle, end)
+        return [(str(item.get("term") or ""), int(item.get("count") or 0))
+                for item in terms if item.get("term")]
+
     for rule in watch.RULE_DEFS:
         if rule.get("weight", 0) <= 0 and rule["name"] != "not_found":
             continue
         try:
             query = rule["query"](fields)
-            result = client.terms(ip_field, query=query, size=2000,
-                                  from_ts=utc_iso(from_dt), to_ts=utc_iso(to_dt),
-                                  filter_str=filter_str)
+            pairs = collect(rule, query, from_dt, to_dt)
         except Exception as exc:  # keep going; one broken rule must not stop us
             print("规则 %s 查询失败: %s" % (rule["name"], exc), file=sys.stderr)
             continue
-        terms = result.get("terms") or []
-        rule_totals[rule["name"]] = sum(int(item.get("count") or 0) for item in terms)
-        for item in terms:
-            ip = str(item.get("term") or "")
-            if not ip:
-                continue
-            counts.setdefault(ip, {})[rule["name"]] = int(item.get("count") or 0)
-        print("规则 %-14s %6d 个 IP" % (rule["name"], len(terms)), file=sys.stderr)
+        merged = {}
+        for ip, count in pairs:
+            merged[ip] = merged.get(ip, 0) + count
+        rule_totals[rule["name"]] = sum(merged.values())
+        for ip, count in merged.items():
+            counts.setdefault(ip, {})[rule["name"]] = count
+        print("规则 %-14s %6d 个 IP" % (rule["name"], len(merged)), file=sys.stderr)
 
     now_iso = watch.now_iso()
     store = watch.Store(args.db)