#!/usr/bin/env python3 """Backfill feature detection for a historical time range. Scans the Graylog F5 stream for every signature rule over [--from, --to], scores the IPs exactly like the live detector and writes them into the local SQLite database. Backfilled rows are marked as already pushed so they do not flood the DingTalk webhook. """ import argparse import datetime import ipaddress import json import os import sys import time ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ROOT) from graylog_ops import watch # noqa: E402 from graylog_ops.asn import AsnDB # noqa: E402 from graylog_ops.client import GraylogClient # noqa: E402 from graylog_ops.geo import GeoIP # noqa: E402 from graylog_ops.http import HttpClient # noqa: E402 from graylog_ops.judge import build_fields, load_profile # noqa: E402 STREAM_ID = "6ac83059a6878e26256fcacc" def parse_local(value): text = value.strip() if text.endswith("Z"): stamp = datetime.datetime.fromisoformat(text.replace("Z", "+00:00")) elif "T" in text and ("+" in text[10:] or text.endswith("00")): stamp = datetime.datetime.fromisoformat(text) else: stamp = datetime.datetime.strptime(text, "%Y-%m-%d %H:%M").astimezone() if stamp.tzinfo is None: stamp = stamp.astimezone() return stamp.astimezone(datetime.timezone.utc) def utc_iso(stamp): return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z") def normalise_timestamp(value): text = str(value or "") if not text: return None try: stamp = datetime.datetime.fromisoformat(text.replace("Z", "+00:00")) except ValueError: return None return utc_iso(stamp.astimezone(datetime.timezone.utc)) def main(): parser = argparse.ArgumentParser() parser.add_argument("--from", dest="from_ts", required=True, help='开始时间，例如 "2026-10-18 04:00"') parser.add_argument("--to", dest="to_ts", required=True, help='结束时间，例如 "2026-10-19 03:55"') parser.add_argument("--db", default=os.path.join(ROOT, "data", "ips.db")) parser.add_argument("--geo-db", default=os.path.join(ROOT, "data", "geo", "geo.db")) parser.add_argument("--asn-db", default=os.path.join(ROOT, "data", "geo", "asn.db")) parser.add_argument("--profile", default=os.path.join(ROOT, "profiles", "f5.json")) parser.add_argument("--config", default=os.path.join(ROOT, "graylog-ops.json")) parser.add_argument("--stream", default=STREAM_ID) parser.add_argument("--size", type=int, default=10000, help="单次聚合最多取多少个 IP（超过会自动按时间切片）") parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() from_dt, to_dt = parse_local(args.from_ts), parse_local(args.to_ts) if to_dt <= from_dt: raise SystemExit("结束时间必须晚于开始时间") span = int((to_dt - from_dt).total_seconds()) cfg = json.load(open(args.config, encoding="utf-8")) client = GraylogClient(HttpClient(cfg["url"], username=cfg["token"], password="token", timeout=60, retries=2)) fields = build_fields(load_profile(args.profile)) ip_field = fields["client_ip"] filter_str = "streams:" + args.stream counts = {} rule_totals = {} def collect(rule, query, start, end): """Query one time slice; split recursively when the terms cap is hit.""" result = client.terms(ip_field, query=query, size=args.size, from_ts=utc_iso(start), to_ts=utc_iso(end), filter_str=filter_str) terms = result.get("terms") or [] if len(terms) >= args.size and (end - start).total_seconds() > 300: middle = start + (end - start) / 2 return collect(rule, query, start, middle) + collect(rule, query, middle, end) return [(str(item.get("term") or ""), int(item.get("count") or 0)) for item in terms if item.get("term")] for rule in watch.RULE_DEFS: if rule.get("weight", 0) <= 0 and rule["name"] != "not_found": continue try: query = rule["query"](fields) pairs = collect(rule, query, from_dt, to_dt) except Exception as exc: # keep going; one broken rule must not stop us print("规则 %s 查询失败: %s" % (rule["name"], exc), file=sys.stderr) continue merged = {} for ip, count in pairs: merged[ip] = merged.get(ip, 0) + count rule_totals[rule["name"]] = sum(merged.values()) for ip, count in merged.items(): counts.setdefault(ip, {})[rule["name"]] = count print("规则 %-14s %6d 个 IP" % (rule["name"], len(merged)), file=sys.stderr) now_iso = watch.now_iso() store = watch.Store(args.db) geo = GeoIP(args.geo_db) if args.geo_db and os.path.exists(args.geo_db) else None asn = AsnDB(args.asn_db) if args.asn_db and os.path.exists(args.asn_db) else None candidates = [] for ip, hits in counts.items(): try: addr = ipaddress.ip_address(ip) except ValueError: continue if not addr.is_global: continue total = sum(hits.values()) tier, score, reasons = watch.classify(total, hits, window_scale=span / 900.0) if tier != watch.TIER_MALICIOUS: continue candidates.append({"ip": ip, "hits": hits, "total": total, "tier": tier, "score": score, "reasons": reasons}) candidates.sort(key=lambda item: (-item["score"], -item["total"], item["ip"])) created = upgraded = extended = 0 touched = [] for item in candidates: ip = item["ip"] first = last = None try: asc = client.search_absolute(query='%s:"%s"' % (ip_field, ip), from_ts=utc_iso(from_dt), to_ts=utc_iso(to_dt), limit=1, sort="timestamp:asc", fields=["timestamp"], filter_str=filter_str) desc = client.search_absolute(query='%s:"%s"' % (ip_field, ip), from_ts=utc_iso(from_dt), to_ts=utc_iso(to_dt), limit=1, sort="timestamp:desc", fields=["timestamp"], filter_str=filter_str) first = normalise_timestamp((asc or [{}])[0].get("timestamp")) last = normalise_timestamp((desc or [{}])[0].get("timestamp")) except Exception as exc: print("时间查询 %s 失败: %s" % (ip, exc), file=sys.stderr) first = first or utc_iso(from_dt) last = last or utc_iso(to_dt) record = { "ip": ip, "tier": item["tier"], "score": item["score"], "total": item["total"], "rules": item["hits"], "reasons": item["reasons"], "first_seen": first, "last_seen": last, "last_malicious_at": last, "risk": "high" if item["score"] >= 85 else "medium", "window": span, "geo": geo.lookup(ip) if geo else {}, "asn": asn.lookup(ip) if asn else {}, } if args.dry_run: candidates[0] = candidates[0] # no-op print("[dry-run] %-16s 评分 %5.1f 命中 %d 次 规则 %s 首次 %s" % (ip, item["score"], item["total"], ",".join(item["hits"]), first), file=sys.stderr) continue existing = store.get(ip) if existing is None: store.upsert(record, now_iso) created += 1 touched.append(ip) elif existing.get("tier") == watch.TIER_MALICIOUS: with store.lock, store.conn: store.conn.execute( "UPDATE ip_stats SET first_seen=?, last_seen=?, last_malicious_at=?, " "ever_malicious=1 WHERE ip=?", (min(first, existing.get("first_seen") or first), max(last, existing.get("last_seen") or last), max(last, existing.get("last_malicious_at") or last), ip)) extended += 1 touched.append(ip) elif float(existing.get("score") or 0) < item["score"]: store.upsert(record, now_iso) upgraded += 1 touched.append(ip) else: extended += 1 touched.append(ip) if not args.dry_run and touched: store.sync_fused() stamp = watch.now_iso() with store.lock, store.conn: store.conn.executemany( "UPDATE ip_fused SET last_pushed_at=? WHERE ip=?", [(stamp, ip) for ip in touched]) print(json.dumps({ "range": [utc_iso(from_dt), utc_iso(to_dt)], "span_hours": round(span / 3600.0, 1), "candidates": len(candidates), "created": created, "upgraded": upgraded, "extended": extended, "rule_totals": rule_totals, "top": [{"ip": c["ip"], "score": c["score"], "total": c["total"], "rules": c["hits"]} for c in candidates[:15]], }, ensure_ascii=False, indent=2)) store.close() return 0 if __name__ == "__main__": raise SystemExit(main())