"""Backfill detection for a historical time window. Used by the startup auto-backfill thread and by tools/backfill_*.py. """ import datetime import ipaddress import time from . import behavior, watch from .asn import AsnDB from .geo import GeoIP def parse_iso(value): try: return datetime.datetime.fromisoformat( str(value).replace("Z", "+00:00")).astimezone(datetime.timezone.utc) except (TypeError, ValueError): return None def iso(stamp): return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z") def _suppress(store, ips): if not ips: return stamp = watch.now_iso() with store.lock, store.conn: store.conn.executemany( "UPDATE ip_fused SET last_pushed_at=?, push_fail_count=0, " "push_next_at=NULL WHERE ip=?", [(stamp, ip) for ip in ips]) def feature_backfill(client, store, fields, filter_str, from_dt, to_dt, geo=None, asn=None, size=10000): """Signature-rule backfill with recursive time slicing.""" ip_field = fields["client_ip"] span = int((to_dt - from_dt).total_seconds()) counts, rule_totals = {}, {} def collect(query, start, end): result = client.terms(ip_field, query=query, size=size, from_ts=iso(start), to_ts=iso(end), filter_str=filter_str) terms = result.get("terms") or [] if len(terms) >= size and (end - start).total_seconds() > 300: middle = start + (end - start) / 2 return collect(query, start, middle) + collect(query, middle, end) return [(str(item.get("term") or ""), int(item.get("count") or 0)) for item in terms if item.get("term")] for rule in watch.RULE_DEFS: if rule.get("weight", 0) <= 0 and rule["name"] != "not_found": continue try: pairs = collect(rule["query"](fields), from_dt, to_dt) except Exception: continue merged = {} for ip, count in pairs: merged[ip] = merged.get(ip, 0) + count rule_totals[rule["name"]] = sum(merged.values()) for ip, count in merged.items(): counts.setdefault(ip, {})[rule["name"]] = count now_iso = watch.now_iso() candidates = [] for ip, hits in counts.items(): try: if not ipaddress.ip_address(ip).is_global: continue except ValueError: continue total = sum(hits.values()) tier, score, reasons = watch.classify(total, hits, window_scale=span / 900.0) if tier != watch.TIER_MALICIOUS: continue candidates.append({"ip": ip, "hits": hits, "total": total, "score": score, "reasons": reasons}) candidates.sort(key=lambda item: (-item["score"], -item["total"], item["ip"])) touched = [] created = upgraded = extended = 0 for item in candidates: ip = item["ip"] first = last = None try: asc = client.search_absolute(query='%s:"%s"' % (ip_field, ip), from_ts=iso(from_dt), to_ts=iso(to_dt), limit=1, sort="timestamp:asc", fields=["timestamp"], filter_str=filter_str) desc = client.search_absolute(query='%s:"%s"' % (ip_field, ip), from_ts=iso(from_dt), to_ts=iso(to_dt), limit=1, sort="timestamp:desc", fields=["timestamp"], filter_str=filter_str) first = (asc or [{}])[0].get("timestamp") last = (desc or [{}])[0].get("timestamp") first = iso(parse_iso(first)) if parse_iso(first) else None last = iso(parse_iso(last)) if parse_iso(last) else None except Exception: pass first = first or iso(from_dt) last = last or iso(to_dt) record = { "ip": ip, "tier": watch.TIER_MALICIOUS, "score": item["score"], "total": item["total"], "rules": item["hits"], "reasons": item["reasons"], "first_seen": first, "last_seen": last, "last_malicious_at": last, "risk": "high" if item["score"] >= 85 else "medium", "window": span, "geo": geo.lookup(ip) if geo else {}, "asn": asn.lookup(ip) if asn else {}, } existing = store.get(ip) if existing is None: store.upsert(record, now_iso) created += 1 touched.append(ip) elif existing.get("tier") == watch.TIER_MALICIOUS: with store.lock, store.conn: store.conn.execute( "UPDATE ip_stats SET first_seen=?, last_seen=?, " "last_malicious_at=?, ever_malicious=1 WHERE ip=?", (min(first, existing.get("first_seen") or first), max(last, existing.get("last_seen") or last), max(last, existing.get("last_malicious_at") or last), ip)) extended += 1 touched.append(ip) elif float(existing.get("score") or 0) < item["score"]: store.upsert(record, now_iso) upgraded += 1 touched.append(ip) else: extended += 1 touched.append(ip) if touched: store.sync_fused() _suppress(store, touched) with store.lock, store.conn: store.conn.executemany( """UPDATE ip_fused SET first_banned_at = ( SELECT s.first_seen FROM ip_stats s WHERE s.ip = ip_fused.ip) WHERE ip=? AND EXISTS ( SELECT 1 FROM ip_stats s WHERE s.ip = ip_fused.ip AND s.first_seen < ip_fused.first_banned_at)""", [(ip,) for ip in touched]) return {"candidates": len(candidates), "created": created, "upgraded": upgraded, "extended": extended, "rule_totals": rule_totals, "ips": [item["ip"] for item in candidates]} def behavior_backfill(client, store, fields, filter_str, from_dt, to_dt, root, geo=None, asn=None): """Run the independent behaviour analyzer over an absolute range.""" behavior.set_range(iso(from_dt), iso(to_dt)) try: analyzer = behavior.BehaviorAnalyzer(client, store, fields, filter_str, root=root, notifier=None) items = analyzer.analyze() finally: behavior.clear_range() for item in items: try: last = client.search_absolute( query='%s:"%s"' % (fields["client_ip"], item["ip"]), from_ts=iso(from_dt), to_ts=iso(to_dt), limit=1, sort="timestamp:desc", fields=["timestamp"], filter_str=filter_str) stamp = (last or [{}])[0].get("timestamp") parsed = parse_iso(stamp) if parsed: item["last_ts"] = parsed.strftime("%Y-%m-%dT%H:%M:%SZ") except Exception: pass flagged = [item for item in items if item.get("verdict") == "建议封禁"] store.upsert_historical_profiles(items) before = {row["ip"] for row in store.conn.execute("SELECT ip FROM ip_fused")} store.sync_fused() added = [item["ip"] for item in flagged if item["ip"] not in before] _suppress(store, added) return {"violations": len(items), "flagged": len(flagged), "newly_fused": len(added), "new_ips": sorted(added)} def auto_backfill_gap(store, client, fields, filter_str, from_dt, to_dt, root, behavior_enabled=True): started = time.time() geo = asn = None try: import os geo_path = os.path.join(root, "data", "geo", "geo.db") asn_path = os.path.join(root, "data", "geo", "asn.db") if os.path.exists(geo_path): geo = GeoIP(geo_path) if os.path.exists(asn_path): asn = AsnDB(asn_path) except Exception: pass summary = {"from": iso(from_dt), "to": iso(to_dt)} try: summary["feature"] = feature_backfill( client, store, fields, filter_str, from_dt, to_dt, geo=geo, asn=asn) gap_hours = (to_dt - from_dt).total_seconds() / 3600.0 if behavior_enabled and 1 <= gap_hours <= 48: summary["behavior"] = behavior_backfill( client, store, fields, filter_str, from_dt, to_dt, root, geo=geo, asn=asn) store.incr_meta("backfill_runs_total") store.set_meta("backfill_last_at", watch.now_iso()) store.set_meta("backfill_last_from", iso(from_dt)) store.set_meta("backfill_last_to", iso(to_dt)) store.set_meta("backfill_last_added", len(summary["feature"].get("ips") or [])) store.set_meta("backfill_last_error", "") except Exception as exc: summary["error"] = str(exc) store.set_meta("backfill_last_error", str(exc)) summary["seconds"] = round(time.time() - started, 1) return summary