225 lines
9.0 KiB
Python
225 lines
9.0 KiB
Python
"""Backfill detection for a historical time window.
|
|
|
|
Used by the startup auto-backfill thread and by tools/backfill_*.py.
|
|
"""
|
|
import datetime
|
|
import ipaddress
|
|
import time
|
|
|
|
from . import behavior, watch
|
|
from .asn import AsnDB
|
|
from .geo import GeoIP
|
|
|
|
|
|
def parse_iso(value):
|
|
try:
|
|
return datetime.datetime.fromisoformat(
|
|
str(value).replace("Z", "+00:00")).astimezone(datetime.timezone.utc)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def iso(stamp):
|
|
return stamp.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
|
|
|
|
|
def _suppress(store, ips):
|
|
if not ips:
|
|
return
|
|
stamp = watch.now_iso()
|
|
with store.lock, store.conn:
|
|
store.conn.executemany(
|
|
"UPDATE ip_fused SET last_pushed_at=?, push_fail_count=0, "
|
|
"push_next_at=NULL WHERE ip=?", [(stamp, ip) for ip in ips])
|
|
|
|
|
|
def feature_backfill(client, store, fields, filter_str, from_dt, to_dt,
|
|
geo=None, asn=None, size=10000):
|
|
"""Signature-rule backfill with recursive time slicing."""
|
|
ip_field = fields["client_ip"]
|
|
span = int((to_dt - from_dt).total_seconds())
|
|
counts, rule_totals = {}, {}
|
|
|
|
def collect(query, start, end):
|
|
result = client.terms(ip_field, query=query, size=size,
|
|
from_ts=iso(start), to_ts=iso(end),
|
|
filter_str=filter_str)
|
|
terms = result.get("terms") or []
|
|
if len(terms) >= size and (end - start).total_seconds() > 300:
|
|
middle = start + (end - start) / 2
|
|
return collect(query, start, middle) + collect(query, middle, end)
|
|
return [(str(item.get("term") or ""), int(item.get("count") or 0))
|
|
for item in terms if item.get("term")]
|
|
|
|
for rule in watch.RULE_DEFS:
|
|
if rule.get("weight", 0) <= 0 and rule["name"] != "not_found":
|
|
continue
|
|
try:
|
|
pairs = collect(rule["query"](fields), from_dt, to_dt)
|
|
except Exception:
|
|
continue
|
|
merged = {}
|
|
for ip, count in pairs:
|
|
merged[ip] = merged.get(ip, 0) + count
|
|
rule_totals[rule["name"]] = sum(merged.values())
|
|
for ip, count in merged.items():
|
|
counts.setdefault(ip, {})[rule["name"]] = count
|
|
|
|
now_iso = watch.now_iso()
|
|
candidates = []
|
|
for ip, hits in counts.items():
|
|
try:
|
|
if not ipaddress.ip_address(ip).is_global:
|
|
continue
|
|
except ValueError:
|
|
continue
|
|
total = sum(hits.values())
|
|
tier, score, reasons = watch.classify(total, hits, window_scale=span / 900.0)
|
|
if tier != watch.TIER_MALICIOUS:
|
|
continue
|
|
candidates.append({"ip": ip, "hits": hits, "total": total,
|
|
"score": score, "reasons": reasons})
|
|
|
|
candidates.sort(key=lambda item: (-item["score"], -item["total"], item["ip"]))
|
|
touched = []
|
|
created = upgraded = extended = 0
|
|
for item in candidates:
|
|
ip = item["ip"]
|
|
first = last = None
|
|
try:
|
|
asc = client.search_absolute(query='%s:"%s"' % (ip_field, ip),
|
|
from_ts=iso(from_dt), to_ts=iso(to_dt),
|
|
limit=1, sort="timestamp:asc",
|
|
fields=["timestamp"], filter_str=filter_str)
|
|
desc = client.search_absolute(query='%s:"%s"' % (ip_field, ip),
|
|
from_ts=iso(from_dt), to_ts=iso(to_dt),
|
|
limit=1, sort="timestamp:desc",
|
|
fields=["timestamp"], filter_str=filter_str)
|
|
first = (asc or [{}])[0].get("timestamp")
|
|
last = (desc or [{}])[0].get("timestamp")
|
|
first = iso(parse_iso(first)) if parse_iso(first) else None
|
|
last = iso(parse_iso(last)) if parse_iso(last) else None
|
|
except Exception:
|
|
pass
|
|
first = first or iso(from_dt)
|
|
last = last or iso(to_dt)
|
|
record = {
|
|
"ip": ip, "tier": watch.TIER_MALICIOUS, "score": item["score"],
|
|
"total": item["total"], "rules": item["hits"],
|
|
"reasons": item["reasons"], "first_seen": first, "last_seen": last,
|
|
"last_malicious_at": last,
|
|
"risk": "high" if item["score"] >= 85 else "medium",
|
|
"window": span,
|
|
"geo": geo.lookup(ip) if geo else {},
|
|
"asn": asn.lookup(ip) if asn else {},
|
|
}
|
|
existing = store.get(ip)
|
|
if existing is None:
|
|
store.upsert(record, now_iso)
|
|
created += 1
|
|
touched.append(ip)
|
|
elif existing.get("tier") == watch.TIER_MALICIOUS:
|
|
with store.lock, store.conn:
|
|
store.conn.execute(
|
|
"UPDATE ip_stats SET first_seen=?, last_seen=?, "
|
|
"last_malicious_at=?, ever_malicious=1 WHERE ip=?",
|
|
(min(first, existing.get("first_seen") or first),
|
|
max(last, existing.get("last_seen") or last),
|
|
max(last, existing.get("last_malicious_at") or last), ip))
|
|
extended += 1
|
|
touched.append(ip)
|
|
elif float(existing.get("score") or 0) < item["score"]:
|
|
store.upsert(record, now_iso)
|
|
upgraded += 1
|
|
touched.append(ip)
|
|
else:
|
|
extended += 1
|
|
touched.append(ip)
|
|
|
|
if touched:
|
|
store.sync_fused()
|
|
_suppress(store, touched)
|
|
with store.lock, store.conn:
|
|
store.conn.executemany(
|
|
"""UPDATE ip_fused SET first_banned_at = (
|
|
SELECT s.first_seen FROM ip_stats s WHERE s.ip = ip_fused.ip)
|
|
WHERE ip=? AND EXISTS (
|
|
SELECT 1 FROM ip_stats s WHERE s.ip = ip_fused.ip
|
|
AND s.first_seen < ip_fused.first_banned_at)""",
|
|
[(ip,) for ip in touched])
|
|
|
|
return {"candidates": len(candidates), "created": created,
|
|
"upgraded": upgraded, "extended": extended,
|
|
"rule_totals": rule_totals,
|
|
"ips": [item["ip"] for item in candidates]}
|
|
|
|
|
|
def behavior_backfill(client, store, fields, filter_str, from_dt, to_dt,
|
|
root, geo=None, asn=None):
|
|
"""Run the independent behaviour analyzer over an absolute range."""
|
|
behavior.set_range(iso(from_dt), iso(to_dt))
|
|
try:
|
|
analyzer = behavior.BehaviorAnalyzer(client, store, fields, filter_str,
|
|
root=root, notifier=None)
|
|
items = analyzer.analyze()
|
|
finally:
|
|
behavior.clear_range()
|
|
for item in items:
|
|
try:
|
|
last = client.search_absolute(
|
|
query='%s:"%s"' % (fields["client_ip"], item["ip"]),
|
|
from_ts=iso(from_dt), to_ts=iso(to_dt), limit=1,
|
|
sort="timestamp:desc", fields=["timestamp"],
|
|
filter_str=filter_str)
|
|
stamp = (last or [{}])[0].get("timestamp")
|
|
parsed = parse_iso(stamp)
|
|
if parsed:
|
|
item["last_ts"] = parsed.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
except Exception:
|
|
pass
|
|
flagged = [item for item in items if item.get("verdict") == "建议封禁"]
|
|
store.upsert_historical_profiles(items)
|
|
before = {row["ip"] for row in store.conn.execute("SELECT ip FROM ip_fused")}
|
|
store.sync_fused()
|
|
added = [item["ip"] for item in flagged if item["ip"] not in before]
|
|
_suppress(store, added)
|
|
return {"violations": len(items), "flagged": len(flagged),
|
|
"newly_fused": len(added), "new_ips": sorted(added)}
|
|
|
|
|
|
def auto_backfill_gap(store, client, fields, filter_str, from_dt, to_dt,
|
|
root, behavior_enabled=True):
|
|
started = time.time()
|
|
geo = asn = None
|
|
try:
|
|
import os
|
|
geo_path = os.path.join(root, "data", "geo", "geo.db")
|
|
asn_path = os.path.join(root, "data", "geo", "asn.db")
|
|
if os.path.exists(geo_path):
|
|
geo = GeoIP(geo_path)
|
|
if os.path.exists(asn_path):
|
|
asn = AsnDB(asn_path)
|
|
except Exception:
|
|
pass
|
|
summary = {"from": iso(from_dt), "to": iso(to_dt)}
|
|
try:
|
|
summary["feature"] = feature_backfill(
|
|
client, store, fields, filter_str, from_dt, to_dt, geo=geo, asn=asn)
|
|
gap_hours = (to_dt - from_dt).total_seconds() / 3600.0
|
|
if behavior_enabled and 1 <= gap_hours <= 48:
|
|
summary["behavior"] = behavior_backfill(
|
|
client, store, fields, filter_str, from_dt, to_dt, root,
|
|
geo=geo, asn=asn)
|
|
store.incr_meta("backfill_runs_total")
|
|
store.set_meta("backfill_last_at", watch.now_iso())
|
|
store.set_meta("backfill_last_from", iso(from_dt))
|
|
store.set_meta("backfill_last_to", iso(to_dt))
|
|
store.set_meta("backfill_last_added",
|
|
len(summary["feature"].get("ips") or []))
|
|
store.set_meta("backfill_last_error", "")
|
|
except Exception as exc:
|
|
summary["error"] = str(exc)
|
|
store.set_meta("backfill_last_error", str(exc))
|
|
summary["seconds"] = round(time.time() - started, 1)
|
|
return summary
|