"""Judgement layer: aggregated Graylog signals -> ranked ban candidates. This module is **read only**. It answers one question: "在这个时间窗里,哪些客户端 IP 看起来像在攻击,为什么,建议怎么处理?" It does not touch the enforcement side (Sangfor AD / WAF / iptables). The output is a list of ``{ip, score, action, reasons, signals}`` which the next phase can feed into whatever blacklist API we end up using. Design notes ------------ * A **fixed number of ``terms`` queries** (one per signal) keeps the load on Elasticsearch predictable no matter how many IPs show up in the window. * Signal queries are templates over field names. The report exposes each signal's query and hit count, so they can be tuned against real data. A signal that matched 0 IPs usually means the query does not fit the field mapping (e.g. ``request_uri`` is a keyword-ish analyzed field). * **Volume alone never bans** (``require_evidence``). A busy but legitimate client must not be blackholed by a rate threshold alone; at least one behavioural signal (auth failures / scanner UA / sensitive path / 404 probing) has to be present for a ban recommendation. """ import ipaddress import json import math import statistics from . import detect from .client import normalize_terms from .http import GraylogError DEFAULT_FIELDS = { "client_ip": "client_ip", "status": "status", "uri": "request_uri", "ua": "http_user_agent", "method": "request_method", } # name, title, required field keys, query template, weight, saturation count SIGNAL_DEFS = ( { "name": "auth_fail", "title": "认证失败", "requires": ("status",), "query": lambda fields: "%s:(401 OR 403)" % fields["status"], "weight": 45.0, "saturate": 10.0, }, { "name": "scanner_ua", "title": "扫描器 UA", "requires": ("ua",), "query": lambda fields: ( "%s:(sqlmap* OR nikto* OR nuclei* OR nmap* OR masscan* OR zgrab* OR " "wpscan* OR gobuster* OR dirbuster* OR dirsearch* OR acunetix*)" % fields["ua"] ), "weight": 50.0, "saturate": 2.0, }, { "name": "sensitive_path", "title": "敏感路径", "requires": ("uri",), # Slashes must be escaped, otherwise Lucene treats /.../ as a regex. "query": lambda fields: ( "%s:(\\/wp-login* OR \\/phpmyadmin* OR \\/.env* OR \\/actuator* " "OR \\/cgi-bin* OR \\/manager\\/html*)" % fields["uri"] ), "weight": 25.0, "saturate": 5.0, }, { "name": "not_found", "title": "404 探测", "requires": ("status",), "query": lambda fields: "%s:404" % fields["status"], "weight": 20.0, "saturate": 20.0, }, { "name": "bad_method", "title": "异常方法", "requires": ("method",), "query": lambda fields: "%s:(PUT OR DELETE OR TRACE OR CONNECT)" % fields["method"], "weight": 10.0, "saturate": 5.0, }, ) # URIs containing these hints mean auth failures are likely credential # attacks rather than an integration with a broken/expired token. AUTH_PATH_HINTS = ("login", "signin", "auth", "token", "captcha", "oauth", "password") DEFAULT_THRESHOLDS = { "min_requests": 20, # ignore IPs with fewer requests in the window "min_score": 40.0, # below this: ignore "short_ban_score": 55.0, # >= this: short ban recommendation "ban_score": 75.0, # >= this: ban recommendation "rate_weight": 25.0, "rate_saturate": 200.0, # requests in window for full rate points "spike_weight": 15.0, "auth_ratio_weight": 25.0, "require_evidence": True, "evidence_signals": ("auth_fail", "scanner_ua", "sensitive_path", "not_found", "bad_method"), } # --------------------------------------------------------------------- profile def load_profile(path): if not path: return {} try: with open(path, "r", encoding="utf-8") as handle: data = json.load(handle) except OSError as exc: raise SystemExit("无法读取 profile %s: %s" % (path, exc)) except ValueError as exc: raise SystemExit("profile %s 不是合法 JSON: %s" % (path, exc)) if not isinstance(data, dict): raise SystemExit("profile %s 必须是 JSON 对象" % path) return data def build_fields(profile, args=None): fields = dict(DEFAULT_FIELDS) for key, value in (profile.get("fields") or {}).items(): if value: fields[key] = value flag_map = { "client_ip": "field", "status": "status_field", "uri": "uri_field", "ua": "ua_field", "method": "method_field", } for key, attribute in flag_map.items(): value = getattr(args, attribute, None) if args is not None else None if value: fields[key] = value return fields def build_signals(fields, overrides=None): overrides = overrides or {} signals = [] for definition in SIGNAL_DEFS: if any(not fields.get(key) for key in definition["requires"]): continue signal = { "name": definition["name"], "title": definition["title"], "query": definition["query"](fields), "weight": float(definition["weight"]), "saturate": float(definition["saturate"]), } override = overrides.get(signal["name"]) or {} if override.get("enabled") is False: continue if override.get("query"): signal["query"] = override["query"] if override.get("weight") is not None: signal["weight"] = float(override["weight"]) if override.get("saturate") is not None: signal["saturate"] = float(override["saturate"]) signals.append(signal) return signals # ------------------------------------------------------------------- allowlist class Allowlist: """Never ban infrastructure, internal addresses or explicitly listed nets.""" def __init__(self, entries=None, exclude_internal=True): self.entries = list(entries or []) self.exclude_internal = exclude_internal def reason(self, ip_text): try: ip = ipaddress.ip_address(ip_text) except ValueError: return None for network, source in self.entries: if ip.version == network.version and ip in network: return "白名单 %s" % source if self.exclude_internal and detect.is_internal(ip): return "内部地址(RFC1918/回环/CGNAT)" return None def load_allowlist(paths): entries = [] for path in paths or []: try: with open(path, "r", encoding="utf-8") as handle: lines = handle.readlines() except OSError as exc: raise SystemExit("无法读取白名单 %s: %s" % (path, exc)) for number, line in enumerate(lines, 1): text = line.split("#", 1)[0].strip() if not text: continue try: network = ipaddress.ip_network(text, strict=False) except ValueError: raise SystemExit("白名单 %s:%d 不是合法 IP/CIDR: %r" % (path, number, text)) entries.append((network, "%s:%d" % (path, number))) return entries # ----------------------------------------------------------------- collection def collect_signals(client, field, signals, window_seconds, query="*", filter_str=None, size=200): """Run one terms query per signal plus one for the total request counts.""" term_frame = lambda data: { row["term"]: int(row["count"] or 0) for row in normalize_terms(data) if row["term"] not in (None, "") } totals = term_frame(client.terms( field=field, query=query, size=size, range_seconds=window_seconds, filter_str=filter_str, )) per_signal = {} errors = [] for signal in signals: try: data = client.terms( field=field, query=signal["query"], size=size, range_seconds=window_seconds, filter_str=filter_str, ) per_signal[signal["name"]] = term_frame(data) except GraylogError as exc: per_signal[signal["name"]] = {} errors.append({ "signal": signal["name"], "status": exc.status, "error": exc.message, }) return totals, per_signal, errors # ------------------------------------------------------------------- verify def verify_candidates(client, candidates, fields, window_seconds, filter_str=None, limit=5, size=10): """Drill into top candidates: is the traffic one automated endpoint poller? A client that hammers a *single* business API at a steady rate is usually an integration with a broken/expired token, not an attacker. Banning it breaks a business system, so such candidates are downgraded to ``observe`` and the decision is left to a human. """ uri_field = fields.get("uri") client_field = fields.get("client_ip") if not uri_field or not client_field: return for candidate in candidates[:limit]: if not candidate["action"].startswith("ban"): continue query = '%s:"%s"' % (client_field, candidate["ip"]) try: data = client.terms( field=uri_field, query=query, size=size, range_seconds=window_seconds, filter_str=filter_str, ) except GraylogError as exc: candidate["verify_error"] = exc.message continue rows = [(row["term"], row["count"]) for row in normalize_terms(data) if row["term"] not in (None, "")] total = sum(count for _, count in rows) candidate["verify"] = { "distinct_uris": len(rows), "total_requests": total, "top_uri": rows[0][0] if rows else None, "top_uri_ratio": round(rows[0][1] / total, 4) if rows and total else 0.0, } if not rows or not total: continue top_uri, top_count = rows[0] ratio = top_count / float(total) login_like = any(hint in str(top_uri).lower() for hint in AUTH_PATH_HINTS) candidate["verify"]["login_like"] = login_like if ratio >= 0.9 and len(rows) <= 2 and not login_like: candidate["verify"]["automation_suspected"] = True candidate["action"] = "observe" candidate["score"] = round(min(candidate["score"], 54.0), 1) candidate["reasons"].append( "流量集中在单一接口 %s(%.0f%%),疑似业务客户端/自动化任务,已降级为 observe" % (top_uri, ratio * 100) ) # -------------------------------------------------------------------- scoring def score_candidates(totals, per_signal, signals, thresholds, allowlist): pool = {ip: count for ip, count in totals.items() if count >= thresholds["min_requests"]} median = statistics.median(pool.values()) if pool else 0 candidates = [] excluded = [] for ip, requests in pool.items(): reason = allowlist.reason(ip) if reason: excluded.append({"ip": ip, "requests": requests, "reason": reason}) continue points = {} counts = {} reasons = [] for signal in signals: count = per_signal.get(signal["name"], {}).get(ip, 0) if not count: continue counts[signal["name"]] = count contributed = signal["weight"] * min(1.0, count / signal["saturate"]) points[signal["name"]] = round(contributed, 1) reasons.append("%s %d 次" % (signal["title"], count)) rate_points = thresholds["rate_weight"] * min( 1.0, requests / float(thresholds["rate_saturate"])) points["rate"] = round(rate_points, 1) if median and requests >= max(50, median * 4): ratio = requests / float(median) spike = thresholds["spike_weight"] * min(1.0, math.log(ratio, 2) / 6.0) points["spike"] = round(spike, 1) reasons.append("请求量是候选中位数的 %.0f 倍" % ratio) auth = counts.get("auth_fail", 0) auth_ratio = (auth / float(requests)) if requests else 0.0 if auth >= 10 and auth_ratio >= 0.5: points["auth_ratio"] = thresholds["auth_ratio_weight"] reasons.append("失败请求占 %.0f%%" % (auth_ratio * 100)) score = min(100.0, sum(points.values())) evidence = any(counts.get(name) for name in thresholds["evidence_signals"]) if evidence and score >= thresholds["ban_score"]: action = "ban_1h" elif evidence and score >= thresholds["short_ban_score"]: action = "ban_30m" elif score >= thresholds["min_score"]: action = "observe" else: action = "ignore" if not evidence and thresholds["require_evidence"] and action in ("ban_1h", "ban_30m"): action = "observe" reasons.append("仅请求量异常、无攻击特征,不自动封禁") candidates.append({ "ip": ip, "score": round(score, 1), "requests": requests, "action": action, "evidence": evidence, "auth_ratio": round(auth_ratio, 4), "signals": counts, "points": points, "reasons": reasons, }) candidates.sort(key=lambda row: (-row["score"], -row["requests"], row["ip"])) excluded.sort(key=lambda row: (-row["requests"], row["ip"])) signal_stats = [] for signal in signals: matched = per_signal.get(signal["name"], {}) signal_stats.append({ "name": signal["name"], "title": signal["title"], "query": signal["query"], "ips_matched": len(matched), "requests_matched": sum(matched.values()), }) return { "median_requests": median, "candidate_pool": len(pool), "candidates": candidates, "excluded": excluded, "signal_stats": signal_stats, }