DeepFal
2d76ed6672
- 特征检测（规则/签名）与行为检测（24h 统计）单进程运行 - 融合名单按首次发现时间倒序，展示位置/归属（GeoIP + ASN） - 钉钉融合推送（加签、24h 节流、去重） - 页面状态灯、CSV 导出、Docker 镜像构建文件
398 lines
14 KiB
Python
398 lines
14 KiB
Python
"""Judgement layer: aggregated Graylog signals -> ranked ban candidates.
|
||
|
||
This module is **read only**. It answers one question:
|
||
|
||
"在这个时间窗里，哪些客户端 IP 看起来像在攻击，为什么，建议怎么处理？"
|
||
|
||
It does not touch the enforcement side (Sangfor AD / WAF / iptables). The
|
||
output is a list of ``{ip, score, action, reasons, signals}`` which the next
|
||
phase can feed into whatever blacklist API we end up using.
|
||
|
||
Design notes
|
||
------------
|
||
* A **fixed number of ``terms`` queries** (one per signal) keeps the load on
|
||
Elasticsearch predictable no matter how many IPs show up in the window.
|
||
* Signal queries are templates over field names. The report exposes each
|
||
signal's query and hit count, so they can be tuned against real data. A
|
||
signal that matched 0 IPs usually means the query does not fit the field
|
||
mapping (e.g. ``request_uri`` is a keyword-ish analyzed field).
|
||
* **Volume alone never bans** (``require_evidence``). A busy but legitimate
|
||
client must not be blackholed by a rate threshold alone; at least one
|
||
behavioural signal (auth failures / scanner UA / sensitive path / 404
|
||
probing) has to be present for a ban recommendation.
|
||
"""
|
||
|
||
import ipaddress
|
||
import json
|
||
import math
|
||
import statistics
|
||
|
||
from . import detect
|
||
from .client import normalize_terms
|
||
from .http import GraylogError
|
||
|
||
DEFAULT_FIELDS = {
|
||
"client_ip": "client_ip",
|
||
"status": "status",
|
||
"uri": "request_uri",
|
||
"ua": "http_user_agent",
|
||
"method": "request_method",
|
||
}
|
||
|
||
# name, title, required field keys, query template, weight, saturation count
|
||
SIGNAL_DEFS = (
|
||
{
|
||
"name": "auth_fail",
|
||
"title": "认证失败",
|
||
"requires": ("status",),
|
||
"query": lambda fields: "%s:(401 OR 403)" % fields["status"],
|
||
"weight": 45.0,
|
||
"saturate": 10.0,
|
||
},
|
||
{
|
||
"name": "scanner_ua",
|
||
"title": "扫描器 UA",
|
||
"requires": ("ua",),
|
||
"query": lambda fields: (
|
||
"%s:(sqlmap* OR nikto* OR nuclei* OR nmap* OR masscan* OR zgrab* OR "
|
||
"wpscan* OR gobuster* OR dirbuster* OR dirsearch* OR acunetix*)"
|
||
% fields["ua"]
|
||
),
|
||
"weight": 50.0,
|
||
"saturate": 2.0,
|
||
},
|
||
{
|
||
"name": "sensitive_path",
|
||
"title": "敏感路径",
|
||
"requires": ("uri",),
|
||
# Slashes must be escaped, otherwise Lucene treats /.../ as a regex.
|
||
"query": lambda fields: (
|
||
"%s:(\\/wp-login* OR \\/phpmyadmin* OR \\/.env* OR \\/actuator* "
|
||
"OR \\/cgi-bin* OR \\/manager\\/html*)"
|
||
% fields["uri"]
|
||
),
|
||
"weight": 25.0,
|
||
"saturate": 5.0,
|
||
},
|
||
{
|
||
"name": "not_found",
|
||
"title": "404 探测",
|
||
"requires": ("status",),
|
||
"query": lambda fields: "%s:404" % fields["status"],
|
||
"weight": 20.0,
|
||
"saturate": 20.0,
|
||
},
|
||
{
|
||
"name": "bad_method",
|
||
"title": "异常方法",
|
||
"requires": ("method",),
|
||
"query": lambda fields: "%s:(PUT OR DELETE OR TRACE OR CONNECT)" % fields["method"],
|
||
"weight": 10.0,
|
||
"saturate": 5.0,
|
||
},
|
||
)
|
||
|
||
# URIs containing these hints mean auth failures are likely credential
|
||
# attacks rather than an integration with a broken/expired token.
|
||
AUTH_PATH_HINTS = ("login", "signin", "auth", "token", "captcha", "oauth", "password")
|
||
|
||
DEFAULT_THRESHOLDS = {
|
||
"min_requests": 20, # ignore IPs with fewer requests in the window
|
||
"min_score": 40.0, # below this: ignore
|
||
"short_ban_score": 55.0, # >= this: short ban recommendation
|
||
"ban_score": 75.0, # >= this: ban recommendation
|
||
"rate_weight": 25.0,
|
||
"rate_saturate": 200.0, # requests in window for full rate points
|
||
"spike_weight": 15.0,
|
||
"auth_ratio_weight": 25.0,
|
||
"require_evidence": True,
|
||
"evidence_signals": ("auth_fail", "scanner_ua", "sensitive_path", "not_found", "bad_method"),
|
||
}
|
||
|
||
|
||
# --------------------------------------------------------------------- profile
|
||
def load_profile(path):
|
||
if not path:
|
||
return {}
|
||
try:
|
||
with open(path, "r", encoding="utf-8") as handle:
|
||
data = json.load(handle)
|
||
except OSError as exc:
|
||
raise SystemExit("无法读取 profile %s: %s" % (path, exc))
|
||
except ValueError as exc:
|
||
raise SystemExit("profile %s 不是合法 JSON: %s" % (path, exc))
|
||
if not isinstance(data, dict):
|
||
raise SystemExit("profile %s 必须是 JSON 对象" % path)
|
||
return data
|
||
|
||
|
||
def build_fields(profile, args=None):
|
||
fields = dict(DEFAULT_FIELDS)
|
||
for key, value in (profile.get("fields") or {}).items():
|
||
if value:
|
||
fields[key] = value
|
||
|
||
flag_map = {
|
||
"client_ip": "field",
|
||
"status": "status_field",
|
||
"uri": "uri_field",
|
||
"ua": "ua_field",
|
||
"method": "method_field",
|
||
}
|
||
for key, attribute in flag_map.items():
|
||
value = getattr(args, attribute, None) if args is not None else None
|
||
if value:
|
||
fields[key] = value
|
||
return fields
|
||
|
||
|
||
def build_signals(fields, overrides=None):
|
||
overrides = overrides or {}
|
||
signals = []
|
||
for definition in SIGNAL_DEFS:
|
||
if any(not fields.get(key) for key in definition["requires"]):
|
||
continue
|
||
signal = {
|
||
"name": definition["name"],
|
||
"title": definition["title"],
|
||
"query": definition["query"](fields),
|
||
"weight": float(definition["weight"]),
|
||
"saturate": float(definition["saturate"]),
|
||
}
|
||
override = overrides.get(signal["name"]) or {}
|
||
if override.get("enabled") is False:
|
||
continue
|
||
if override.get("query"):
|
||
signal["query"] = override["query"]
|
||
if override.get("weight") is not None:
|
||
signal["weight"] = float(override["weight"])
|
||
if override.get("saturate") is not None:
|
||
signal["saturate"] = float(override["saturate"])
|
||
signals.append(signal)
|
||
return signals
|
||
|
||
|
||
# ------------------------------------------------------------------- allowlist
|
||
class Allowlist:
|
||
"""Never ban infrastructure, internal addresses or explicitly listed nets."""
|
||
|
||
def __init__(self, entries=None, exclude_internal=True):
|
||
self.entries = list(entries or [])
|
||
self.exclude_internal = exclude_internal
|
||
|
||
def reason(self, ip_text):
|
||
try:
|
||
ip = ipaddress.ip_address(ip_text)
|
||
except ValueError:
|
||
return None
|
||
for network, source in self.entries:
|
||
if ip.version == network.version and ip in network:
|
||
return "白名单 %s" % source
|
||
if self.exclude_internal and detect.is_internal(ip):
|
||
return "内部地址（RFC1918/回环/CGNAT）"
|
||
return None
|
||
|
||
|
||
def load_allowlist(paths):
|
||
entries = []
|
||
for path in paths or []:
|
||
try:
|
||
with open(path, "r", encoding="utf-8") as handle:
|
||
lines = handle.readlines()
|
||
except OSError as exc:
|
||
raise SystemExit("无法读取白名单 %s: %s" % (path, exc))
|
||
for number, line in enumerate(lines, 1):
|
||
text = line.split("#", 1)[0].strip()
|
||
if not text:
|
||
continue
|
||
try:
|
||
network = ipaddress.ip_network(text, strict=False)
|
||
except ValueError:
|
||
raise SystemExit("白名单 %s:%d 不是合法 IP/CIDR: %r" % (path, number, text))
|
||
entries.append((network, "%s:%d" % (path, number)))
|
||
return entries
|
||
|
||
|
||
# ----------------------------------------------------------------- collection
|
||
def collect_signals(client, field, signals, window_seconds, query="*",
|
||
filter_str=None, size=200):
|
||
"""Run one terms query per signal plus one for the total request counts."""
|
||
term_frame = lambda data: {
|
||
row["term"]: int(row["count"] or 0)
|
||
for row in normalize_terms(data)
|
||
if row["term"] not in (None, "")
|
||
}
|
||
|
||
totals = term_frame(client.terms(
|
||
field=field, query=query, size=size,
|
||
range_seconds=window_seconds, filter_str=filter_str,
|
||
))
|
||
|
||
per_signal = {}
|
||
errors = []
|
||
for signal in signals:
|
||
try:
|
||
data = client.terms(
|
||
field=field, query=signal["query"], size=size,
|
||
range_seconds=window_seconds, filter_str=filter_str,
|
||
)
|
||
per_signal[signal["name"]] = term_frame(data)
|
||
except GraylogError as exc:
|
||
per_signal[signal["name"]] = {}
|
||
errors.append({
|
||
"signal": signal["name"],
|
||
"status": exc.status,
|
||
"error": exc.message,
|
||
})
|
||
return totals, per_signal, errors
|
||
|
||
|
||
# ------------------------------------------------------------------- verify
|
||
def verify_candidates(client, candidates, fields, window_seconds,
|
||
filter_str=None, limit=5, size=10):
|
||
"""Drill into top candidates: is the traffic one automated endpoint poller?
|
||
|
||
A client that hammers a *single* business API at a steady rate is usually an
|
||
integration with a broken/expired token, not an attacker. Banning it breaks
|
||
a business system, so such candidates are downgraded to ``observe`` and the
|
||
decision is left to a human.
|
||
"""
|
||
uri_field = fields.get("uri")
|
||
client_field = fields.get("client_ip")
|
||
if not uri_field or not client_field:
|
||
return
|
||
|
||
for candidate in candidates[:limit]:
|
||
if not candidate["action"].startswith("ban"):
|
||
continue
|
||
query = '%s:"%s"' % (client_field, candidate["ip"])
|
||
try:
|
||
data = client.terms(
|
||
field=uri_field, query=query, size=size,
|
||
range_seconds=window_seconds, filter_str=filter_str,
|
||
)
|
||
except GraylogError as exc:
|
||
candidate["verify_error"] = exc.message
|
||
continue
|
||
|
||
rows = [(row["term"], row["count"]) for row in normalize_terms(data)
|
||
if row["term"] not in (None, "")]
|
||
total = sum(count for _, count in rows)
|
||
candidate["verify"] = {
|
||
"distinct_uris": len(rows),
|
||
"total_requests": total,
|
||
"top_uri": rows[0][0] if rows else None,
|
||
"top_uri_ratio": round(rows[0][1] / total, 4) if rows and total else 0.0,
|
||
}
|
||
if not rows or not total:
|
||
continue
|
||
|
||
top_uri, top_count = rows[0]
|
||
ratio = top_count / float(total)
|
||
login_like = any(hint in str(top_uri).lower() for hint in AUTH_PATH_HINTS)
|
||
candidate["verify"]["login_like"] = login_like
|
||
|
||
if ratio >= 0.9 and len(rows) <= 2 and not login_like:
|
||
candidate["verify"]["automation_suspected"] = True
|
||
candidate["action"] = "observe"
|
||
candidate["score"] = round(min(candidate["score"], 54.0), 1)
|
||
candidate["reasons"].append(
|
||
"流量集中在单一接口 %s（%.0f%%），疑似业务客户端/自动化任务，已降级为 observe"
|
||
% (top_uri, ratio * 100)
|
||
)
|
||
|
||
|
||
# -------------------------------------------------------------------- scoring
|
||
def score_candidates(totals, per_signal, signals, thresholds, allowlist):
|
||
pool = {ip: count for ip, count in totals.items()
|
||
if count >= thresholds["min_requests"]}
|
||
median = statistics.median(pool.values()) if pool else 0
|
||
|
||
candidates = []
|
||
excluded = []
|
||
|
||
for ip, requests in pool.items():
|
||
reason = allowlist.reason(ip)
|
||
if reason:
|
||
excluded.append({"ip": ip, "requests": requests, "reason": reason})
|
||
continue
|
||
|
||
points = {}
|
||
counts = {}
|
||
reasons = []
|
||
|
||
for signal in signals:
|
||
count = per_signal.get(signal["name"], {}).get(ip, 0)
|
||
if not count:
|
||
continue
|
||
counts[signal["name"]] = count
|
||
contributed = signal["weight"] * min(1.0, count / signal["saturate"])
|
||
points[signal["name"]] = round(contributed, 1)
|
||
reasons.append("%s %d 次" % (signal["title"], count))
|
||
|
||
rate_points = thresholds["rate_weight"] * min(
|
||
1.0, requests / float(thresholds["rate_saturate"]))
|
||
points["rate"] = round(rate_points, 1)
|
||
|
||
if median and requests >= max(50, median * 4):
|
||
ratio = requests / float(median)
|
||
spike = thresholds["spike_weight"] * min(1.0, math.log(ratio, 2) / 6.0)
|
||
points["spike"] = round(spike, 1)
|
||
reasons.append("请求量是候选中位数的 %.0f 倍" % ratio)
|
||
|
||
auth = counts.get("auth_fail", 0)
|
||
auth_ratio = (auth / float(requests)) if requests else 0.0
|
||
if auth >= 10 and auth_ratio >= 0.5:
|
||
points["auth_ratio"] = thresholds["auth_ratio_weight"]
|
||
reasons.append("失败请求占 %.0f%%" % (auth_ratio * 100))
|
||
|
||
score = min(100.0, sum(points.values()))
|
||
evidence = any(counts.get(name) for name in thresholds["evidence_signals"])
|
||
|
||
if evidence and score >= thresholds["ban_score"]:
|
||
action = "ban_1h"
|
||
elif evidence and score >= thresholds["short_ban_score"]:
|
||
action = "ban_30m"
|
||
elif score >= thresholds["min_score"]:
|
||
action = "observe"
|
||
else:
|
||
action = "ignore"
|
||
|
||
if not evidence and thresholds["require_evidence"] and action in ("ban_1h", "ban_30m"):
|
||
action = "observe"
|
||
reasons.append("仅请求量异常、无攻击特征，不自动封禁")
|
||
|
||
candidates.append({
|
||
"ip": ip,
|
||
"score": round(score, 1),
|
||
"requests": requests,
|
||
"action": action,
|
||
"evidence": evidence,
|
||
"auth_ratio": round(auth_ratio, 4),
|
||
"signals": counts,
|
||
"points": points,
|
||
"reasons": reasons,
|
||
})
|
||
|
||
candidates.sort(key=lambda row: (-row["score"], -row["requests"], row["ip"]))
|
||
excluded.sort(key=lambda row: (-row["requests"], row["ip"]))
|
||
|
||
signal_stats = []
|
||
for signal in signals:
|
||
matched = per_signal.get(signal["name"], {})
|
||
signal_stats.append({
|
||
"name": signal["name"],
|
||
"title": signal["title"],
|
||
"query": signal["query"],
|
||
"ips_matched": len(matched),
|
||
"requests_matched": sum(matched.values()),
|
||
})
|
||
|
||
return {
|
||
"median_requests": median,
|
||
"candidate_pool": len(pool),
|
||
"candidates": candidates,
|
||
"excluded": excluded,
|
||
"signal_stats": signal_stats,
|
||
}
|