初始提交：恶意 IP 特征+行为融合检测系统
- 特征检测（规则/签名）与行为检测（24h 统计）单进程运行 - 融合名单按首次发现时间倒序，展示位置/归属（GeoIP + ASN） - 钉钉融合推送（加签、24h 节流、去重） - 页面状态灯、CSV 导出、Docker 镜像构建文件
This commit is contained in:
DeepFal committed
commit
2d76ed6672
42 files changed
+10645
No files matched your search
@@ -0,0 +1,397 @@
|
||||
"""Judgement layer: aggregated Graylog signals -> ranked ban candidates.
|
||||
|
||||
This module is **read only**. It answers one question:
|
||||
|
||||
"在这个时间窗里，哪些客户端 IP 看起来像在攻击，为什么，建议怎么处理？"
|
||||
|
||||
It does not touch the enforcement side (Sangfor AD / WAF / iptables). The
|
||||
output is a list of ``{ip, score, action, reasons, signals}`` which the next
|
||||
phase can feed into whatever blacklist API we end up using.
|
||||
|
||||
Design notes
|
||||
------------
|
||||
* A **fixed number of ``terms`` queries** (one per signal) keeps the load on
|
||||
Elasticsearch predictable no matter how many IPs show up in the window.
|
||||
* Signal queries are templates over field names. The report exposes each
|
||||
signal's query and hit count, so they can be tuned against real data. A
|
||||
signal that matched 0 IPs usually means the query does not fit the field
|
||||
mapping (e.g. ``request_uri`` is a keyword-ish analyzed field).
|
||||
* **Volume alone never bans** (``require_evidence``). A busy but legitimate
|
||||
client must not be blackholed by a rate threshold alone; at least one
|
||||
behavioural signal (auth failures / scanner UA / sensitive path / 404
|
||||
probing) has to be present for a ban recommendation.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
|
||||
from . import detect
|
||||
from .client import normalize_terms
|
||||
from .http import GraylogError
|
||||
|
||||
DEFAULT_FIELDS = {
|
||||
"client_ip": "client_ip",
|
||||
"status": "status",
|
||||
"uri": "request_uri",
|
||||
"ua": "http_user_agent",
|
||||
"method": "request_method",
|
||||
}
|
||||
|
||||
# name, title, required field keys, query template, weight, saturation count
|
||||
SIGNAL_DEFS = (
|
||||
{
|
||||
"name": "auth_fail",
|
||||
"title": "认证失败",
|
||||
"requires": ("status",),
|
||||
"query": lambda fields: "%s:(401 OR 403)" % fields["status"],
|
||||
"weight": 45.0,
|
||||
"saturate": 10.0,
|
||||
},
|
||||
{
|
||||
"name": "scanner_ua",
|
||||
"title": "扫描器 UA",
|
||||
"requires": ("ua",),
|
||||
"query": lambda fields: (
|
||||
"%s:(sqlmap* OR nikto* OR nuclei* OR nmap* OR masscan* OR zgrab* OR "
|
||||
"wpscan* OR gobuster* OR dirbuster* OR dirsearch* OR acunetix*)"
|
||||
% fields["ua"]
|
||||
),
|
||||
"weight": 50.0,
|
||||
"saturate": 2.0,
|
||||
},
|
||||
{
|
||||
"name": "sensitive_path",
|
||||
"title": "敏感路径",
|
||||
"requires": ("uri",),
|
||||
# Slashes must be escaped, otherwise Lucene treats /.../ as a regex.
|
||||
"query": lambda fields: (
|
||||
"%s:(\\/wp-login* OR \\/phpmyadmin* OR \\/.env* OR \\/actuator* "
|
||||
"OR \\/cgi-bin* OR \\/manager\\/html*)"
|
||||
% fields["uri"]
|
||||
),
|
||||
"weight": 25.0,
|
||||
"saturate": 5.0,
|
||||
},
|
||||
{
|
||||
"name": "not_found",
|
||||
"title": "404 探测",
|
||||
"requires": ("status",),
|
||||
"query": lambda fields: "%s:404" % fields["status"],
|
||||
"weight": 20.0,
|
||||
"saturate": 20.0,
|
||||
},
|
||||
{
|
||||
"name": "bad_method",
|
||||
"title": "异常方法",
|
||||
"requires": ("method",),
|
||||
"query": lambda fields: "%s:(PUT OR DELETE OR TRACE OR CONNECT)" % fields["method"],
|
||||
"weight": 10.0,
|
||||
"saturate": 5.0,
|
||||
},
|
||||
)
|
||||
|
||||
# URIs containing these hints mean auth failures are likely credential
|
||||
# attacks rather than an integration with a broken/expired token.
|
||||
AUTH_PATH_HINTS = ("login", "signin", "auth", "token", "captcha", "oauth", "password")
|
||||
|
||||
DEFAULT_THRESHOLDS = {
|
||||
"min_requests": 20, # ignore IPs with fewer requests in the window
|
||||
"min_score": 40.0, # below this: ignore
|
||||
"short_ban_score": 55.0, # >= this: short ban recommendation
|
||||
"ban_score": 75.0, # >= this: ban recommendation
|
||||
"rate_weight": 25.0,
|
||||
"rate_saturate": 200.0, # requests in window for full rate points
|
||||
"spike_weight": 15.0,
|
||||
"auth_ratio_weight": 25.0,
|
||||
"require_evidence": True,
|
||||
"evidence_signals": ("auth_fail", "scanner_ua", "sensitive_path", "not_found", "bad_method"),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- profile
|
||||
def load_profile(path):
|
||||
if not path:
|
||||
return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
except OSError as exc:
|
||||
raise SystemExit("无法读取 profile %s: %s" % (path, exc))
|
||||
except ValueError as exc:
|
||||
raise SystemExit("profile %s 不是合法 JSON: %s" % (path, exc))
|
||||
if not isinstance(data, dict):
|
||||
raise SystemExit("profile %s 必须是 JSON 对象" % path)
|
||||
return data
|
||||
|
||||
|
||||
def build_fields(profile, args=None):
|
||||
fields = dict(DEFAULT_FIELDS)
|
||||
for key, value in (profile.get("fields") or {}).items():
|
||||
if value:
|
||||
fields[key] = value
|
||||
|
||||
flag_map = {
|
||||
"client_ip": "field",
|
||||
"status": "status_field",
|
||||
"uri": "uri_field",
|
||||
"ua": "ua_field",
|
||||
"method": "method_field",
|
||||
}
|
||||
for key, attribute in flag_map.items():
|
||||
value = getattr(args, attribute, None) if args is not None else None
|
||||
if value:
|
||||
fields[key] = value
|
||||
return fields
|
||||
|
||||
|
||||
def build_signals(fields, overrides=None):
|
||||
overrides = overrides or {}
|
||||
signals = []
|
||||
for definition in SIGNAL_DEFS:
|
||||
if any(not fields.get(key) for key in definition["requires"]):
|
||||
continue
|
||||
signal = {
|
||||
"name": definition["name"],
|
||||
"title": definition["title"],
|
||||
"query": definition["query"](fields),
|
||||
"weight": float(definition["weight"]),
|
||||
"saturate": float(definition["saturate"]),
|
||||
}
|
||||
override = overrides.get(signal["name"]) or {}
|
||||
if override.get("enabled") is False:
|
||||
continue
|
||||
if override.get("query"):
|
||||
signal["query"] = override["query"]
|
||||
if override.get("weight") is not None:
|
||||
signal["weight"] = float(override["weight"])
|
||||
if override.get("saturate") is not None:
|
||||
signal["saturate"] = float(override["saturate"])
|
||||
signals.append(signal)
|
||||
return signals
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- allowlist
|
||||
class Allowlist:
|
||||
"""Never ban infrastructure, internal addresses or explicitly listed nets."""
|
||||
|
||||
def __init__(self, entries=None, exclude_internal=True):
|
||||
self.entries = list(entries or [])
|
||||
self.exclude_internal = exclude_internal
|
||||
|
||||
def reason(self, ip_text):
|
||||
try:
|
||||
ip = ipaddress.ip_address(ip_text)
|
||||
except ValueError:
|
||||
return None
|
||||
for network, source in self.entries:
|
||||
if ip.version == network.version and ip in network:
|
||||
return "白名单 %s" % source
|
||||
if self.exclude_internal and detect.is_internal(ip):
|
||||
return "内部地址（RFC1918/回环/CGNAT）"
|
||||
return None
|
||||
|
||||
|
||||
def load_allowlist(paths):
|
||||
entries = []
|
||||
for path in paths or []:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
lines = handle.readlines()
|
||||
except OSError as exc:
|
||||
raise SystemExit("无法读取白名单 %s: %s" % (path, exc))
|
||||
for number, line in enumerate(lines, 1):
|
||||
text = line.split("#", 1)[0].strip()
|
||||
if not text:
|
||||
continue
|
||||
try:
|
||||
network = ipaddress.ip_network(text, strict=False)
|
||||
except ValueError:
|
||||
raise SystemExit("白名单 %s:%d 不是合法 IP/CIDR: %r" % (path, number, text))
|
||||
entries.append((network, "%s:%d" % (path, number)))
|
||||
return entries
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- collection
|
||||
def collect_signals(client, field, signals, window_seconds, query="*",
|
||||
filter_str=None, size=200):
|
||||
"""Run one terms query per signal plus one for the total request counts."""
|
||||
term_frame = lambda data: {
|
||||
row["term"]: int(row["count"] or 0)
|
||||
for row in normalize_terms(data)
|
||||
if row["term"] not in (None, "")
|
||||
}
|
||||
|
||||
totals = term_frame(client.terms(
|
||||
field=field, query=query, size=size,
|
||||
range_seconds=window_seconds, filter_str=filter_str,
|
||||
))
|
||||
|
||||
per_signal = {}
|
||||
errors = []
|
||||
for signal in signals:
|
||||
try:
|
||||
data = client.terms(
|
||||
field=field, query=signal["query"], size=size,
|
||||
range_seconds=window_seconds, filter_str=filter_str,
|
||||
)
|
||||
per_signal[signal["name"]] = term_frame(data)
|
||||
except GraylogError as exc:
|
||||
per_signal[signal["name"]] = {}
|
||||
errors.append({
|
||||
"signal": signal["name"],
|
||||
"status": exc.status,
|
||||
"error": exc.message,
|
||||
})
|
||||
return totals, per_signal, errors
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- verify
|
||||
def verify_candidates(client, candidates, fields, window_seconds,
|
||||
filter_str=None, limit=5, size=10):
|
||||
"""Drill into top candidates: is the traffic one automated endpoint poller?
|
||||
|
||||
A client that hammers a *single* business API at a steady rate is usually an
|
||||
integration with a broken/expired token, not an attacker. Banning it breaks
|
||||
a business system, so such candidates are downgraded to ``observe`` and the
|
||||
decision is left to a human.
|
||||
"""
|
||||
uri_field = fields.get("uri")
|
||||
client_field = fields.get("client_ip")
|
||||
if not uri_field or not client_field:
|
||||
return
|
||||
|
||||
for candidate in candidates[:limit]:
|
||||
if not candidate["action"].startswith("ban"):
|
||||
continue
|
||||
query = '%s:"%s"' % (client_field, candidate["ip"])
|
||||
try:
|
||||
data = client.terms(
|
||||
field=uri_field, query=query, size=size,
|
||||
range_seconds=window_seconds, filter_str=filter_str,
|
||||
)
|
||||
except GraylogError as exc:
|
||||
candidate["verify_error"] = exc.message
|
||||
continue
|
||||
|
||||
rows = [(row["term"], row["count"]) for row in normalize_terms(data)
|
||||
if row["term"] not in (None, "")]
|
||||
total = sum(count for _, count in rows)
|
||||
candidate["verify"] = {
|
||||
"distinct_uris": len(rows),
|
||||
"total_requests": total,
|
||||
"top_uri": rows[0][0] if rows else None,
|
||||
"top_uri_ratio": round(rows[0][1] / total, 4) if rows and total else 0.0,
|
||||
}
|
||||
if not rows or not total:
|
||||
continue
|
||||
|
||||
top_uri, top_count = rows[0]
|
||||
ratio = top_count / float(total)
|
||||
login_like = any(hint in str(top_uri).lower() for hint in AUTH_PATH_HINTS)
|
||||
candidate["verify"]["login_like"] = login_like
|
||||
|
||||
if ratio >= 0.9 and len(rows) <= 2 and not login_like:
|
||||
candidate["verify"]["automation_suspected"] = True
|
||||
candidate["action"] = "observe"
|
||||
candidate["score"] = round(min(candidate["score"], 54.0), 1)
|
||||
candidate["reasons"].append(
|
||||
"流量集中在单一接口 %s（%.0f%%），疑似业务客户端/自动化任务，已降级为 observe"
|
||||
% (top_uri, ratio * 100)
|
||||
)
|
||||
|
||||
|
||||
# -------------------------------------------------------------------- scoring
|
||||
def score_candidates(totals, per_signal, signals, thresholds, allowlist):
|
||||
pool = {ip: count for ip, count in totals.items()
|
||||
if count >= thresholds["min_requests"]}
|
||||
median = statistics.median(pool.values()) if pool else 0
|
||||
|
||||
candidates = []
|
||||
excluded = []
|
||||
|
||||
for ip, requests in pool.items():
|
||||
reason = allowlist.reason(ip)
|
||||
if reason:
|
||||
excluded.append({"ip": ip, "requests": requests, "reason": reason})
|
||||
continue
|
||||
|
||||
points = {}
|
||||
counts = {}
|
||||
reasons = []
|
||||
|
||||
for signal in signals:
|
||||
count = per_signal.get(signal["name"], {}).get(ip, 0)
|
||||
if not count:
|
||||
continue
|
||||
counts[signal["name"]] = count
|
||||
contributed = signal["weight"] * min(1.0, count / signal["saturate"])
|
||||
points[signal["name"]] = round(contributed, 1)
|
||||
reasons.append("%s %d 次" % (signal["title"], count))
|
||||
|
||||
rate_points = thresholds["rate_weight"] * min(
|
||||
1.0, requests / float(thresholds["rate_saturate"]))
|
||||
points["rate"] = round(rate_points, 1)
|
||||
|
||||
if median and requests >= max(50, median * 4):
|
||||
ratio = requests / float(median)
|
||||
spike = thresholds["spike_weight"] * min(1.0, math.log(ratio, 2) / 6.0)
|
||||
points["spike"] = round(spike, 1)
|
||||
reasons.append("请求量是候选中位数的 %.0f 倍" % ratio)
|
||||
|
||||
auth = counts.get("auth_fail", 0)
|
||||
auth_ratio = (auth / float(requests)) if requests else 0.0
|
||||
if auth >= 10 and auth_ratio >= 0.5:
|
||||
points["auth_ratio"] = thresholds["auth_ratio_weight"]
|
||||
reasons.append("失败请求占 %.0f%%" % (auth_ratio * 100))
|
||||
|
||||
score = min(100.0, sum(points.values()))
|
||||
evidence = any(counts.get(name) for name in thresholds["evidence_signals"])
|
||||
|
||||
if evidence and score >= thresholds["ban_score"]:
|
||||
action = "ban_1h"
|
||||
elif evidence and score >= thresholds["short_ban_score"]:
|
||||
action = "ban_30m"
|
||||
elif score >= thresholds["min_score"]:
|
||||
action = "observe"
|
||||
else:
|
||||
action = "ignore"
|
||||
|
||||
if not evidence and thresholds["require_evidence"] and action in ("ban_1h", "ban_30m"):
|
||||
action = "observe"
|
||||
reasons.append("仅请求量异常、无攻击特征，不自动封禁")
|
||||
|
||||
candidates.append({
|
||||
"ip": ip,
|
||||
"score": round(score, 1),
|
||||
"requests": requests,
|
||||
"action": action,
|
||||
"evidence": evidence,
|
||||
"auth_ratio": round(auth_ratio, 4),
|
||||
"signals": counts,
|
||||
"points": points,
|
||||
"reasons": reasons,
|
||||
})
|
||||
|
||||
candidates.sort(key=lambda row: (-row["score"], -row["requests"], row["ip"]))
|
||||
excluded.sort(key=lambda row: (-row["requests"], row["ip"]))
|
||||
|
||||
signal_stats = []
|
||||
for signal in signals:
|
||||
matched = per_signal.get(signal["name"], {})
|
||||
signal_stats.append({
|
||||
"name": signal["name"],
|
||||
"title": signal["title"],
|
||||
"query": signal["query"],
|
||||
"ips_matched": len(matched),
|
||||
"requests_matched": sum(matched.values()),
|
||||
})
|
||||
|
||||
return {
|
||||
"median_requests": median,
|
||||
"candidate_pool": len(pool),
|
||||
"candidates": candidates,
|
||||
"excluded": excluded,
|
||||
"signal_stats": signal_stats,
|
||||
}
|
||||
Reference in new issue
Block a user