初始提交：恶意 IP 特征+行为融合检测系统
- 特征检测（规则/签名）与行为检测（24h 统计）单进程运行 - 融合名单按首次发现时间倒序，展示位置/归属（GeoIP + ASN） - 钉钉融合推送（加签、24h 节流、去重） - 页面状态灯、CSV 导出、Docker 镜像构建文件
This commit is contained in:
DeepFal committed
commit
2d76ed6672
42 files changed
+10645
No files matched your search
@@ -0,0 +1,300 @@
|
||||
"""Field heuristics: find the field that actually holds the client IP.
|
||||
|
||||
The ban pipeline is only as good as the field it keys on. In a typical
|
||||
deployment behind a reverse proxy / CDN, the field you *think* is the client
|
||||
IP is the proxy's address:
|
||||
|
||||
* ``source`` -- the host that shipped the log line to Graylog
|
||||
* ``gl2_remote_ip`` -- the host that opened the GELF/Beats connection
|
||||
* ``remote_addr`` -- could be the proxy if XFF is not parsed
|
||||
|
||||
So instead of assuming, sample real messages and score every field by
|
||||
"how often does this field hold an IP address" plus "does the field name look
|
||||
like a client address".
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import re
|
||||
|
||||
# Field names that strongly suggest a client address.
|
||||
STRONG_HINT = re.compile(
|
||||
r"(?i)client[_. -]?ip|remote[_. -]?addr|real[_. -]?ip|"
|
||||
r"(^|[_. -])ip([_. -]|$)|x[_. -]?forwarded[_. -]?for|"
|
||||
r"(^|[_. -])xff([_. -]|$)|src[_. -]?ip|c[_. -]?ip"
|
||||
)
|
||||
|
||||
WEAK_HINT = re.compile(
|
||||
r"(?i)addr|host|client|proxy|peer|origin|caller|visitor|remote|forward|endpoint"
|
||||
)
|
||||
|
||||
# Graylog internals that are never the client IP (gl2_remote_ip is the shipper).
|
||||
INTERNAL_PREFIXES = ("gl2_", "_")
|
||||
INTERNAL_NAMES = ("timestamp", "streams", "message")
|
||||
|
||||
HTTP_HINTS = {
|
||||
"status": re.compile(r"(?i)^(http[_. ]?)?status([_. ]?code)?$|^response([_. ]?(code|status))?$|^code$"),
|
||||
"uri": re.compile(r"(?i)(request[_. ]?)?uri$|^url$|^path$|^request$|^endpoint$|^target$"),
|
||||
"method": re.compile(r"(?i)^(http[_. ]?)?method$|^verb$"),
|
||||
"user_agent": re.compile(r"(?i)user[_. -]?agent|^ua$|^agent$"),
|
||||
"host": re.compile(r"(?i)^(http[_. ]?)?host$|^vhost$|^domain$|^server_name$"),
|
||||
"bytes": re.compile(r"(?i)bytes|^size$|^body_bytes_sent$"),
|
||||
"referer": re.compile(r"(?i)refer(r)?er$"),
|
||||
"duration": re.compile(r"(?i)request[_. ]?time|response[_. ]?time|duration|latency|elapsed"),
|
||||
}
|
||||
|
||||
# Deliberately not using ``ipaddress.is_private``: that property also covers
|
||||
# documentation ranges (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) and
|
||||
# other special-use blocks, which would make every public test address look
|
||||
# like an internal proxy. For ban decisions "internal" means RFC1918 plus
|
||||
# loopback, link-local and CGNAT.
|
||||
_INTERNAL_V4 = tuple(ipaddress.ip_network(cidr) for cidr in (
|
||||
"10.0.0.0/8",
|
||||
"172.16.0.0/12",
|
||||
"192.168.0.0/16",
|
||||
"100.64.0.0/10", # CGNAT
|
||||
"127.0.0.0/8",
|
||||
"169.254.0.0/16",
|
||||
))
|
||||
_INTERNAL_V6 = tuple(ipaddress.ip_network(cidr) for cidr in (
|
||||
"::1/128",
|
||||
"fc00::/7", # unique local
|
||||
"fe80::/10", # link local
|
||||
))
|
||||
|
||||
|
||||
def is_internal(ip):
|
||||
networks = _INTERNAL_V4 if ip.version == 4 else _INTERNAL_V6
|
||||
return any(ip in network for network in networks)
|
||||
|
||||
|
||||
def as_ip(value):
|
||||
"""Return an ``ipaddress`` object if *value* looks like a bare IP.
|
||||
|
||||
Accepts ``1.2.3.4``, ``2001:db8::1``, ``[2001:db8::1]:443`` and
|
||||
``1.2.3.4:5678``. Rejects CIDRs, hostnames and arbitrary integers.
|
||||
"""
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
text = value.strip().strip('"')
|
||||
if not text or len(text) > 64:
|
||||
return None
|
||||
try:
|
||||
return ipaddress.ip_address(text)
|
||||
except ValueError:
|
||||
pass
|
||||
if text.startswith("[") and "]" in text:
|
||||
try:
|
||||
return ipaddress.ip_address(text[1:text.index("]")])
|
||||
except ValueError:
|
||||
return None
|
||||
if text.count(":") == 1 and "." in text:
|
||||
host = text.split(":", 1)[0]
|
||||
try:
|
||||
return ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _empty(name):
|
||||
return {
|
||||
"name": name,
|
||||
"present": 0,
|
||||
"ip_messages": 0,
|
||||
"ip_values": 0,
|
||||
"distinct": set(),
|
||||
"distinct_ip": set(),
|
||||
"private": 0,
|
||||
"versions": set(),
|
||||
"examples": [],
|
||||
}
|
||||
|
||||
|
||||
def _analyse(messages):
|
||||
total = len(messages)
|
||||
stats = {}
|
||||
embedded = 0
|
||||
for message in messages:
|
||||
embedded_here = False
|
||||
for name, raw in message.items():
|
||||
values = raw if isinstance(raw, list) else [raw]
|
||||
entry = stats.get(name)
|
||||
if entry is None:
|
||||
entry = stats[name] = _empty(name)
|
||||
emitted = False
|
||||
has_ip = False
|
||||
for value in values:
|
||||
if value is None or value == "":
|
||||
continue
|
||||
text = str(value)[:160]
|
||||
entry["distinct"].add(text)
|
||||
emitted = True
|
||||
ip = as_ip(value)
|
||||
if ip is not None:
|
||||
has_ip = True
|
||||
entry["ip_values"] += 1
|
||||
entry["distinct_ip"].add(str(ip))
|
||||
entry["versions"].add(ip.version)
|
||||
if is_internal(ip):
|
||||
entry["private"] += 1
|
||||
text_ip = str(ip)
|
||||
if len(entry["examples"]) < 3 and text_ip not in entry["examples"]:
|
||||
entry["examples"].append(text_ip)
|
||||
if name == "message":
|
||||
embedded_here = True
|
||||
if emitted:
|
||||
entry["present"] += 1
|
||||
if has_ip:
|
||||
entry["ip_messages"] += 1
|
||||
if embedded_here:
|
||||
embedded += 1
|
||||
return stats, total, embedded
|
||||
|
||||
|
||||
def _score(name, entry, total):
|
||||
present = max(entry["present"], 1)
|
||||
ip_rate = entry["ip_messages"] / present
|
||||
presence = entry["present"] / max(total, 1)
|
||||
score = 60.0 * ip_rate + 5.0 * presence
|
||||
notes = []
|
||||
|
||||
if STRONG_HINT.search(name):
|
||||
score += 30
|
||||
elif WEAK_HINT.search(name):
|
||||
score += 10
|
||||
|
||||
if name == "source":
|
||||
score -= 20
|
||||
notes.append("`source` 是日志发送方（shipper/反代），通常不是真实客户端")
|
||||
elif name == "gl2_remote_ip":
|
||||
score -= 25
|
||||
notes.append("`gl2_remote_ip` 是向 Graylog 发送消息的对端地址，不是访问者")
|
||||
return score, notes
|
||||
|
||||
|
||||
def _confidence(best):
|
||||
if not best:
|
||||
return "none"
|
||||
if best["score"] >= 90 and best["ip_rate"] >= 0.95:
|
||||
return "high"
|
||||
if best["score"] >= 60:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def analyse_messages(messages):
|
||||
"""Build the field report used by ``graylog-ops discover``."""
|
||||
stats, total, embedded = _analyse(messages)
|
||||
if total == 0:
|
||||
return {
|
||||
"sampled": 0,
|
||||
"fields": [],
|
||||
"candidates": [],
|
||||
"http_fields": {},
|
||||
"embedded_ip_in_message_rate": 0.0,
|
||||
"recommendation": {
|
||||
"field": None,
|
||||
"confidence": "none",
|
||||
"notes": ["没有采到消息：检查时间范围、查询条件、stream 过滤或权限"],
|
||||
"warnings": [],
|
||||
},
|
||||
"sample_message": None,
|
||||
}
|
||||
|
||||
fields = []
|
||||
candidates = []
|
||||
for name, entry in stats.items():
|
||||
if entry["present"] == 0:
|
||||
continue
|
||||
ip_rate = entry["ip_messages"] / entry["present"]
|
||||
row = {
|
||||
"name": name,
|
||||
"present": entry["present"],
|
||||
"presence": round(entry["present"] / total, 4),
|
||||
"distinct": len(entry["distinct"]),
|
||||
"distinct_ips": len(entry["distinct_ip"]),
|
||||
"ip_rate": round(ip_rate, 4),
|
||||
"examples": entry["examples"],
|
||||
}
|
||||
fields.append(row)
|
||||
|
||||
if name in INTERNAL_NAMES or name.startswith(INTERNAL_PREFIXES):
|
||||
continue
|
||||
if ip_rate <= 0:
|
||||
continue
|
||||
score, notes = _score(name, entry, total)
|
||||
private_ratio = (entry["private"] / entry["ip_values"]) if entry["ip_values"] else 0.0
|
||||
candidates.append({
|
||||
"field": name,
|
||||
"score": round(score, 1),
|
||||
"ip_rate": round(ip_rate, 4),
|
||||
"presence": round(entry["present"] / total, 4),
|
||||
"distinct_ips": len(entry["distinct_ip"]),
|
||||
"private_ratio": round(private_ratio, 4),
|
||||
"versions": sorted(entry["versions"]),
|
||||
"examples": entry["examples"],
|
||||
"notes": notes,
|
||||
})
|
||||
|
||||
fields.sort(key=lambda row: (-row["present"], row["name"]))
|
||||
candidates.sort(key=lambda row: (-row["score"], -row["distinct_ips"]))
|
||||
|
||||
http_fields = {kind: [] for kind in HTTP_HINTS}
|
||||
for name in stats:
|
||||
for kind, pattern in HTTP_HINTS.items():
|
||||
if pattern.search(name):
|
||||
http_fields[kind].append(name)
|
||||
http_fields = {kind: names for kind, names in http_fields.items() if names}
|
||||
|
||||
best = candidates[0] if candidates else None
|
||||
source_entry = stats.get("source")
|
||||
warnings = []
|
||||
notes = []
|
||||
|
||||
if best:
|
||||
notes.append(
|
||||
"字段 `%s` 的 IP 占比 %.0f%%，样例 %s"
|
||||
% (best["field"], best["ip_rate"] * 100, ", ".join(best["examples"][:3]) or "-")
|
||||
)
|
||||
if source_entry and len(source_entry["distinct"]) <= 2 and best["distinct_ips"] > 2:
|
||||
warnings.append(
|
||||
"`source` 只有 %d 个不同取值，而 `%s` 有 %d 个：`source` 是采集器/反代，"
|
||||
"封禁必须用 `%s`" % (len(source_entry["distinct"]), best["field"],
|
||||
best["distinct_ips"], best["field"])
|
||||
)
|
||||
if best["private_ratio"] >= 0.5:
|
||||
warnings.append(
|
||||
"`%s` 中 %.0f%% 是内网地址（私有/回环），可能仍是 LB 或代理地址，"
|
||||
"需要确认 X-Forwarded-For 是否已在采集侧解析"
|
||||
% (best["field"], best["private_ratio"] * 100)
|
||||
)
|
||||
else:
|
||||
notes.append("没有发现任何字段存放纯 IP 值")
|
||||
|
||||
# Only worth flagging when there is no usable field: then the IP lives in
|
||||
# the free-text `message` and has to be extracted server side.
|
||||
if embedded / max(total, 1) >= 0.5 and not best:
|
||||
warnings.append(
|
||||
"%.0f%% 的消息在 `message` 文本里内嵌了 IP。要么在 input 上加 extractor / "
|
||||
"pipeline 解析成独立字段，要么在采集端（nginx log_format）直接输出 client_ip 字段"
|
||||
% (100.0 * embedded / total)
|
||||
)
|
||||
|
||||
# HTTP-ish fields help with the next phase (per-URI thresholds), surface them.
|
||||
if not best and http_fields:
|
||||
warnings.append("只找到 HTTP 相关字段，没有 IP 字段，见 http_fields")
|
||||
|
||||
return {
|
||||
"sampled": total,
|
||||
"fields": fields,
|
||||
"candidates": candidates,
|
||||
"http_fields": http_fields,
|
||||
"embedded_ip_in_message_rate": round(embedded / total, 4),
|
||||
"recommendation": {
|
||||
"field": best["field"] if best else None,
|
||||
"confidence": _confidence(best),
|
||||
"notes": notes,
|
||||
"warnings": warnings,
|
||||
},
|
||||
"sample_message": messages[0] if messages else None,
|
||||
}
|
||||
Reference in new issue
Block a user