DeepFal
2d76ed6672
- 特征检测（规则/签名）与行为检测（24h 统计）单进程运行 - 融合名单按首次发现时间倒序，展示位置/归属（GeoIP + ASN） - 钉钉融合推送（加签、24h 节流、去重） - 页面状态灯、CSV 导出、Docker 镜像构建文件
301 lines
11 KiB
Python
301 lines
11 KiB
Python
"""Field heuristics: find the field that actually holds the client IP.
|
|
|
|
The ban pipeline is only as good as the field it keys on. In a typical
|
|
deployment behind a reverse proxy / CDN, the field you *think* is the client
|
|
IP is the proxy's address:
|
|
|
|
* ``source`` -- the host that shipped the log line to Graylog
|
|
* ``gl2_remote_ip`` -- the host that opened the GELF/Beats connection
|
|
* ``remote_addr`` -- could be the proxy if XFF is not parsed
|
|
|
|
So instead of assuming, sample real messages and score every field by
|
|
"how often does this field hold an IP address" plus "does the field name look
|
|
like a client address".
|
|
"""
|
|
|
|
import ipaddress
|
|
import re
|
|
|
|
# Field names that strongly suggest a client address.
|
|
STRONG_HINT = re.compile(
|
|
r"(?i)client[_. -]?ip|remote[_. -]?addr|real[_. -]?ip|"
|
|
r"(^|[_. -])ip([_. -]|$)|x[_. -]?forwarded[_. -]?for|"
|
|
r"(^|[_. -])xff([_. -]|$)|src[_. -]?ip|c[_. -]?ip"
|
|
)
|
|
|
|
WEAK_HINT = re.compile(
|
|
r"(?i)addr|host|client|proxy|peer|origin|caller|visitor|remote|forward|endpoint"
|
|
)
|
|
|
|
# Graylog internals that are never the client IP (gl2_remote_ip is the shipper).
|
|
INTERNAL_PREFIXES = ("gl2_", "_")
|
|
INTERNAL_NAMES = ("timestamp", "streams", "message")
|
|
|
|
HTTP_HINTS = {
|
|
"status": re.compile(r"(?i)^(http[_. ]?)?status([_. ]?code)?$|^response([_. ]?(code|status))?$|^code$"),
|
|
"uri": re.compile(r"(?i)(request[_. ]?)?uri$|^url$|^path$|^request$|^endpoint$|^target$"),
|
|
"method": re.compile(r"(?i)^(http[_. ]?)?method$|^verb$"),
|
|
"user_agent": re.compile(r"(?i)user[_. -]?agent|^ua$|^agent$"),
|
|
"host": re.compile(r"(?i)^(http[_. ]?)?host$|^vhost$|^domain$|^server_name$"),
|
|
"bytes": re.compile(r"(?i)bytes|^size$|^body_bytes_sent$"),
|
|
"referer": re.compile(r"(?i)refer(r)?er$"),
|
|
"duration": re.compile(r"(?i)request[_. ]?time|response[_. ]?time|duration|latency|elapsed"),
|
|
}
|
|
|
|
# Deliberately not using ``ipaddress.is_private``: that property also covers
|
|
# documentation ranges (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) and
|
|
# other special-use blocks, which would make every public test address look
|
|
# like an internal proxy. For ban decisions "internal" means RFC1918 plus
|
|
# loopback, link-local and CGNAT.
|
|
_INTERNAL_V4 = tuple(ipaddress.ip_network(cidr) for cidr in (
|
|
"10.0.0.0/8",
|
|
"172.16.0.0/12",
|
|
"192.168.0.0/16",
|
|
"100.64.0.0/10", # CGNAT
|
|
"127.0.0.0/8",
|
|
"169.254.0.0/16",
|
|
))
|
|
_INTERNAL_V6 = tuple(ipaddress.ip_network(cidr) for cidr in (
|
|
"::1/128",
|
|
"fc00::/7", # unique local
|
|
"fe80::/10", # link local
|
|
))
|
|
|
|
|
|
def is_internal(ip):
|
|
networks = _INTERNAL_V4 if ip.version == 4 else _INTERNAL_V6
|
|
return any(ip in network for network in networks)
|
|
|
|
|
|
def as_ip(value):
|
|
"""Return an ``ipaddress`` object if *value* looks like a bare IP.
|
|
|
|
Accepts ``1.2.3.4``, ``2001:db8::1``, ``[2001:db8::1]:443`` and
|
|
``1.2.3.4:5678``. Rejects CIDRs, hostnames and arbitrary integers.
|
|
"""
|
|
if not isinstance(value, str):
|
|
return None
|
|
text = value.strip().strip('"')
|
|
if not text or len(text) > 64:
|
|
return None
|
|
try:
|
|
return ipaddress.ip_address(text)
|
|
except ValueError:
|
|
pass
|
|
if text.startswith("[") and "]" in text:
|
|
try:
|
|
return ipaddress.ip_address(text[1:text.index("]")])
|
|
except ValueError:
|
|
return None
|
|
if text.count(":") == 1 and "." in text:
|
|
host = text.split(":", 1)[0]
|
|
try:
|
|
return ipaddress.ip_address(host)
|
|
except ValueError:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _empty(name):
|
|
return {
|
|
"name": name,
|
|
"present": 0,
|
|
"ip_messages": 0,
|
|
"ip_values": 0,
|
|
"distinct": set(),
|
|
"distinct_ip": set(),
|
|
"private": 0,
|
|
"versions": set(),
|
|
"examples": [],
|
|
}
|
|
|
|
|
|
def _analyse(messages):
|
|
total = len(messages)
|
|
stats = {}
|
|
embedded = 0
|
|
for message in messages:
|
|
embedded_here = False
|
|
for name, raw in message.items():
|
|
values = raw if isinstance(raw, list) else [raw]
|
|
entry = stats.get(name)
|
|
if entry is None:
|
|
entry = stats[name] = _empty(name)
|
|
emitted = False
|
|
has_ip = False
|
|
for value in values:
|
|
if value is None or value == "":
|
|
continue
|
|
text = str(value)[:160]
|
|
entry["distinct"].add(text)
|
|
emitted = True
|
|
ip = as_ip(value)
|
|
if ip is not None:
|
|
has_ip = True
|
|
entry["ip_values"] += 1
|
|
entry["distinct_ip"].add(str(ip))
|
|
entry["versions"].add(ip.version)
|
|
if is_internal(ip):
|
|
entry["private"] += 1
|
|
text_ip = str(ip)
|
|
if len(entry["examples"]) < 3 and text_ip not in entry["examples"]:
|
|
entry["examples"].append(text_ip)
|
|
if name == "message":
|
|
embedded_here = True
|
|
if emitted:
|
|
entry["present"] += 1
|
|
if has_ip:
|
|
entry["ip_messages"] += 1
|
|
if embedded_here:
|
|
embedded += 1
|
|
return stats, total, embedded
|
|
|
|
|
|
def _score(name, entry, total):
|
|
present = max(entry["present"], 1)
|
|
ip_rate = entry["ip_messages"] / present
|
|
presence = entry["present"] / max(total, 1)
|
|
score = 60.0 * ip_rate + 5.0 * presence
|
|
notes = []
|
|
|
|
if STRONG_HINT.search(name):
|
|
score += 30
|
|
elif WEAK_HINT.search(name):
|
|
score += 10
|
|
|
|
if name == "source":
|
|
score -= 20
|
|
notes.append("`source` 是日志发送方（shipper/反代），通常不是真实客户端")
|
|
elif name == "gl2_remote_ip":
|
|
score -= 25
|
|
notes.append("`gl2_remote_ip` 是向 Graylog 发送消息的对端地址，不是访问者")
|
|
return score, notes
|
|
|
|
|
|
def _confidence(best):
|
|
if not best:
|
|
return "none"
|
|
if best["score"] >= 90 and best["ip_rate"] >= 0.95:
|
|
return "high"
|
|
if best["score"] >= 60:
|
|
return "medium"
|
|
return "low"
|
|
|
|
|
|
def analyse_messages(messages):
|
|
"""Build the field report used by ``graylog-ops discover``."""
|
|
stats, total, embedded = _analyse(messages)
|
|
if total == 0:
|
|
return {
|
|
"sampled": 0,
|
|
"fields": [],
|
|
"candidates": [],
|
|
"http_fields": {},
|
|
"embedded_ip_in_message_rate": 0.0,
|
|
"recommendation": {
|
|
"field": None,
|
|
"confidence": "none",
|
|
"notes": ["没有采到消息：检查时间范围、查询条件、stream 过滤或权限"],
|
|
"warnings": [],
|
|
},
|
|
"sample_message": None,
|
|
}
|
|
|
|
fields = []
|
|
candidates = []
|
|
for name, entry in stats.items():
|
|
if entry["present"] == 0:
|
|
continue
|
|
ip_rate = entry["ip_messages"] / entry["present"]
|
|
row = {
|
|
"name": name,
|
|
"present": entry["present"],
|
|
"presence": round(entry["present"] / total, 4),
|
|
"distinct": len(entry["distinct"]),
|
|
"distinct_ips": len(entry["distinct_ip"]),
|
|
"ip_rate": round(ip_rate, 4),
|
|
"examples": entry["examples"],
|
|
}
|
|
fields.append(row)
|
|
|
|
if name in INTERNAL_NAMES or name.startswith(INTERNAL_PREFIXES):
|
|
continue
|
|
if ip_rate <= 0:
|
|
continue
|
|
score, notes = _score(name, entry, total)
|
|
private_ratio = (entry["private"] / entry["ip_values"]) if entry["ip_values"] else 0.0
|
|
candidates.append({
|
|
"field": name,
|
|
"score": round(score, 1),
|
|
"ip_rate": round(ip_rate, 4),
|
|
"presence": round(entry["present"] / total, 4),
|
|
"distinct_ips": len(entry["distinct_ip"]),
|
|
"private_ratio": round(private_ratio, 4),
|
|
"versions": sorted(entry["versions"]),
|
|
"examples": entry["examples"],
|
|
"notes": notes,
|
|
})
|
|
|
|
fields.sort(key=lambda row: (-row["present"], row["name"]))
|
|
candidates.sort(key=lambda row: (-row["score"], -row["distinct_ips"]))
|
|
|
|
http_fields = {kind: [] for kind in HTTP_HINTS}
|
|
for name in stats:
|
|
for kind, pattern in HTTP_HINTS.items():
|
|
if pattern.search(name):
|
|
http_fields[kind].append(name)
|
|
http_fields = {kind: names for kind, names in http_fields.items() if names}
|
|
|
|
best = candidates[0] if candidates else None
|
|
source_entry = stats.get("source")
|
|
warnings = []
|
|
notes = []
|
|
|
|
if best:
|
|
notes.append(
|
|
"字段 `%s` 的 IP 占比 %.0f%%，样例 %s"
|
|
% (best["field"], best["ip_rate"] * 100, ", ".join(best["examples"][:3]) or "-")
|
|
)
|
|
if source_entry and len(source_entry["distinct"]) <= 2 and best["distinct_ips"] > 2:
|
|
warnings.append(
|
|
"`source` 只有 %d 个不同取值，而 `%s` 有 %d 个：`source` 是采集器/反代，"
|
|
"封禁必须用 `%s`" % (len(source_entry["distinct"]), best["field"],
|
|
best["distinct_ips"], best["field"])
|
|
)
|
|
if best["private_ratio"] >= 0.5:
|
|
warnings.append(
|
|
"`%s` 中 %.0f%% 是内网地址（私有/回环），可能仍是 LB 或代理地址，"
|
|
"需要确认 X-Forwarded-For 是否已在采集侧解析"
|
|
% (best["field"], best["private_ratio"] * 100)
|
|
)
|
|
else:
|
|
notes.append("没有发现任何字段存放纯 IP 值")
|
|
|
|
# Only worth flagging when there is no usable field: then the IP lives in
|
|
# the free-text `message` and has to be extracted server side.
|
|
if embedded / max(total, 1) >= 0.5 and not best:
|
|
warnings.append(
|
|
"%.0f%% 的消息在 `message` 文本里内嵌了 IP。要么在 input 上加 extractor / "
|
|
"pipeline 解析成独立字段，要么在采集端（nginx log_format）直接输出 client_ip 字段"
|
|
% (100.0 * embedded / total)
|
|
)
|
|
|
|
# HTTP-ish fields help with the next phase (per-URI thresholds), surface them.
|
|
if not best and http_fields:
|
|
warnings.append("只找到 HTTP 相关字段，没有 IP 字段，见 http_fields")
|
|
|
|
return {
|
|
"sampled": total,
|
|
"fields": fields,
|
|
"candidates": candidates,
|
|
"http_fields": http_fields,
|
|
"embedded_ip_in_message_rate": round(embedded / total, 4),
|
|
"recommendation": {
|
|
"field": best["field"] if best else None,
|
|
"confidence": _confidence(best),
|
|
"notes": notes,
|
|
"warnings": warnings,
|
|
},
|
|
"sample_message": messages[0] if messages else None,
|
|
}
|