DeepFal/graylog-ops
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
897828f506bae39e5eeac88ce04e10ce829b26df
graylog-ops/graylog_ops/watch.py
T

2802 lines
123 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Continuous malicious-IP detection, persistent list, local web dashboard.
Design
------
* Every ``interval`` seconds the detector runs a **fixed set of terms queries**
over the last ``window`` seconds, so the load on Elasticsearch does not grow
with the number of IPs.
* Rules were derived from an 8 hour exploration of the F5 access logs
(2026-10-17). The four observed patterns:
1. 漏洞探测 -- low volume, hits /actuator, /.env, /wp-login, /druid, /nacos,
/swagger, /ISAPI, /console ... (very high signal, few requests)
2. 登录爆破 -- repeated 401/403 on /login, /passport, /captcha endpoints
3. 爬虫/机器人 -- known bot UA (DotBot) + high 404 ratio + many content URIs
4. 自动化客户端 -- one single business API at a steady rate (a broken token,
not an attacker) -> downgraded, never bans
* Every candidate is persisted in SQLite (``data/ips.db``) so the list survives
restarts, and the same process serves a small read-only dashboard on
``http://127.0.0.1:8088``.
"""
import csv
import html
import io
import ipaddress
import re
import json
import os
import sqlite3
import threading
import time
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlparse
from . import detect
from .client import normalize_histogram, normalize_terms
from .http import GraylogError
from .notify import load_notifier, notifier_from_env
from .behavior import BehaviorAnalyzer, notify_fused, RULE_TITLES
from .asn import AsnDB
# --------------------------------------------------------------------- rules
# Queries run against Graylog's query_string. Field names are substituted from
# the profile (client_ip / status / uri / ua). Slashes must be escaped: an
# unescaped /wp-login* would be parsed as a Lucene regex.
PROBE_RULES = ("probe_java", "probe_config", "probe_cms", "probe_iot",
"probe_devops", "probe_webshell", "exploit_payload")
RULE_DEFS = (
{
"name": "probe_java",
"title": "Spring/中间件探测",
"weight": 60.0,
"saturate": 2.0,
"query": lambda f: (
"%s:(\\/actuator* OR \\/druid* OR \\/nacos* OR \\/swagger* "
"OR \\/v2\\/api-docs* OR \\/v3\\/api-docs* OR \\/openapi* OR \\/jmx* "
"OR \\/heapdump* OR \\/threaddump* OR \\/console* OR \\/manager\\/html* "
"OR \\/solr* OR \\/struts* OR \\/weblogic* OR \\/log4j*)"
) % f["uri"],
},
{
"name": "probe_config",
"title": "配置/凭证探测",
"weight": 60.0,
"saturate": 2.0,
"query": lambda f: (
"%s:(\\/.env* OR \\/.git* OR \\/.svn* OR \\/.hg* OR \\/.aws* OR \\/.ssh* "
"OR \\/.kube* OR \\/.docker* OR \\/appsettings* OR \\/web.config "
"OR \\/WEB-INF* OR \\/META-INF* OR \\/backup* OR \\/dump* "
"OR \\/db.sql* OR \\/.DS_Store)"
) % f["uri"],
},
{
"name": "probe_cms",
"title": "CMS 探测",
"weight": 60.0,
"saturate": 2.0,
"query": lambda f: (
"%s:(\\/wp-* OR \\/xmlrpc.php OR \\/phpmyadmin* OR \\/pma* "
"OR \\/administrator* OR \\/joomla* OR \\/drupal* OR \\/thinkphp* "
"OR \\/uc_server* OR \\/dede* OR \\/index.php?s=*)"
) % f["uri"],
},
{
"name": "probe_iot",
"title": "IoT/路由器探测",
"weight": 60.0,
"saturate": 2.0,
"query": lambda f: (
"%s:(\\/ISAPI* OR \\/LAPI* OR \\/HNAP1* OR \\/boaform* OR \\/GponForm* "
"OR \\/cgi-bin\\/luci* OR \\/goform* OR \\/setup.cgi* OR \\/login.cgi*)"
) % f["uri"],
},
{
"name": "probe_devops",
"title": "DevOps/监控探测",
"weight": 60.0,
"saturate": 2.0,
"query": lambda f: (
"%s:(\\/jenkins* OR \\/grafana* OR \\/kibana* OR \\/zabbix* "
"OR \\/prometheus* OR \\/consul* OR \\/eureka* OR \\/harbor* "
"OR \\/nexus* OR \\/sonarqube* OR \\/gitlab* OR \\/portainer*)"
) % f["uri"],
},
{
"name": "probe_webshell",
"title": "WebShell/上传探测",
"weight": 60.0,
"saturate": 2.0,
"query": lambda f: (
"%s:(\\/shell.php OR \\/shell.asp OR \\/shell.jsp OR \\/cmd.php "
"OR \\/cmd.asp OR \\/webshell* OR \\/phpinfo* OR \\/dbadmin* "
"OR \\/eval-stdin* OR \\/vendor\\/phpunit*)"
) % f["uri"],
},
{
# Regex: only over 1 hour max, a 25h regex aggregation kills the ES node.
"name": "exploit_payload",
"title": "注入/穿越载荷",
"weight": 65.0,
"saturate": 2.0,
"window_cap": 3600,
"query": lambda f: (
"%s:/.*(etc\\/passwd|proc\\/self|jndi:|union.*select|information_schema"
"|sleep\\(|benchmark\\(|\\.\\.%%2f|%%2e%%2e|%%252f|base64_decode|xp_cmdshell).*/"
) % f["uri"],
},
{
"name": "login_attack",
"title": "登录失败尝试",
"weight": 55.0,
"saturate": 8.0,
"query": lambda f: (
"(%s:(\\/login* OR \\/passport* OR \\/captcha* OR \\/jsxy\\/api\\/login*)) "
"AND %s:(401 OR 403)"
) % (f["uri"], f["status"]),
},
{
"name": "login_burst",
"title": "登录接口高频",
"weight": 15.0,
"saturate": 2000.0,
"query": lambda f: (
"%s:(\\/login* OR \\/passport* OR \\/captcha* OR \\/jsxy\\/api\\/login*)"
) % f["uri"],
},
{
"name": "bad_ua",
"title": "扫描器 UA",
"weight": 55.0,
"saturate": 1.0,
"query": lambda f: (
"%s:(sqlmap* OR nikto* OR nuclei* OR nmap* OR masscan* OR zgrab* "
"OR wpscan* OR gobuster* OR dirsearch* OR dirbuster* OR ffuf* "
"OR feroxbuster* OR wfuzz* OR joomscan* OR droopescan* OR whatweb* "
"OR xray* OR vulmap* OR afrog* OR hydra*)"
) % f["ua"],
},
{
"name": "bot_ua",
"title": "爬虫 UA",
"weight": 50.0,
"saturate": 20.0,
"query": lambda f: (
'%s:"Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)"'
) % f["ua"],
},
{
"name": "not_found",
"title": "404",
"weight": 0.0, # scored via the 404 ratio, not the raw count
"saturate": 1.0,
"query": lambda f: "%s:404" % f["status"],
},
)
DEFAULT_FIELDS = {
"client_ip": "client_ip",
"status": "status",
"uri": "uri",
"ua": "ua",
"method": "method",
"host": "host",
"server_ip": "server_ip",
"bigip": "bigip",
}
AUTH_PATH_HINTS = ("login", "signin", "auth", "token", "captcha", "oauth", "password")
# How far back to look when dating the first attack of an IP.
BACKFILL_WINDOW = 129600 # ~36h, as far back as the index retention allows
TIER_MALICIOUS = "malicious"
TIER_SUSPICIOUS = "suspicious"
TIER_AUTOMATION = "automation"
TIER_OBSERVE = "observe"
TIER_LABELS = {
TIER_MALICIOUS: "恶意",
TIER_SUSPICIOUS: "可疑",
TIER_AUTOMATION: "自动化",
TIER_OBSERVE: "观察",
}
# ------------------------------------------------------------------ classify
RATE_RULES = ("login_attack", "login_burst")
def classify(total, hits, window_scale=1.0):
"""Pure scoring function: request counts per rule -> (tier, score, reasons).
``window_scale`` normalises rate-like rules to a 15 minute baseline, so an
8 hour analysis does not turn a student who mistyped a password six times
into a "suspicious IP". Behavioural rules (probing / bad UA / crawler) are
never scaled: any probing is bad regardless of window length.
"""
scale = max(1.0, float(window_scale))
score = 0.0
reasons = []
def add(rule, title, count):
nonlocal score
if count <= 0:
return
saturate = rule["saturate"]
if rule["name"] in RATE_RULES:
saturate *= scale
points = rule["weight"] * min(1.0, count / saturate)
score += points
reasons.append("%s %d 次" % (title, count))
for rule in RULE_DEFS:
if rule["weight"] <= 0:
continue
add(rule, rule["title"], hits.get(rule["name"], 0))
not_found = hits.get("not_found", 0)
if not_found and total:
ratio = not_found / float(total)
if ratio >= 0.5 and not_found >= 20:
score += 25.0 * min(1.0, ratio / 0.8)
reasons.append("404 占比 %.0f%%（%d/%d）" % (ratio * 100, not_found, total))
score = min(100.0, round(score, 1))
if score >= 60:
tier = TIER_MALICIOUS
elif score >= 30:
tier = TIER_SUSPICIOUS
elif score > 0:
tier = TIER_OBSERVE
else:
tier = TIER_OBSERVE
return tier, score, reasons
def is_public(text):
try:
return not detect.is_internal(detect.as_ip(text))
except (TypeError, ValueError):
return False
# --------------------------------------------------------------------- store
class Store:
def __init__(self, path):
directory = os.path.dirname(os.path.abspath(path))
if directory:
os.makedirs(directory, exist_ok=True)
self.path = path
self.lock = threading.Lock()
self.conn = sqlite3.connect(path, check_same_thread=False)
try:
self.conn.execute("PRAGMA journal_mode=WAL")
self.conn.execute("PRAGMA busy_timeout=5000")
except sqlite3.Error:
pass
self.conn.row_factory = sqlite3.Row
self._init()
def _init(self):
with self.conn:
self.conn.execute(
"""CREATE TABLE IF NOT EXISTS ip_stats (
ip TEXT PRIMARY KEY,
tier TEXT NOT NULL,
score REAL NOT NULL,
total INTEGER NOT NULL DEFAULT 0,
rules TEXT NOT NULL DEFAULT '{}',
reasons TEXT NOT NULL DEFAULT '[]',
first_seen TEXT,
last_seen TEXT,
risk TEXT,
window INTEGER,
evidence TEXT,
evidence_at TEXT,
ever_malicious INTEGER NOT NULL DEFAULT 0,
last_malicious_at TEXT,
hits INTEGER NOT NULL DEFAULT 1,
updated_at TEXT
)"""
)
for column in ("risk TEXT", "window INTEGER", "evidence TEXT",
"evidence_at TEXT", "ever_malicious INTEGER DEFAULT 0",
"last_malicious_at TEXT", "country_code TEXT",
"country_name TEXT", "region TEXT", "city TEXT",
"asn TEXT", "asn_org TEXT", "asn_brand TEXT",
"asn_type TEXT", "asn_icon TEXT"):
try:
self.conn.execute("ALTER TABLE ip_stats ADD COLUMN %s" % column)
except sqlite3.OperationalError:
pass # column already exists
try:
self.conn.execute(
"""UPDATE ip_stats SET ever_malicious = 1,
last_malicious_at = COALESCE(
last_malicious_at,
(SELECT MAX(e.ts) FROM events e
WHERE e.ip = ip_stats.ip AND e.tier = 'malicious'),
last_seen)
WHERE COALESCE(ever_malicious, 0) = 0 AND (
tier = 'malicious' OR EXISTS (
SELECT 1 FROM events e
WHERE e.ip = ip_stats.ip
AND (e.tier = 'malicious'
OR e.detail LIKE '%malicious%')))"""
)
except sqlite3.OperationalError:
pass # events table may not exist yet on a fresh db
self.conn.execute(
"""CREATE TABLE IF NOT EXISTS behavior_profiles (
ip TEXT PRIMARY KEY,
score REAL, verdict TEXT, signals TEXT, reason_text TEXT,
share_note TEXT, behavior_type TEXT,
asn TEXT, asn_org TEXT, shared_type TEXT,
asn_brand TEXT, asn_type TEXT, asn_icon TEXT,
total INTEGER, pct4 REAL, uri_card INTEGER,
uri_templates INTEGER, ua_card INTEGER, host_card INTEGER,
coverage REAL, peak_share REAL, top_uri TEXT, top_ua TEXT,
last_ts TEXT, updated_at TEXT,
country_code TEXT, country_name TEXT, region TEXT, city TEXT
)"""
)
for column in ("asn_brand TEXT", "asn_type TEXT", "asn_icon TEXT",
"historical INTEGER DEFAULT 0"):
try:
self.conn.execute(
"ALTER TABLE behavior_profiles ADD COLUMN %s" % column)
except sqlite3.OperationalError:
pass
self.conn.execute(
"""CREATE TABLE IF NOT EXISTS ip_fused (
ip TEXT PRIMARY KEY,
sources TEXT NOT NULL,
feature_score REAL, behavior_score REAL,
first_banned_at TEXT, last_activity_at TEXT,
last_pushed_at TEXT, push_count INTEGER NOT NULL DEFAULT 0,
fused_reason TEXT, updated_at TEXT
)"""
)
self.conn.execute(
"""CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts TEXT, ip TEXT, tier TEXT, score REAL, detail TEXT
)"""
)
def upsert(self, record, now):
with self.lock, self.conn:
row = self.conn.execute(
"SELECT tier, score, ever_malicious, last_malicious_at, "
"last_seen, rules, total FROM ip_stats WHERE ip = ?",
(record["ip"],),
).fetchone()
malicious = record["tier"] == TIER_MALICIOUS
geo = record.get("geo") or {}
info = record.get("asn") or {}
if row is None:
last_seen = record.get("last_seen") or now
last_malicious_at = record.get("last_malicious_at") or (
last_seen if malicious else None)
self.conn.execute(
"""INSERT INTO ip_stats
(ip, tier, score, total, rules, reasons,
first_seen, last_seen, risk, window, hits, updated_at,
ever_malicious, last_malicious_at,
country_code, country_name, region, city,
asn, asn_org, asn_brand, asn_type, asn_icon)
VALUES (?,?,?,?,?,?,?,?,?,?,1,?,?,?,?,?,?,?,?,?,?,?,?)""",
(record["ip"], record["tier"], record["score"], record["total"],
json.dumps(record["rules"], ensure_ascii=False),
json.dumps(record["reasons"], ensure_ascii=False),
record.get("first_seen") or now, last_seen,
record.get("risk"), record.get("window"), now,
1 if malicious else 0, last_malicious_at,
geo.get("country_code"), geo.get("country_name"),
geo.get("region"), geo.get("city"),
info.get("asn"), info.get("org"), info.get("brand"),
info.get("type"), info.get("icon")),
)
self._event(now, record, "new")
return "new"
else:
ever = 1 if (malicious or row["ever_malicious"]) else 0
# Timestamps only move when the detector verified new evidence,
# otherwise a stale window would keep refreshing them to "now".
last_seen = record.get("last_seen") or row["last_seen"] or now
if record.get("last_malicious_at"):
last_malicious_at = record["last_malicious_at"]
elif malicious and row["tier"] != TIER_MALICIOUS:
last_malicious_at = last_seen
else:
last_malicious_at = row["last_malicious_at"]
self.conn.execute(
"""UPDATE ip_stats
SET tier=?, score=?, total=?, rules=?, reasons=?,
last_seen=?, risk=?, window=?, hits=hits+1, updated_at=?,
ever_malicious=?, last_malicious_at=?,
country_code=COALESCE(?, country_code),
country_name=COALESCE(?, country_name),
region=COALESCE(?, region),
city=COALESCE(?, city),
asn=COALESCE(?, asn), asn_org=COALESCE(?, asn_org),
asn_brand=COALESCE(?, asn_brand),
asn_type=COALESCE(?, asn_type),
asn_icon=COALESCE(?, asn_icon)
WHERE ip=?""",
(record["tier"], record["score"], record["total"],
json.dumps(record["rules"], ensure_ascii=False),
json.dumps(record["reasons"], ensure_ascii=False),
last_seen, record.get("risk"), record.get("window"), now,
ever, last_malicious_at,
geo.get("country_code"), geo.get("country_name"),
geo.get("region"), geo.get("city"),
info.get("asn"), info.get("org"), info.get("brand"),
info.get("type"), info.get("icon"), record["ip"]),
)
if row["tier"] != record["tier"]:
self._event(now, record, "%s->%s" % (row["tier"], record["tier"]))
if malicious and row["tier"] != TIER_MALICIOUS:
return "upgraded"
return "unchanged" if row["tier"] == record["tier"] else "changed"
def _event(self, now, record, detail):
self.conn.execute(
"INSERT INTO events (ts, ip, tier, score, detail) VALUES (?,?,?,?,?)",
(now, record["ip"], record["tier"], record["score"], detail),
)
SORT_COLUMNS = {"first_seen": "first_seen",
"last_malicious": "last_malicious_at",
"score": "score",
"total": "total"}
def list(self, limit=500, tier=None, scope=None, sort="first_seen", order="desc"):
query = ("SELECT ip, tier, score, total, rules, reasons, first_seen, "
"last_seen, risk, window, hits, updated_at, evidence_at, "
"ever_malicious, last_malicious_at, country_code, country_name, "
"region, city, asn, asn_org, asn_brand, asn_type, asn_icon "
"FROM ip_stats")
args = []
where = []
if tier:
where.append("tier = ?")
args.append(tier)
if scope == "core":
where.append("ever_malicious = 1")
if where:
query += " WHERE " + " AND ".join(where)
column = self.SORT_COLUMNS.get(sort, "first_seen")
direction = "DESC" if str(order).lower() == "desc" else "ASC"
query += (" ORDER BY (%s IS NULL) ASC, %s %s, "
"COALESCE(last_malicious_at, last_seen) DESC, ip ASC LIMIT ?"
% (column, column, direction))
args.append(limit)
with self.lock:
rows = self.conn.execute(query, args).fetchall()
result = []
for row in rows:
item = dict(row)
item["rules"] = json.loads(item["rules"] or "{}")
item["reasons"] = json.loads(item["reasons"] or "[]")
result.append(item)
return result
def get(self, ip):
with self.lock:
row = self.conn.execute("SELECT * FROM ip_stats WHERE ip = ?", (ip,)).fetchone()
if row is None:
return None
item = dict(row)
item["rules"] = json.loads(item["rules"] or "{}")
item["reasons"] = json.loads(item["reasons"] or "[]")
return item
# ---------------------------------------------------------- behaviour
def upsert_historical_profiles(self, items):
"""Insert backfilled behaviour profiles; live rows win on conflict."""
stamp = now_iso()
with self.lock, self.conn:
for item in items:
self.conn.execute(
"""INSERT OR IGNORE INTO behavior_profiles
(ip, score, verdict, signals, reason_text, share_note,
behavior_type, asn, asn_org, shared_type, asn_brand,
asn_type, asn_icon, total, pct4, uri_card, uri_templates,
ua_card, host_card, coverage, peak_share, top_uri, top_ua,
last_ts, updated_at, country_code, country_name, region,
city, historical)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1)""",
(item["ip"], item.get("score"), item.get("verdict"),
json.dumps(item.get("signals") or [], ensure_ascii=False),
item.get("reason_text"), item.get("share_note"),
item.get("behavior_type"), item.get("asn"),
item.get("asn_org"), item.get("shared_type"),
item.get("asn_brand"), item.get("asn_type"),
item.get("asn_icon"), int(item.get("total") or 0),
item.get("pct4"), item.get("uri_card"),
item.get("uri_templates"), item.get("ua_card"),
item.get("host_card"), item.get("coverage"),
item.get("peak_share"), item.get("top_uri"),
item.get("top_ua"), item.get("last_ts") or stamp, stamp,
item.get("country_code"), item.get("country_name"),
item.get("region"), item.get("city")))
def replace_behavior_profiles(self, items):
stamp = now_iso()
with self.lock, self.conn:
self.conn.execute(
"DELETE FROM behavior_profiles WHERE COALESCE(historical,0)=0")
for item in items:
self.conn.execute(
"""INSERT OR REPLACE INTO behavior_profiles
(ip, score, verdict, signals, reason_text, share_note,
behavior_type, asn, asn_org, shared_type, asn_brand,
asn_type, asn_icon, total, pct4, uri_card, uri_templates,
ua_card, host_card, coverage, peak_share, top_uri, top_ua,
last_ts, updated_at, country_code, country_name, region,
city, historical)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0)""",
(item["ip"], item.get("score"), item.get("verdict"),
json.dumps(item.get("signals") or [], ensure_ascii=False),
item.get("reason_text"), item.get("share_note"),
item.get("behavior_type"), item.get("asn"),
item.get("asn_org"), item.get("shared_type"),
item.get("asn_brand"), item.get("asn_type"),
item.get("asn_icon"),
int(item.get("total") or 0), item.get("pct4"),
int(item.get("uri_card") or 0),
int(item.get("uri_templates") or 0),
int(item.get("ua_card") or 0),
int(item.get("host_card") or 0), item.get("coverage"),
item.get("peak_share"), item.get("top_uri"),
item.get("top_ua"), stamp, stamp,
item.get("country_code"), item.get("country_name"),
item.get("region"), item.get("city")),
)
def behavior_profile(self, ip):
with self.lock:
row = self.conn.execute(
"SELECT * FROM behavior_profiles WHERE ip = ?", (ip,)).fetchone()
if row is None:
return None
item = dict(row)
try:
item["signals"] = json.loads(item.get("signals") or "[]")
except ValueError:
item["signals"] = []
return item
def fused_row(self, ip):
with self.lock:
row = self.conn.execute(
"SELECT * FROM ip_fused WHERE ip = ?", (ip,)).fetchone()
return dict(row) if row else None
def fused_items(self, limit=500):
with self.lock:
rows = self.conn.execute(
"""SELECT f.*, s.tier AS tier, s.total AS total, s.rules AS s_rules,
b.total AS b_total,
s.asn AS s_asn, s.asn_org AS s_asn_org,
s.asn_brand AS s_asn_brand, s.asn_type AS s_asn_type,
s.asn_icon AS s_asn_icon,
b.asn AS b_asn, b.asn_org AS b_asn_org,
b.asn_brand AS b_asn_brand, b.asn_type AS b_asn_type,
b.asn_icon AS b_asn_icon,
s.country_code AS s_cc, s.country_name AS s_cn,
s.region AS s_region, s.city AS s_city,
b.country_code AS b_cc, b.country_name AS b_cn,
b.region AS b_region, b.city AS b_city
FROM ip_fused f
LEFT JOIN ip_stats s ON s.ip = f.ip
LEFT JOIN behavior_profiles b ON b.ip = f.ip
ORDER BY f.first_banned_at DESC, f.ip ASC
LIMIT ?""", (int(limit),)).fetchall()
items = []
for row in rows:
item = dict(row)
item["country_code"] = item.get("s_cc") or item.get("b_cc") or ""
item["country_name"] = item.get("s_cn") or item.get("b_cn") or ""
item["region"] = item.get("s_region") or item.get("b_region") or ""
item["city"] = item.get("s_city") or item.get("b_city") or ""
item["asn"] = item.get("s_asn") or item.get("b_asn") or ""
item["asn_org"] = item.get("s_asn_org") or item.get("b_asn_org") or ""
item["asn_brand"] = item.get("s_asn_brand") or item.get("b_asn_brand") or ""
item["asn_type"] = item.get("s_asn_type") or item.get("b_asn_type") or ""
item["asn_icon"] = item.get("s_asn_icon") or item.get("b_asn_icon") or ""
item["total"] = item.get("total") or item.get("b_total") or 0
items.append(item)
return items
def _fused_reason(self, feature_rules, behavior_reason):
parts = []
if feature_rules:
try:
rules = json.loads(feature_rules or "{}")
except ValueError:
rules = {}
if rules:
ranked = sorted(rules.items(), key=lambda kv: -kv[1])[:3]
parts.append("特征命中：" + "、".join(
"%s %d 次" % (RULE_TITLES.get(name, name), count)
for name, count in ranked))
if behavior_reason:
parts.append("行为表现：" + behavior_reason)
return "；".join(parts)
def sync_fused(self, now=None):
"""Merge feature + behaviour verdicts into the fused block list.
Rule: feature-confirmed IPs always stay (option A, historical list);
behaviour adds IPs only when its verdict is "建议封禁" (shared/NAT
networks never enter the block list). first_banned_at is pinned so an
IP only reaches the top of the list once.
"""
now = now or now_iso()
with self.lock, self.conn:
feature_rows = self.conn.execute(
"SELECT ip, score, rules, first_seen, last_seen, "
"last_malicious_at, country_code, country_name, region, city "
"FROM ip_stats WHERE ever_malicious = 1").fetchall()
behavior_rows = self.conn.execute(
"SELECT * FROM behavior_profiles").fetchall()
existing = {row["ip"]: row for row in self.conn.execute(
"SELECT * FROM ip_fused").fetchall()}
initial = not existing
behavior = {row["ip"]: row for row in behavior_rows}
wanted = {}
def iso(value):
return value or now
for row in feature_rows:
ip = row["ip"]
beh = behavior.get(ip)
beh_ban = bool(beh and beh["verdict"] == "建议封禁")
prev = existing.get(ip)
first = (prev["first_banned_at"] if prev
else iso(row["first_seen"] or row["last_malicious_at"]))
activity = max(filter(None, [row["last_seen"],
beh["last_ts"] if beh else None]),
default=now)
wanted[ip] = {
"sources": "both" if beh_ban else "feature",
"feature_score": row["score"],
"behavior_score": beh["score"] if beh else None,
"first_banned_at": first,
"last_activity_at": iso(activity),
"fused_reason": self._fused_reason(
row["rules"], beh["reason_text"] if beh else None),
}
for ip, beh in behavior.items():
if beh["verdict"] != "建议封禁" or ip in wanted:
continue
prev = existing.get(ip)
historical = bool(beh["historical"]) if "historical" in beh.keys() else False
fallback = (beh["last_ts"] if historical and beh["last_ts"] else now)
wanted[ip] = {
"sources": "behavior",
"feature_score": None,
"behavior_score": beh["score"],
"first_banned_at": prev["first_banned_at"] if prev else fallback,
"last_activity_at": iso(beh["last_ts"]),
"fused_reason": "行为表现：" + (beh["reason_text"] or ""),
}
for ip, data in wanted.items():
prev = existing.get(ip)
if prev:
self.conn.execute(
"""UPDATE ip_fused
SET sources=?, feature_score=?, behavior_score=?,
last_activity_at=?, fused_reason=?, updated_at=?
WHERE ip=?""",
(data["sources"], data["feature_score"],
data["behavior_score"], data["last_activity_at"],
data["fused_reason"], now, ip))
else:
self.conn.execute(
"""INSERT INTO ip_fused
(ip, sources, feature_score, behavior_score,
first_banned_at, last_activity_at, last_pushed_at,
push_count, fused_reason, updated_at)
VALUES (?,?,?,?,?,?,?,0,?,?)""",
(ip, data["sources"], data["feature_score"],
data["behavior_score"], data["first_banned_at"],
data["last_activity_at"], now if initial else None,
data["fused_reason"], now))
stale = [ip for ip in existing if ip not in wanted]
for ip in stale:
self.conn.execute("DELETE FROM ip_fused WHERE ip = ?", (ip,))
return len(wanted)
def due_pushes(self, active_hours=24, cooldown_hours=24):
now = time.time()
active_cut = time.strftime("%Y-%m-%dT%H:%M:%SZ",
time.gmtime(now - active_hours * 3600))
push_cut = time.strftime("%Y-%m-%dT%H:%M:%SZ",
time.gmtime(now - cooldown_hours * 3600))
with self.lock:
rows = self.conn.execute(
"""SELECT f.*, s.country_name AS s_cn, s.region AS s_region,
s.total AS s_total, s.asn AS s_asn,
s.asn_brand AS s_asn_brand, s.asn_type AS s_asn_type,
s.asn_icon AS s_asn_icon,
b.country_name AS b_cn, b.region AS b_region,
b.total AS b_total, b.asn AS b_asn,
b.asn_brand AS b_asn_brand, b.asn_type AS b_asn_type,
b.asn_icon AS b_asn_icon
FROM ip_fused f
LEFT JOIN ip_stats s ON s.ip = f.ip
LEFT JOIN behavior_profiles b ON b.ip = f.ip
WHERE f.last_activity_at >= ?
AND (f.last_pushed_at IS NULL OR f.last_pushed_at <= ?)
ORDER BY f.first_banned_at DESC LIMIT 50""",
(active_cut, push_cut)).fetchall()
items = []
for row in rows:
item = dict(row)
item["country_name"] = item.get("s_cn") or item.get("b_cn") or ""
item["region"] = item.get("s_region") or item.get("b_region") or ""
item["total"] = item.get("s_total") or item.get("b_total") or 0
item["asn"] = item.get("s_asn") or item.get("b_asn") or ""
item["asn_brand"] = (item.get("s_asn_brand")
or item.get("b_asn_brand") or "")
item["asn_type"] = (item.get("s_asn_type")
or item.get("b_asn_type") or "")
item["asn_icon"] = (item.get("s_asn_icon")
or item.get("b_asn_icon") or "")
items.append(item)
return items
def mark_pushed(self, ips, stamp):
with self.lock, self.conn:
for ip in ips:
self.conn.execute(
"UPDATE ip_fused SET last_pushed_at=?, push_count=push_count+1 "
"WHERE ip=?", (stamp, ip))
def stats(self):
with self.lock:
rows = self.conn.execute(
"SELECT tier, COUNT(*) AS n FROM ip_stats GROUP BY tier"
).fetchall()
core = self.conn.execute(
"SELECT COUNT(*) AS n FROM ip_stats WHERE ever_malicious = 1"
).fetchone()["n"]
permanent = self.conn.execute(
"SELECT COUNT(*) AS n FROM ip_stats WHERE ever_malicious = 1"
).fetchone()["n"]
events = self.conn.execute("SELECT COUNT(*) AS n FROM events").fetchone()["n"]
updated = self.conn.execute(
"SELECT MAX(updated_at) AS ts FROM ip_stats"
).fetchone()["ts"]
counts = {row["tier"]: row["n"] for row in rows}
with self.lock:
fused_total = self.conn.execute(
"SELECT COUNT(*) AS n FROM ip_fused").fetchone()["n"]
fused_both = self.conn.execute(
"SELECT COUNT(*) AS n FROM ip_fused WHERE sources='both'"
).fetchone()["n"]
fused_behavior = self.conn.execute(
"SELECT COUNT(*) AS n FROM ip_fused WHERE sources='behavior'"
).fetchone()["n"]
fused_new = self.conn.execute(
"SELECT COUNT(*) AS n FROM ip_fused WHERE first_banned_at >= ?",
(time.strftime("%Y-%m-%dT%H:%M:%SZ",
time.gmtime(time.time() - 86400)),)
).fetchone()["n"]
pushed = self.conn.execute(
"SELECT COUNT(*) AS n FROM ip_fused WHERE last_pushed_at IS NOT NULL"
).fetchone()["n"]
return {
"total": sum(counts.values()),
"core": core,
"permanent": permanent,
"by_tier": counts,
"events": events,
"updated_at": updated,
"fused_total": fused_total,
"fused_new_24h": fused_new,
"fused_both": fused_both,
"fused_behavior_only": fused_behavior,
"fused_pushed": pushed,
}
def update_first_seen(self, ip, timestamp):
with self.lock, self.conn:
cursor = self.conn.execute(
"UPDATE ip_stats SET first_seen = ? "
"WHERE ip = ? AND (first_seen IS NULL OR first_seen > ?)",
(timestamp, ip, timestamp),
)
return cursor.rowcount
def update_asn(self, ip, info):
with self.lock, self.conn:
self.conn.execute(
"UPDATE ip_stats SET asn=?, asn_org=?, asn_brand=?, asn_type=?, "
"asn_icon=? WHERE ip=?",
(info.get("asn"), info.get("org"), info.get("brand"),
info.get("type"), info.get("icon"), ip))
def update_geo(self, ip, geo):
with self.lock, self.conn:
self.conn.execute(
"UPDATE ip_stats SET country_code=?, country_name=?, region=?, "
"city=? WHERE ip=?", (geo.get("country_code"), geo.get("country_name"),
geo.get("region"), geo.get("city"), ip))
def update_times(self, ip, last_seen, last_malicious_at=None):
with self.lock, self.conn:
self.conn.execute(
"UPDATE ip_stats SET last_seen = ?, "
"last_malicious_at = COALESCE(?, last_malicious_at) WHERE ip = ?",
(last_seen, last_malicious_at, ip),
)
def get_evidence(self, ip):
with self.lock:
row = self.conn.execute(
"SELECT evidence FROM ip_stats WHERE ip = ?", (ip,)
).fetchone()
return row["evidence"] if row else None
def set_evidence(self, ip, payload, now):
with self.lock, self.conn:
self.conn.execute(
"UPDATE ip_stats SET evidence = ?, evidence_at = ? WHERE ip = ?",
(json.dumps(payload, ensure_ascii=False), now, ip),
)
def forget(self, older_than_seconds, now_epoch):
cutoff = now_epoch - older_than_seconds
with self.lock, self.conn:
self.conn.execute(
"DELETE FROM ip_stats WHERE last_seen < ? "
"AND COALESCE(ever_malicious, 0) = 0",
(_iso(datetime.fromtimestamp(cutoff, timezone.utc)),),
)
def close(self):
self.conn.close()
def _iso(moment):
return moment.strftime("%Y-%m-%dT%H:%M:%SZ")
def now_iso():
return _iso(datetime.now(timezone.utc))
# ------------------------------------------------------------------ detector
class Detector:
def __init__(self, client, store, fields, window=900, size=200,
min_requests=3, verify_limit=8, forget=7200, allowlist=None,
filter_str=None, min_observe_score=10.0, warm_limit=5, geo_db=None,
asn_db=None):
self.client = client
self.store = store
self.fields = fields
self.window = window
self.size = size
self.min_requests = min_requests
self.verify_limit = verify_limit
self.forget = forget
self.allowlist = allowlist
self.filter_str = filter_str
self.min_observe_score = float(min_observe_score)
self.warm_limit = int(warm_limit)
self.asn = None
if asn_db:
try:
self.asn = AsnDB(asn_db)
except (FileNotFoundError, sqlite3.Error):
self.asn = None
self.geo = None
if geo_db:
try:
from .geo import GeoIP
self.geo = GeoIP(geo_db)
except (FileNotFoundError, sqlite3.Error):
self.geo = None
self.rules = [
{"name": rule["name"], "title": rule["title"],
"window_cap": rule.get("window_cap"),
"query": rule["query"](fields)}
for rule in RULE_DEFS
]
self.last_run = None
self.last_success = None
self.last_error = None
self.runs = 0
def _terms(self, query, window_cap=None):
window = self.window
if window_cap:
window = min(window, window_cap)
data = self.client.terms(
field=self.fields["client_ip"], query=query, size=self.size,
range_seconds=window, filter_str=self.filter_str,
)
return {row["term"]: int(row["count"] or 0)
for row in normalize_terms(data)
if row["term"] not in (None, "")}
def _batch_totals(self, ips, chunk=40):
"""One terms query can return totals for many IPs at once."""
field = self.fields["client_ip"]
result = {}
for start in range(0, len(ips), chunk):
part = ips[start:start + chunk]
query = "%s:(%s)" % (field, " OR ".join('"%s"' % ip for ip in part))
try:
result.update(self._terms(query))
except GraylogError as exc:
self.last_error = "batch totals: %s" % exc
return result
def collect(self):
hits = {}
for rule in self.rules:
try:
hits[rule["name"]] = self._terms(
rule["query"], window_cap=rule.get("window_cap"))
except GraylogError as exc:
hits[rule["name"]] = {}
self.last_error = "%s: %s" % (rule["name"], exc)
# Low-volume scanners never make the "top requests" list, so the
# candidate pool must be the union of every rule hit plus the volume
# top. Totals for the missing ones are fetched in batches.
totals = self._terms("*")
candidates = set(totals)
for mapping in hits.values():
candidates.update(mapping)
missing = [ip for ip in candidates if ip not in totals]
if len(missing) > 1200: # keep the load bounded
missing = missing[:1200]
if missing:
totals.update(self._batch_totals(missing))
return totals, hits
def _verify(self, records):
"""Downgrade single-endpoint automation (broken token, not an attacker)."""
limit = self.verify_limit
for record in records[:limit]:
if record["tier"] not in (TIER_MALICIOUS, TIER_SUSPICIOUS):
continue
try:
data = self.client.terms(
field=self.fields["uri"],
query='%s:"%s"' % (self.fields["client_ip"], record["ip"]),
size=8, range_seconds=self.window, filter_str=self.filter_str,
)
except GraylogError as exc:
record["reasons"].append("核验失败: %s" % exc.message)
continue
rows = [(row["term"], row["count"]) for row in normalize_terms(data)
if row["term"] not in (None, "")]
total = sum(count for _, count in rows)
if not rows or not total:
continue
top_uri, top_count = rows[0]
ratio = top_count / float(total)
login_like = any(hint in str(top_uri).lower() for hint in AUTH_PATH_HINTS)
if ratio >= 0.9 and len(rows) <= 2 and not login_like:
record["tier"] = TIER_AUTOMATION
record["score"] = min(record["score"], 45.0)
record["rules"]["automation"] = {
"top_uri": top_uri, "ratio": round(ratio, 4)}
record["reasons"].append(
"流量集中在单一接口 %s（%.0f%%），疑似业务客户端" % (top_uri, ratio * 100))
def first_attack_at(self, ip, window=None):
"""Earliest message for this IP inside the window (probe-only first)."""
if getattr(self, "client", None) is None:
return None
window = int(window or self.window)
base = '%s:"%s"' % (self.fields["client_ip"], ip)
probe = _probe_query(self, None, window)
queries = []
if probe:
queries.append("(%s) AND %s" % (probe, base))
queries.append(base)
for query in queries:
try:
messages = self.client.search_views_messages(
query=query, range_seconds=window, limit=1,
filter_str=self.filter_str, sort_order="ASC")
except GraylogError:
continue
if messages and messages[0].get("timestamp"):
return messages[0]["timestamp"]
return None
def last_message_at(self, ip, window=None):
if getattr(self, "client", None) is None:
return None
window = int(window or self.window)
base = '%s:"%s"' % (self.fields["client_ip"], ip)
try:
messages = self.client.search_views_messages(
query=base, range_seconds=window, limit=1,
filter_str=self.filter_str, sort_order="DESC")
except GraylogError:
return None
return messages[0].get("timestamp") if messages else None
def last_attack_at(self, ip, window=None):
if getattr(self, "client", None) is None:
return None
window = int(window or self.window)
base = '%s:"%s"' % (self.fields["client_ip"], ip)
probe = _probe_query(self, None, window)
if probe:
try:
messages = self.client.search_views_messages(
query="(%s) AND %s" % (probe, base),
range_seconds=window, limit=1,
filter_str=self.filter_str, sort_order="DESC")
except GraylogError:
messages = []
if messages and messages[0].get("timestamp"):
return messages[0]["timestamp"]
return self.last_message_at(ip, window)
def backfill_asn(self, limit=5000):
if self.asn is None:
return 0
done = 0
for row in self.store.list(limit=limit, scope=None):
if row.get("asn"):
continue
info = self.asn.lookup(row["ip"])
if info:
self.store.update_asn(row["ip"], info)
done += 1
return done
def backfill_geo(self, limit=5000):
if self.geo is None:
return 0
done = 0
for row in self.store.list(limit=limit, scope=None):
if row.get("country_code"):
continue
geo = self.geo.lookup(row["ip"])
if geo:
self.store.update_geo(row["ip"], geo)
done += 1
return done
def backfill_times(self, limit=500):
"""Re-derive first-attack / last-attack / last-seen from the logs."""
if getattr(self, "client", None) is None:
return 0
rows = self.store.list(limit=20000, scope=None)
done = 0
for row in rows:
if limit and done >= limit:
break
window = max(row.get("window") or self.window, BACKFILL_WINDOW)
first = self.first_attack_at(row["ip"], window)
last = self.last_message_at(row["ip"], window)
attack = (self.last_attack_at(row["ip"], window)
if row.get("ever_malicious") else None)
if first:
self.store.update_first_seen(row["ip"], first)
if last:
self.store.update_times(row["ip"], last, attack)
done += 1
return done
# backwards-compatible alias
backfill_first_seen = backfill_times
def warm_snapshots(self, limit=10):
"""Pre-generate compact evidence for listed malicious IPs."""
if getattr(self, "client", None) is None:
return 0
rows = self.store.list(limit=10000, tier=TIER_MALICIOUS)
done = 0
for row in rows:
if limit and done >= limit:
break
if row.get("evidence_at"):
continue
try:
evidence = collect_evidence(self, row["ip"], row.get("rules"),
row.get("window") or self.window)
except Exception:
continue
if evidence.get("available"):
self.store.set_evidence(row["ip"], _compact_evidence(evidence),
now_iso())
done += 1
return done
def run_once(self):
started = time.time()
totals, hits = self.collect()
records = []
for ip, total in totals.items():
if total < self.min_requests or not is_public(ip):
continue
if self.allowlist and self.allowlist(ip):
continue
rule_hits = {name: mapping.get(ip, 0) for name, mapping in hits.items()}
tier, score, reasons = classify(
total, rule_hits, window_scale=max(1.0, self.window / 900.0))
risk = None
if tier == TIER_OBSERVE and (not reasons or score < self.min_observe_score):
continue
records.append({
"ip": ip,
"tier": tier,
"score": score,
"total": total,
"rules": {name: count for name, count in rule_hits.items() if count},
"reasons": reasons,
"risk": risk,
"window": self.window,
})
records.sort(key=lambda row: (-row["score"], -row["total"]))
for record in records:
if self.geo is not None:
record["geo"] = self.geo.lookup(record["ip"])
if self.asn is not None:
record["asn"] = self.asn.lookup(record["ip"])
existing = self.store.get(record["ip"])
if existing is None:
window = max(self.window, BACKFILL_WINDOW)
first = self.first_attack_at(record["ip"], window)
last = self.last_message_at(record["ip"], window)
if first:
record["first_seen"] = first
if last:
record["last_seen"] = last
if record["tier"] == TIER_MALICIOUS:
attack = self.last_attack_at(record["ip"], window)
if attack:
record["last_malicious_at"] = attack
elif last:
record["last_malicious_at"] = last
continue
changed = (existing.get("total") != record["total"]
or (existing.get("rules") or {}) != record["rules"])
if not changed:
# Same evidence as last round: do not move any timestamp.
continue
last = self.last_message_at(record["ip"], self.window)
if last:
record["last_seen"] = last
if record["tier"] == TIER_MALICIOUS:
attack = self.last_attack_at(record["ip"], self.window)
if attack:
record["last_malicious_at"] = attack
elif last:
record["last_malicious_at"] = last
self._verify(records)
stamp = now_iso()
# Notifications are sent only by the fused pipeline (notify_fused) so a
# new IP produces exactly one message, never one per detection source.
for record in records:
self.store.upsert(record, stamp)
self.store.forget(self.forget, time.time())
try:
self.warm_snapshots(int(getattr(self, "warm_limit", 0) or 0))
except Exception:
pass
self.last_run = stamp
self.last_success = time.time()
self.runs += 1
return {
"window": self.window,
"request_seconds": round(time.time() - started, 2),
"candidates": len(records),
"malicious": sum(1 for row in records if row["tier"] == TIER_MALICIOUS),
"suspicious": sum(1 for row in records if row["tier"] == TIER_SUSPICIOUS),
"automation": sum(1 for row in records if row["tier"] == TIER_AUTOMATION),
}
def _csv_text(items):
buffer = io.StringIO()
writer = csv.writer(buffer)
writer.writerow(["ip", "tier", "score", "total", "first_seen",
"last_malicious_at", "risk", "reasons",
"country_code", "country_name", "region", "city",
"asn", "asn_org", "asn_brand", "asn_type"])
for item in items:
writer.writerow([
item.get("ip", ""),
item.get("tier", ""),
item.get("score", ""),
item.get("total", ""),
item.get("first_seen") or "",
item.get("last_malicious_at") or "",
item.get("risk") or "",
"; ".join(item.get("reasons") or []),
item.get("country_code") or "",
item.get("country_name") or "",
item.get("region") or "",
item.get("city") or "",
item.get("asn") or "",
item.get("asn_org") or "",
item.get("asn_brand") or "",
item.get("asn_type") or "",
])
# BOM so Excel opens the UTF-8 file without mojibake
return "\ufeff" + buffer.getvalue()
def export_blocklist(store, path):
"""Write every IP ever convicted as malicious."""
rows = store.list(limit=20000, scope="core", sort="first_seen", order="asc")
chosen = [row["ip"] for row in rows]
directory = os.path.dirname(os.path.abspath(path))
if directory:
os.makedirs(directory, exist_ok=True)
with open(path, "w", encoding="utf-8") as handle:
handle.write("\n".join(chosen) + ("\n" if chosen else ""))
return chosen
# --------------------------------------------------------------------- web
EVIDENCE_CACHE = {}
EVIDENCE_TTL = 600
SAMPLE_KEYS = ("timestamp", "client_ip", "client_port", "server_ip",
"server_port", "host", "bigip", "vs", "method", "uri", "query",
"status", "ua", "referer", "req_len", "rt",
"client_ip_country_code", "client_ip_city_name",
"client_ip_geolocation", "event_type", "source",
"gl2_remote_ip", "message")
HEADER_FIELDS = {"method", "status", "uri", "host", "client_ip", "server_ip",
"client_port", "server_port", "vs", "bigip", "event_type",
"source", "gl2_remote_ip"}
def _clean_text(value, limit=280):
"""Strip control chars / replacement chars that source logs sometimes carry."""
out = []
for char in str(value):
code = ord(char)
if char in "\n\r\t":
out.append(" ")
elif code < 32 or code == 127 or char == "\ufffd":
out.append("·")
else:
out.append(char)
text = re.sub(r"·{2,}", "·", "".join(out))
return _clip(text, limit)
def _scrub_binary(text):
"""Header-ish fields must be ASCII; a high non-ASCII ratio means garbage."""
if not text:
return text
non_ascii = sum(1 for char in text if ord(char) > 127)
if non_ascii / max(1, len(text)) > 0.2:
return "[非 UTF-8 数据]"
return text
def _clip(value, limit=280):
text = str(value)
return text[:limit] + ("…" if len(text) > limit else "")
def _sample(message):
payload = {}
for key in SAMPLE_KEYS:
value = message.get(key)
if value in (None, "", "N/A"):
continue
text = _clean_text(value, 4000 if key == "message" else 600)
if key in HEADER_FIELDS:
text = _scrub_binary(text)
payload[key] = text
return payload
def _compact_evidence(evidence):
"""Small summary persisted in SQLite: survives Graylog index rotation."""
return {
"compact": True,
"window": evidence.get("window"),
"captured_at": now_iso(),
"uris": (evidence.get("uris") or [])[:10],
"statuses": (evidence.get("statuses") or [])[:6],
"methods": (evidence.get("methods") or [])[:6],
"hosts": (evidence.get("hosts") or [])[:6],
"server_ips": (evidence.get("server_ips") or [])[:6],
"vips": (evidence.get("vips") or [])[:6],
"uas": (evidence.get("uas") or [])[:3],
"peak": evidence.get("peak"),
"interval": evidence.get("interval"),
"timeline": (evidence.get("timeline") or [])[-72:],
"sample_count": len(evidence.get("samples") or []),
}
def _probe_query(detector, item_rules=None, window=None):
"""Sample only the probe families that actually fired for this IP.
Combining all seven families (plus a regex) over a 24h window made the
Elasticsearch node drop the connection, so keep the sample query narrow and
skip rules whose window cap is smaller than the evidence window.
"""
names = {rule["name"] for rule in detector.rules if rule["name"] in PROBE_RULES}
if item_rules:
fired = sorted(((name, item_rules.get(name) or 0) for name in names
if item_rules.get(name)), key=lambda pair: -pair[1])
if fired:
# Only the strongest two families: keeps the sample query cheap for
# IPs that fired six or seven rules at once.
names = {name for name, _ in fired[:2]}
queries = []
window = window or detector.window
for rule in detector.rules:
if rule["name"] not in names or not rule.get("query"):
continue
cap = rule.get("window_cap")
if cap and window > cap:
continue
queries.append(rule["query"])
if not queries:
return None
return " OR ".join("(%s)" % query for query in queries)
def collect_evidence(detector, ip, item_rules=None, window=None):
"""Fetch a handful of typical requests for one IP (never the whole set).
All Graylog calls run concurrently and the result is persisted by the
caller, so opening the same IP a second time is instant.
"""
if detector is None or getattr(detector, "client", None) is None:
return {"available": False, "reason": "未连接 Graylog，仅展示入库统计"}
window = int(window or detector.window)
cache_key = (ip, window)
cached = EVIDENCE_CACHE.get(cache_key)
if cached and time.time() - cached[0] < EVIDENCE_TTL:
return cached[1]
from concurrent.futures import ThreadPoolExecutor
fields = detector.fields
client = detector.client
query = '%s:"%s"' % (fields["client_ip"], ip)
result = {"available": True, "window": window, "samples": [],
"uris": [], "uas": [], "statuses": [], "methods": [], "hosts": [],
"timeline": [],
"rule_titles": {rule["name"]: rule.get("title") or rule["name"]
for rule in detector.rules}}
def terms_call(field, size):
data = client.terms(field=field, query=query, size=size,
range_seconds=window, filter_str=detector.filter_str)
rows = [{"term": _scrub_binary(_clean_text(row["term"], 240)),
"count": row["count"]}
for row in normalize_terms(data) if row["term"] not in (None, "")]
if field == fields.get("status"):
# "200 OK" and "200" are the same code; merge them for the chart.
merged = {}
for row in rows:
match = re.match(r"\s*(\d{3})", str(row["term"]))
key = match.group(1) if match else str(row["term"])
merged[key] = merged.get(key, 0) + row["count"]
rows = [{"term": key, "count": count}
for key, count in sorted(merged.items(),
key=lambda pair: (-pair[1], pair[0]))]
return rows
def samples_call():
"""Latest 3 messages per fired rule, so each attack type keeps evidence."""
fired = []
for rule in detector.rules:
name = rule["name"]
if not item_rules or not int(item_rules.get(name) or 0):
continue
if not rule.get("query"):
continue
cap = rule.get("window_cap")
rule_window = min(window, int(cap)) if cap else window
fired.append((name, rule["query"], rule_window))
if not fired:
try:
messages = client.search_views_messages(
query=query, range_seconds=window, limit=5,
filter_str=detector.filter_str)
except GraylogError:
messages = []
return [(None, messages)]
def fetch(job):
rule_name, rule_query, rule_window = job
try:
messages = client.search_views_messages(
query="(%s) AND %s" % (rule_query, query),
range_seconds=rule_window, limit=3,
filter_str=detector.filter_str)
except GraylogError:
messages = []
return rule_name, messages
results = []
with ThreadPoolExecutor(max_workers=6) as pool:
for item in pool.map(fetch, fired):
results.append(item)
return results
def timeline_call():
interval = "5m" if window <= 3600 else "1h"
history = client.histogram(query=query, range_seconds=window,
interval=interval, filter_str=detector.filter_str)
return interval, normalize_histogram(history)
dimensions = (("uris", "uri", 10), ("uas", "ua", 5), ("statuses", "status", 6),
("methods", "method", 6), ("hosts", "host", 6),
("server_ips", "server_ip", 6), ("vips", "bigip", 6))
with ThreadPoolExecutor(max_workers=8) as pool:
futures = {}
for name, field_key, size in dimensions:
field = fields.get(field_key)
if field:
futures[name] = pool.submit(terms_call, field, size)
futures["samples"] = pool.submit(samples_call)
futures["timeline"] = pool.submit(timeline_call)
for name, future in futures.items():
try:
value = future.result()
except GraylogError as exc:
result[name + "_error"] = str(exc)
continue
if name == "samples":
merged = []
seen = {}
for rule_name, messages in value:
for message in messages:
# second precision: the same request is often logged
# twice (two client ports) within the same second.
fingerprint = ((message.get("timestamp") or "")[:19],
message.get("uri"), message.get("status"),
message.get("method"))
if fingerprint in seen:
entry = seen[fingerprint]
if rule_name and rule_name not in entry["rules"]:
entry["rules"].append(rule_name)
continue
payload = _sample(message)
payload["rules"] = [rule_name] if rule_name else []
seen[fingerprint] = payload
merged.append(payload)
merged.sort(key=lambda entry: entry.get("timestamp") or "",
reverse=True)
result["samples"] = merged[:24]
elif name == "timeline":
interval, timeline = value
result["timeline"] = timeline
result["interval"] = interval
if timeline:
peak = max(timeline, key=lambda row: row["count"])
result["peak"] = {"timestamp": peak["timestamp"],
"count": peak["count"]}
result["active_buckets"] = sum(1 for row in timeline if row["count"])
else:
result[name] = value
result["distinct_uris_shown"] = len(result.get("uris") or [])
if not result.get("samples") and not result.get("samples_error"):
result["samples_error"] = "该窗口内没有可抽取的样本"
if not result.get("uris") and not result.get("samples"):
result["available"] = False
if not result.get("error") and result.get("uris_error"):
result["error"] = result["uris_error"]
EVIDENCE_CACHE[cache_key] = (time.time(), result)
return result
DASHBOARD = r"""<!doctype html>
<html lang="zh"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>恶意 IP 清单</title>
<style>
:root{
--bg:#F6F4EE;
--surface:#FFFFFF;
--surface-2:#FBFAF6;
--ink:#242119;
--ink-2:#4B463B;
--muted:#8B8577;
--line:#E8E3D9;
--line-strong:#D9D2C4;
--accent:#C15F3C;
--accent-deep:#A94E30;
--accent-soft:#F4E6DE;
--green:#2E6B4F;
--green-soft:#E3EEE7;
--amber:#8F6418;
--amber-soft:#F6EBD2;
--red:#A63A28;
--red-soft:#F6E1DB;
--blue:#34557F;
--blue-soft:#E6ECF5;
--radius:14px;
--shadow:0 1px 2px rgba(36,33,25,.05), 0 10px 30px rgba(36,33,25,.045);
}
*{box-sizing:border-box}
html,body{margin:0}
body{
background:var(--bg);
color:var(--ink);
font:14px/1.62 -apple-system,BlinkMacSystemFont,"PingFang SC","Hiragino Sans GB",
"Microsoft YaHei",Segoe UI,sans-serif;
min-height:100vh;
-webkit-font-smoothing:antialiased;
}
header.top{
position:sticky;top:0;z-index:30;display:flex;justify-content:space-between;
align-items:center;gap:16px;padding:18px 32px;
border-bottom:1px solid var(--line);
background:rgba(246,244,238,.88);backdrop-filter:blur(12px);
}
.brand{display:flex;align-items:center;gap:14px}
.logo{width:34px;height:34px;border-radius:11px;display:grid;place-items:center;
background:linear-gradient(145deg,#D97757,#B4532F);color:#fff;font-weight:700;
font-size:13px;letter-spacing:.5px;box-shadow:0 2px 8px rgba(193,95,60,.28)}
h1{font-family:"Iowan Old Style","Palatino Linotype",Palatino,Georgia,"Songti SC",serif;
font-size:19px;font-weight:600;margin:0;letter-spacing:.2px;color:var(--ink)}
.sub{color:var(--muted);font-size:12px;margin-top:1px}
.head-right{display:flex;align-items:center;gap:10px}
.status-dot{width:10px;height:10px;border-radius:50%;background:#C9C2B4;
transition:background .2s,box-shadow .2s;flex:none}
.status-dot.ok{background:#3E8E63;box-shadow:0 0 0 3px rgba(62,142,99,.18)}
.status-dot.error{background:#C0503A;box-shadow:0 0 0 3px rgba(192,80,58,.2)}
.status-dot.stale{background:#C08A2E;box-shadow:0 0 0 3px rgba(192,138,46,.2)}
.meta{color:var(--muted);font-size:12px;text-align:right;line-height:1.9}
.meta b{color:var(--ink-2);font-weight:600}
.meta a{color:var(--accent);text-decoration:none;border-bottom:1px solid transparent}
.meta a:hover{border-bottom-color:var(--accent)}
main{padding:26px 32px 80px;max-width:1500px;margin:0 auto}
.stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(168px,1fr));
gap:14px;margin-bottom:22px}
.stat{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);
padding:16px 18px;box-shadow:var(--shadow)}
.stat .n{font-family:"Iowan Old Style",Palatino,Georgia,serif;font-size:30px;
font-weight:600;line-height:1.15;letter-spacing:.4px;color:var(--ink)}
.stat .k{color:var(--muted);font-size:12px;margin-top:3px}
.stat.hot .n{color:var(--accent)}
.stat.warm .n{color:var(--amber)}
.stat.cool .n{color:var(--blue)}
.toolbar{display:flex;align-items:center;gap:10px;margin:2px 0 14px;flex-wrap:wrap}
input[type=search]{
flex:1;min-width:260px;background:var(--surface);border:1px solid var(--line);
color:var(--ink);border-radius:11px;padding:11px 15px;outline:none;font-size:13px;
box-shadow:0 1px 2px rgba(36,33,25,.03);transition:.15s;
}
input[type=search]::placeholder{color:#AAA392}
input[type=search]:focus{border-color:#D3B39F;box-shadow:0 0 0 3px rgba(193,95,60,.1)}
.chips{display:flex;gap:2px;background:#EFEAE0;border:1px solid var(--line);
padding:3px;border-radius:11px}
.chip{background:transparent;border:0;color:var(--ink-2);padding:7px 14px;
border-radius:8px;cursor:pointer;font-size:13px;transition:.15s;white-space:nowrap}
.chip:hover{color:var(--ink)}
.chip.active{background:var(--surface);color:var(--accent-deep);font-weight:600;
box-shadow:0 1px 3px rgba(36,33,25,.08)}
.sortgroup{display:flex;gap:8px}
.select{background:var(--surface);border:1px solid var(--line);color:var(--ink-2);
border-radius:11px;padding:10px 13px;font-size:13px;outline:none;cursor:pointer;
box-shadow:0 1px 2px rgba(36,33,25,.03);transition:.15s;appearance:none;
padding-right:32px;
background-image:url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='10' height='6' viewBox='0 0 10 6'%3E%3Cpath d='M1 1l4 4 4-4' stroke='%238B8577' stroke-width='1.6' fill='none' stroke-linecap='round'/%3E%3C/svg%3E");
background-repeat:no-repeat;background-position:right 12px center}
.select:focus{border-color:#D3B39F;box-shadow:0 0 0 3px rgba(193,95,60,.1)}
.switch{display:flex;align-items:center;gap:7px;color:var(--muted);font-size:12px;
cursor:pointer;user-select:none;white-space:nowrap}
.switch input{accent-color:var(--accent);width:15px;height:15px}
.actions{display:flex;gap:8px;margin-left:auto}
.btn{background:var(--surface);border:1px solid var(--line);color:var(--ink-2);
border-radius:11px;padding:10px 15px;font-size:13px;cursor:pointer;transition:.15s;
box-shadow:0 1px 2px rgba(36,33,25,.03);white-space:nowrap;font-weight:500}
.btn:hover{border-color:#D3B39F;color:var(--accent-deep);background:#FBF7F2}
.btn:active{transform:translateY(1px)}
.btn.primary{background:var(--accent);border-color:var(--accent);color:#fff}
.btn.primary:hover{background:var(--accent-deep);color:#fff;border-color:var(--accent-deep)}
.menu-wrap{position:relative}
.caret{font-size:10px;opacity:.65;margin-left:5px}
.menu{position:absolute;right:0;top:calc(100% + 7px);min-width:200px;
background:var(--surface);border:1px solid var(--line-strong);border-radius:12px;
box-shadow:0 18px 44px rgba(36,33,25,.16);padding:6px;display:none;z-index:60}
.menu.open{display:block}
.menu button{display:block;width:100%;text-align:left;background:transparent;border:0;
color:var(--ink-2);padding:9px 12px;border-radius:8px;font-size:13px;cursor:pointer;
transition:.12s;white-space:nowrap}
.menu button:hover{background:var(--accent-soft);color:var(--accent-deep)}
.toast{position:fixed;left:50%;bottom:30px;transform:translateX(-50%) translateY(12px);
background:#242119;color:#F6F4EE;padding:11px 20px;border-radius:12px;font-size:13px;
opacity:0;pointer-events:none;transition:.22s;z-index:300;
box-shadow:0 12px 32px rgba(36,33,25,.28)}
.toast.show{opacity:1;transform:translateX(-50%) translateY(0)}
.tablewrap{overflow:auto;border:1px solid var(--line);border-radius:var(--radius);
background:var(--surface);box-shadow:var(--shadow)}
table{width:100%;border-collapse:collapse}
thead th{position:sticky;top:0;background:var(--surface-2);color:var(--muted);
font-weight:600;font-size:11.5px;letter-spacing:.4px;text-transform:uppercase;
text-align:left;padding:12px 16px;border-bottom:1px solid var(--line);
white-space:nowrap;z-index:5}
tbody td{padding:13px 16px;border-bottom:1px solid #F0ECE3;vertical-align:middle;
font-size:13px}
tbody tr{cursor:pointer;transition:background .12s}
tbody tr:hover{background:#FBF9F4}
tbody tr:last-child td{border-bottom:0}
.ip{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:var(--ink);
font-weight:500;white-space:nowrap}
.copyable{cursor:pointer;transition:.12s}
.ip.copyable:hover{color:var(--accent-deep);text-decoration:underline dotted}
h2.copyable:hover{color:var(--accent-deep)}
.num{font-variant-numeric:tabular-nums;white-space:nowrap;color:var(--ink-2)}
.muted{color:var(--muted);font-size:12px}
tbody td.muted{white-space:nowrap}
.geo{white-space:nowrap;font-size:12.5px;color:var(--ink-2);line-height:1.55}
.geo-l1{font-size:12.5px;color:var(--ink-2)}
.geo-l2{font-size:11.5px;color:var(--muted)}
.badge{display:inline-block;padding:3px 10px;border-radius:20px;font-size:11.5px;
line-height:1.55;white-space:nowrap;vertical-align:middle;font-weight:500}
.badge-danger{background:var(--red-soft);color:var(--red)}
.badge-warn{background:var(--amber-soft);color:var(--amber)}
.badge-info{background:var(--blue-soft);color:var(--blue)}
.badge-muted{background:#EFEBE2;color:var(--muted)}
.badge-ok{background:var(--green-soft);color:var(--green)}
.badge-risk{background:var(--amber-soft);color:var(--amber);
border:1px dashed #D8B77E}
.scorecell{display:flex;align-items:center;gap:9px}
.scorecell span{font-variant-numeric:tabular-nums;color:var(--ink-2);
min-width:34px;font-weight:600}
.bar{height:6px;width:66px;border-radius:4px;background:#EDE8DE;overflow:hidden;flex:none}
.bar i{display:block;height:100%;border-radius:4px;
background:linear-gradient(90deg,#D98A6B,#B4532F)}
.reason{color:var(--muted);font-size:12px;max-width:520px}
.empty{padding:64px 20px;text-align:center;color:var(--muted)}
a{color:var(--accent);text-decoration:none}
a:hover{text-decoration:underline}
.detail-head{display:flex;justify-content:space-between;align-items:flex-start;
gap:20px;flex-wrap:wrap;margin-bottom:4px}
.back{display:inline-flex;align-items:center;gap:6px;color:var(--muted);
font-size:13px;margin-bottom:18px}
.back:hover{color:var(--accent-deep);text-decoration:none}
.detail-title{display:flex;align-items:center;gap:12px;flex-wrap:wrap}
.geo-line{display:flex;gap:6px;flex-wrap:wrap;margin-top:8px}
.detail-title h2{margin:0;font-size:26px;font-weight:600;letter-spacing:.2px;
font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:var(--ink)}
.ring{--pct:0;width:82px;height:82px;border-radius:50%;display:grid;
place-items:center;flex:none;position:relative;
background:conic-gradient(var(--accent) calc(var(--pct)*1%),#EDE8DE 0)}
.ring::after{content:"";position:absolute;inset:8px;border-radius:50%;
background:var(--surface);box-shadow:inset 0 1px 2px rgba(36,33,25,.05)}
.ring b{position:relative;z-index:1;font-size:20px;font-variant-numeric:tabular-nums;
font-family:"Iowan Old Style",Palatino,Georgia,serif;color:var(--ink)}
.kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(172px,1fr));
gap:14px;margin:20px 0 18px}
.kpi{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);
padding:15px 18px;box-shadow:var(--shadow)}
.kpi .n{font-family:"Iowan Old Style",Palatino,Georgia,serif;font-size:23px;
font-weight:600;font-variant-numeric:tabular-nums;line-height:1.2}
.kpi .k{color:var(--muted);font-size:12px;margin-top:3px}
.panel{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);
padding:18px 20px;margin-top:16px;box-shadow:var(--shadow)}
.panel h3{margin:0 0 14px;font-size:14px;font-weight:600;color:var(--ink);
letter-spacing:.2px;display:flex;align-items:center;gap:8px}
.panel h3::before{content:"";width:3px;height:14px;border-radius:2px;
background:var(--accent);display:inline-block}
.reasons{display:grid;gap:9px;margin:0;padding:0;list-style:none}
.reasons li{display:flex;gap:10px;align-items:flex-start;font-size:13px;color:var(--ink-2)}
.reasons li::before{content:"";width:6px;height:6px;border-radius:50%;
background:var(--accent);margin-top:8px;flex:none}
.rows{display:grid;gap:9px}
.row{display:grid;grid-template-columns:170px 1fr 66px;gap:12px;align-items:center;
font-size:12px}
.row .t{color:var(--ink-2);overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.row .b{height:8px;background:#EFEAE0;border-radius:5px;overflow:hidden}
.row .b i{display:block;height:100%;border-radius:5px;
background:linear-gradient(90deg,#E3A183,#C15F3C)}
.row .v{text-align:right;color:var(--ink-2);font-variant-numeric:tabular-nums;
font-weight:500}
.tl{display:flex;align-items:flex-end;justify-content:center;gap:4px;
height:132px;padding-top:8px}
.tl div{flex:1 1 0;max-width:64px;min-height:2px;border-radius:4px 4px 0 0;
background:linear-gradient(180deg,#E3A183,#C15F3C);transition:.15s}
.tl div:hover{background:#D98A6B}
.tl-axis{display:flex;justify-content:space-between;color:var(--muted);
font-size:11px;margin-top:7px}
.s-card{background:var(--surface-2);border:1px solid var(--line);border-radius:12px;
padding:14px 16px;margin-bottom:10px}
.s-head{display:flex;align-items:center;gap:8px;color:var(--muted);font-size:12px}
.s-head .badge{font-size:11px}
.s-uri{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:13px;
color:var(--ink);margin:8px 0 10px;word-break:break-all;line-height:1.55}
.s-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));
gap:7px 20px;font-size:12px;color:var(--ink-2)}
.s-grid b,.s-ua b{color:var(--muted);font-weight:500;margin-right:7px}
.s-ua{font-size:12px;color:var(--ink-2);margin-top:10px;word-break:break-all;line-height:1.5}
.s-card details{margin-top:10px}
.s-card summary{cursor:pointer;color:var(--muted);font-size:12px}
.s-card pre{white-space:pre-wrap;word-break:break-all;background:#F4F1E9;
border:1px solid var(--line);border-radius:9px;padding:10px 12px;font-size:11.5px;
color:#5B5648;margin:8px 0 0;line-height:1.7}
.notes{margin-top:13px;display:grid;gap:7px}
.notes div{font-size:12.5px;color:var(--ink-2);background:var(--surface-2);
border:1px solid var(--line);border-radius:9px;padding:9px 12px;line-height:1.55}
.hint{color:var(--muted);font-size:12px;margin-top:11px;line-height:1.7}
.advice{display:flex;gap:12px;align-items:flex-start;padding:15px 18px;
border-radius:12px;border:1px solid #EBC9BD;background:var(--red-soft);
color:#7E3020;font-size:13px;line-height:1.6}
.advice.warn{border-color:#E7D2A9;background:var(--amber-soft);color:#6E4E14}
.expandable{cursor:pointer;border-bottom:1px dashed #C9BFA9;transition:.12s}
.expandable:hover{color:var(--accent-deep);border-bottom-color:#C98C72}
.linkbtn{background:var(--surface);border:1px solid var(--line);color:var(--accent-deep);
border-radius:9px;padding:6px 12px;font-size:12px;cursor:pointer;margin-top:10px;
transition:.15s;box-shadow:0 1px 2px rgba(36,33,25,.04)}
.linkbtn:hover{background:var(--accent-soft);border-color:#DCB7A5}
.modal{position:fixed;inset:0;background:rgba(36,33,25,.34);display:none;z-index:100;
align-items:center;justify-content:center;padding:26px;backdrop-filter:blur(3px)}
.modal.open{display:flex}
.modal-box{width:min(980px,100%);max-height:82vh;display:flex;flex-direction:column;
background:var(--surface);border:1px solid var(--line-strong);border-radius:16px;
box-shadow:0 30px 80px rgba(36,33,25,.22)}
.modal-head{display:flex;justify-content:space-between;align-items:center;
padding:15px 20px;border-bottom:1px solid var(--line)}
.modal-head h4{margin:0;font-size:14px;color:var(--ink);font-weight:600}
.modal-close{background:transparent;border:0;color:var(--muted);font-size:22px;
cursor:pointer;line-height:1;padding:0 4px}
.modal-close:hover{color:var(--ink)}
.modal-body{padding:16px 20px;overflow:auto}
.modal-body pre{margin:0;white-space:pre-wrap;word-break:break-all;font-size:12.5px;
line-height:1.8;color:var(--ink-2);font-family:ui-monospace,Menlo,monospace}
.skeleton{padding:48px;text-align:center;color:var(--muted)}
@media (max-width:860px){
main{padding:18px 16px}
header.top{padding:14px 18px}
.reason{display:none}
.stats{grid-template-columns:repeat(2,1fr)}
.sortgroup{width:100%}
.select{flex:1}
}
@media (max-width:560px){
.chips{width:100%;justify-content:space-between}
.chip{flex:1;text-align:center}
.stats{grid-template-columns:1fr}
}
</style></head><body>
<header class="top">
<div class="brand">
<div class="logo">IP</div>
<div><h1>恶意 IP 清单</h1><div class="sub">特征 + 行为融合检测 · Graylog · F5 访问日志</div></div>
</div>
<div class="head-right">
<span class="status-dot" id="statusDot" title="Graylog 连接状态"></span>
<div class="meta" id="meta">加载中…</div>
</div>
</header>
<main>
<section id="listView">
<div class="stats" id="stats"></div>
<div class="toolbar">
<input type="search" id="search" placeholder="搜索 IP 或判定原因，例如 42.92 或 actuator">
<label class="switch"><input type="checkbox" id="auto" checked> 自动刷新</label>
<div class="actions">
<div class="menu-wrap">
<button class="btn" id="copyBtn" aria-haspopup="true">复制 IP <span class="caret">▾</span></button>
<div class="menu" id="copyMenu">
<button data-copy="comma">逗号分隔</button>
<button data-copy="newline">换行分隔</button>
</div>
</div>
<div class="menu-wrap">
<button class="btn primary" id="exportBtn" aria-haspopup="true">导出 <span class="caret">▾</span></button>
<div class="menu" id="exportMenu">
<button data-export="csv">CSV（完整字段）</button>
<button data-export="txt">仅 IP 列表（换行）</button>
</div>
</div>
</div>
</div>
<div class="tablewrap">
<table>
<thead><tr>
<th>IP 地址</th><th>位置 / 归属</th><th>来源</th><th>评分</th><th>请求数</th><th>频率</th>
<th>发现时间</th><th>最近活动</th><th>判定理由</th><th></th>
</tr></thead>
<tbody id="rows"></tbody>
</table>
<div class="empty" id="empty" hidden>没有匹配的记录</div>
</div>
<div class="hint" id="listHint"></div>
</section>
<section id="detailView" hidden></section>
</main>
<div class="toast" id="toast"></div>
<div class="modal" id="modal">
<div class="modal-box">
<div class="modal-head"><h4 id="modalTitle">详情</h4>
<button class="modal-close" id="modalClose" title="关闭">×</button></div>
<div class="modal-body"><pre id="modalBody"></pre></div>
</div>
</div>
<script>
window.addEventListener('error', function (event) {
var meta = document.getElementById('meta');
if (meta) meta.textContent = '前端脚本错误: ' + (event.message || 'unknown');
});
window.addEventListener('unhandledrejection', function (event) {
var meta = document.getElementById('meta');
if (meta) meta.textContent = '请求失败: ' + ((event.reason && event.reason.message) || event.reason || 'unknown');
});
const BASE = "__BASE__";
const $ = (sel, root = document) => root.querySelector(sel);
const flagOf = code => {
if (!code || code.length !== 2 || code === 'ZZ') return '';
return String.fromCodePoint(...[...code.toUpperCase()]
.map(ch => 0x1F1E6 + ch.charCodeAt(0) - 65));
};
const esc = v => String(v == null ? '' : v).replace(/[&<>"']/g, c => (
{'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
const fmt = n => Number(n || 0).toLocaleString('en-US');
const truncate = (text, limit) => {
const value = String(text == null ? '' : text);
return value.length > limit ? value.slice(0, limit - 1) + '…' : value;
};
const TIER = {
malicious: ['恶意', 'badge-danger'],
suspicious: ['可疑', 'badge-warn'],
automation: ['自动化', 'badge-info'],
observe: ['观察', 'badge-muted']
};
const SOURCE = {
feature: ['特征', 'badge-danger'],
behavior: ['行为', 'badge-info'],
both: ['双来源', 'badge-warn'],
};
const TYPE_LABEL = {
cloud: '云服务商', idc: 'IDC/机房', mobile: '移动网络',
broadband: '宽带', education: '教育网', enterprise: '企业/政府',
unknown: '未知',
};
const SHARE_LABEL = {
mobile: '移动网络', broadband: '宽带 ISP', education: '教育网',
hosting: '云主机/机房', unknown: '未知网络',
};
const RULE_TITLES = {
probe_java: 'Spring/中间件探测', probe_config: '配置/凭证探测', probe_cms: 'CMS 探测',
probe_iot: 'IoT/路由器探测', probe_devops: 'DevOps/监控探测', probe_webshell: 'WebShell/上传探测',
exploit_payload: '注入/穿越载荷', login_attack: '登录失败尝试', login_burst: '登录接口高频',
bad_ua: '扫描器 UA', bot_ua: '爬虫 UA', not_found: '404 探测'
};
const state = { q: '', items: [], timer: null, inDetail: false, detailSamples: [] };
const modal = $('#modal');
function openModal(title, text) {
if (!modal) return;
$('#modalTitle').textContent = title;
$('#modalBody').textContent = text || '(空)';
modal.classList.add('open');
}
function closeModal() { if (modal) modal.classList.remove('open'); }
if (modal) {
$('#modalClose').addEventListener('click', closeModal);
modal.addEventListener('click', e => { if (e.target === modal) closeModal(); });
}
document.addEventListener('keydown', e => { if (e.key === 'Escape') closeModal(); });
document.addEventListener('click', e => {
const el = e.target.closest('[data-mfield]');
if (!el) return;
const sample = (state.detailSamples || [])[Number(el.dataset.idx)] || {};
const field = el.dataset.mfield;
openModal(field === 'message' ? '原始日志行' : field.toUpperCase(), sample[field] || '');
});
function localTime(iso) {
if (!iso) return '-';
const d = new Date(iso);
if (isNaN(d)) return iso;
const p = n => String(n).padStart(2, '0');
return `${p(d.getMonth()+1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}`;
}
function perMinute(total, window) {
if (!window) return '-';
return (Number(total) / window * 60).toFixed(1);
}
async function jget(url) {
const r = await fetch(url, { cache: 'no-store' });
if (!r.ok) throw new Error('HTTP ' + r.status);
return r.json();
}
async function loadStatus() {
const dot = $('#statusDot');
try {
const s = await jget(BASE + '/api/status');
dot.className = 'status-dot ' + (s.graylog || 'unknown');
dot.title = s.graylog === 'ok' ? 'Graylog 已连接'
: s.graylog === 'error' ? 'Graylog 连接失败'
: s.graylog === 'stale' ? 'Graylog 状态过期'
: 'Graylog 状态未知';
} catch (err) {
dot.className = 'status-dot error';
dot.title = '后端状态接口不可用';
}
}
async function loadStats() {
try {
const s = await jget(BASE + '/api/stats');
$('#stats').innerHTML = `
<div class="stat hot"><div class="n">${fmt(s.fused_total)}</div><div class="k">应封禁（融合）</div></div>
<div class="stat warm"><div class="n">${fmt(s.fused_new_24h)}</div><div class="k">今日新增</div></div>
<div class="stat cool"><div class="n">${fmt(s.fused_both)}</div><div class="k">双来源</div></div>
<div class="stat"><div class="n">${fmt(s.fused_behavior_only)}</div><div class="k">行为新发现</div></div>`;
$('#meta').innerHTML = `最近更新 <b>${localTime(s.updated_at)}</b><br>
<a href="${BASE}/api/ips" target="_blank">JSON</a> ·
<a href="${BASE}/api/stats" target="_blank">stats</a>`;
} catch (err) {
$('#meta').textContent = '统计加载失败: ' + err.message;
}
}
async function loadList() {
try {
const data = await jget(BASE + '/api/ips?limit=500');
state.items = data.items || [];
render();
$('#listHint').textContent =
`共 ${state.items.length} 条 · 点击任意一行查看融合证据 · 按发现时间倒序（最新在前）· 每 60 秒更新`;
} catch (err) {
$('#rows').innerHTML = `<tr><td colspan="10" class="empty">加载失败：${esc(err.message)}</td></tr>`;
}
}
function filteredItems() {
const q = state.q.trim().toLowerCase();
if (!q) return state.items;
return state.items.filter(item => {
const hay = [item.ip, item.country_name, item.region, item.asn_brand,
item.asn_org, item.asn, item.sources,
...(item.reasons || []),
...Object.keys(item.rules || {})].join(' ').toLowerCase();
return hay.includes(q);
});
}
let toastTimer = null;
function toast(message) {
const el = $('#toast');
el.textContent = message;
el.classList.add('show');
clearTimeout(toastTimer);
toastTimer = setTimeout(() => el.classList.remove('show'), 2200);
}
function legacyCopy(text) {
const area = document.createElement('textarea');
area.value = text;
area.style.position = 'fixed';
area.style.opacity = '0';
area.style.left = '-9999px';
document.body.appendChild(area);
area.select();
document.execCommand('copy');
document.body.removeChild(area);
}
async function copyText(text, message) {
try {
if (navigator.clipboard && window.isSecureContext) {
await navigator.clipboard.writeText(text);
} else {
legacyCopy(text);
}
toast(message);
} catch (err) {
legacyCopy(text);
toast(message + '（兼容模式）');
}
}
async function copyIps(separator) {
const rows = filteredItems();
if (!rows.length) { toast('当前列表为空'); return; }
const newline = separator === 'newline';
const text = rows.map(item => item.ip).join(newline ? '\n' : ',');
const label = newline ? '换行分隔' : '逗号分隔';
try {
if (navigator.clipboard && window.isSecureContext) {
await navigator.clipboard.writeText(text);
} else {
legacyCopy(text);
}
toast(`已复制 ${rows.length} 个 IP（${label}）`);
} catch (err) {
legacyCopy(text);
toast(`已复制 ${rows.length} 个 IP（${label}，兼容模式）`);
}
}
function downloadFile(content, filename, mime) {
const blob = new Blob([content], { type: mime });
const link = document.createElement('a');
link.href = URL.createObjectURL(blob);
link.download = filename;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
URL.revokeObjectURL(link.href);
}
function exportList(format) {
const rows = filteredItems();
if (!rows.length) { toast('当前列表为空'); return; }
const stamp = new Date().toISOString().slice(0, 10);
if (format === 'txt') {
const text = rows.map(item => item.ip).join('\n') + '\n';
downloadFile(text, 'malicious-ips-' + stamp + '.txt',
'text/plain;charset=utf-8');
toast(`已导出 ${rows.length} 个 IP（换行分隔）`);
return;
}
const quote = value => '"' + String(value == null ? '' : value)
.replace(/"/g, '""') + '"';
const header = ['ip', 'total', 'first_seen', 'last_malicious_at',
'feature_score', 'behavior_score', 'sources', 'asn', 'asn_brand',
'asn_type', 'reasons'];
const lines = [header.join(',')];
rows.forEach(item => {
lines.push([
item.ip, item.total, item.first_seen || '', item.last_malicious_at || '',
item.feature_score ?? '', item.behavior_score ?? '', item.sources || '',
item.asn || '', item.asn_brand || '', item.asn_type || '',
(item.reasons || []).join('; ')
].map(quote).join(','));
});
downloadFile('\ufeff' + lines.join('\n'), 'malicious-ips-' + stamp + '.csv',
'text/csv;charset=utf-8');
toast(`已导出 ${rows.length} 条到 CSV`);
}
function closeMenus() {
document.querySelectorAll('.menu.open').forEach(menu => menu.classList.remove('open'));
}
function toggleMenu(id) {
const menu = $('#' + id);
const willOpen = !menu.classList.contains('open');
closeMenus();
if (willOpen) menu.classList.add('open');
}
document.addEventListener('click', e => {
if (!e.target.closest('.menu-wrap')) closeMenus();
});
document.addEventListener('keydown', e => {
if (e.key === 'Escape') closeMenus();
});
function render() {
const rows = filteredItems();
$('#empty').hidden = rows.length > 0;
$('#rows').innerHTML = rows.map(item => {
const src = SOURCE[item.sources] || ['特征', 'badge-danger'];
const reasons = (item.reasons || []).slice(0, 2).join('；') || '-';
const scoreTitle = `特征分 ${item.feature_score ?? '-'} / 行为分 ${item.behavior_score ?? '-'}`;
const geoFlag = flagOf(item.country_code);
const geoName = [item.country_name, item.region].filter(Boolean).join(' · ');
const geoTitle = [item.country_name, item.region, item.city]
.filter(Boolean).join(' · ') || '未知';
return `<tr data-ip="${esc(item.ip)}">
<td class="ip copyable" data-copyip="${esc(item.ip)}" title="点击复制 IP">${esc(item.ip)}</td>
<td class="geo">
<div class="geo-l1" title="${esc(geoTitle)}">${geoFlag ? geoFlag + ' ' : ''}${
esc(geoName || '-')}</div>
<div class="geo-l2" title="AS${esc(item.asn || '-')} · ${esc(item.asn_org || '-')} · ${esc(TYPE_LABEL[item.asn_type] || '未知')}">${
item.asn_brand ? esc(item.asn_icon + ' ' + item.asn_brand) : '❓ 未知'}</div>
</td>
<td><span class="badge ${src[1]}">${src[0]}</span></td>
<td title="${esc(scoreTitle)}"><div class="scorecell"><span>${Number(item.score).toFixed(0)}</span>
<div class="bar"><i style="width:${Math.min(100, item.score)}%"></i></div></div></td>
<td class="num">${fmt(item.total)}</td>
<td class="num">${perMinute(item.total, item.window)} /分</td>
<td class="muted">${localTime(item.first_seen)}</td>
<td class="muted">${item.last_malicious_at ? localTime(item.last_malicious_at) : '-'}</td>
<td class="reason" title="${esc(reasons)}">${esc(reasons)}</td>
<td class="muted">›</td>
</tr>`;
}).join('');
$('#rows').querySelectorAll('tr[data-ip]').forEach(tr => {
tr.addEventListener('click', e => {
if (e.target.closest('[data-copyip]')) return;
location.hash = '#/ip/' + encodeURIComponent(tr.dataset.ip);
});
});
}
function bars(items, valueKey, labelKey) {
const max = Math.max(1, ...items.map(x => Number(x[valueKey] || 0)));
return items.map(x => `<div class="row">
<div class="t" title="${esc(x[labelKey])}">${esc(x[labelKey])}</div>
<div class="b"><i style="width:${Math.round(100 * x[valueKey] / max)}%"></i></div>
<div class="v">${fmt(x[valueKey])}</div>
</div>`).join('');
}
function timeline(items) {
if (!items || !items.length) return '<div class="muted">无时间分布数据</div>';
const max = Math.max(1, ...items.map(x => Number(x.count || 0)));
const barsHtml = items.map(x => `<div style="height:${Math.max(2, Math.round(100 * x.count / max))}%"
title="${esc(localTime(x.timestamp))} · ${fmt(x.count)} 次"></div>`).join('');
return `<div class="tl">${barsHtml}</div>
<div class="tl-axis"><span>${esc(localTime(items[0].timestamp))}</span>
<span>${esc(localTime(items[items.length - 1].timestamp))}</span></div>`;
}
function advice(item) {
if (item.sources === 'both') {
return `<div class="advice"><b>建议：封禁（特征 + 行为双来源）。</b>
该 IP 命中明确攻击特征，且行为模式与攻击一致，置信度最高，可直接下发封禁。</div>`;
}
if (item.sources === 'behavior') {
return `<div class="advice"><b>建议：封禁（行为检测判定）。</b>
行为特征高度符合自动化扫描/攻击模式，已进入融合封禁名单。</div>`;
}
return `<div class="advice"><b>建议：封禁（特征检测）。</b>
该 IP 命中明确攻击特征，可直接下发到 F5 / AD 黑名单。</div>`;
}
async function showDetail(ip) {
state.inDetail = true;
$('#listView').hidden = true;
const view = $('#detailView');
view.hidden = false;
view.innerHTML = '<div class="skeleton">正在加载证据…</div>';
document.title = ip + ' · 恶意 IP 清单';
loadStats();
try {
const data = await jget(BASE + '/api/ip/' + encodeURIComponent(ip));
const item = data.item;
const ev = data.evidence || {};
const tier = TIER[item.tier] || ['未知', 'badge-muted'];
const src = SOURCE[item.sources] || ['特征', 'badge-danger'];
const window = item.window || ev.window || 0;
const windowText = window >= 86400 ? (window / 86400).toFixed(1) + ' 天'
: window >= 3600 ? (window / 3600).toFixed(1) + ' 小时'
: Math.round(window / 60) + ' 分钟';
const rules = Object.entries(item.rules || {})
.map(([name, count]) => ({ name: RULE_TITLES[name] || name, count }));
const ruleRows = rules.length
? `<div class="rows">${bars(rules, 'count', 'name')}</div>`
: '<div class="muted">无规则命中明细</div>';
const extraNotes = (item.reasons || []).filter(reason =>
!rules.some(rule => String(reason).startsWith(rule.name)));
const dimension = (items, empty) => (items || []).length
? `<div class="rows">${bars(items.map(x => ({name: x.term, count: x.count})), 'count', 'name')}</div>`
: `<div class="muted">${empty}</div>`;
const statusRows = dimension(ev.statuses, '无状态码分布');
const methodRows = dimension(ev.methods, '无方法分布');
const hostRows = dimension(ev.hosts, '无域名分布');
const statusBadge = value => {
const text = String(value || '').trim();
if (!text) return '<span class="badge badge-muted">无状态</span>';
const cls = text.startsWith('2') ? 'badge-ok'
: text.startsWith('3') ? 'badge-info' : 'badge-danger';
return `<span class="badge ${cls}">${esc(text)}</span>`;
};
state.detailSamples = ev.samples || [];
const ruleTitle = name => (ev.rule_titles || {})[name] || name;
const sampleCards = (ev.samples || []).map((s, idx) => {
const ruleBadges = (s.rules || []).map(name =>
`<span class="badge badge-warn" title="${esc(name)}">${esc(ruleTitle(name))}</span>`
).join('');
const client = [s.client_ip, s.client_port].filter(Boolean).join(':');
const server = [s.server_ip, s.server_port].filter(Boolean).join(':');
const geo = [s.client_ip_country_code, s.client_ip_city_name]
.filter(x => x && x !== 'N/A').join(' / ');
const facts = [
`<div><b>客户端</b><span class="mono">${esc(client || '-')}</span></div>`,
`<div><b>服务端</b><span class="mono">${esc(server || '-')}</span></div>`,
s.host ? `<div><b>Host</b><span class="mono expandable" data-idx="${idx}" data-mfield="host">${esc(s.host)}</span></div>` : '',
s.bigip ? `<div><b>VIP</b><span class="mono">${esc(s.bigip)}${s.vs ? ' · ' + esc(s.vs) : ''}</span></div>` : '',
geo ? `<div><b>地区</b>${esc(geo)}</div>` : '',
s.rt ? `<div><b>响应</b>${esc(s.rt)} s</div>` : '',
s.req_len ? `<div><b>请求长度</b>${esc(s.req_len)} B</div>` : '',
s.referer ? `<div><b>Referer</b><span class="mono expandable" data-idx="${idx}" data-mfield="referer">${esc(truncate(s.referer, 66))}</span></div>` : '',
s.source ? `<div><b>采集来源</b><span class="mono">${esc(s.source)}</span></div>` : '',
].join('');
return `<div class="s-card">
<div class="s-head">
<span class="mono">${esc(localTime(s.timestamp))}</span>
<span class="badge badge-info">${esc(s.method || '?')}</span>
${statusBadge(s.status)}
${ruleBadges}
</div>
<div class="s-uri mono"><span class="expandable" data-idx="${idx}" data-mfield="uri">${esc(s.uri || '-')}</span>${
s.query ? '<span class="muted expandable" data-idx="' + idx + '" data-mfield="query"> ?'
+ esc(truncate(s.query, 60)) + '</span>' : ''}</div>
<div class="s-grid">${facts}</div>
<div class="s-ua"><b>UA</b><span class="mono expandable" data-idx="${idx}" data-mfield="ua">${
esc(truncate(s.ua || '-', 90))}</span></div>
${s.message ? `<button class="linkbtn" data-idx="${idx}" data-mfield="message">查看完整原始日志行</button>` : ''}
</div>`;
}).join('');
const sampleNote = (ev.samples || []).length
? `按攻击类型抽样：每种类型保留最近 3 条，共 ${(ev.samples || []).length} 条，不是该 IP 的全部日志。`
: `暂无样本${ev.samples_error ? '：' + esc(ev.samples_error) : ''}${
ev.available ? '' : '（证据服务不可用：' + esc(ev.reason || ev.error || '未知原因') + '）'}`;
const behavior = item.behavior;
const behaviorPanel = behavior ? `<div class="panel">
<h3>行为检测画像</h3>
<ul class="reasons">
<li>建议：${esc(behavior.verdict || '-')}（行为分 ${Number(behavior.score || 0).toFixed(0)}）</li>
${behavior.reason_text ? `<li>${esc(behavior.reason_text)}</li>` : ''}
<li>出口：${behavior.asn_brand ? esc((behavior.asn_icon || '') + ' ' + behavior.asn_brand) : '未知'}${
behavior.asn ? ' · AS' + esc(behavior.asn) : ''}（${esc(TYPE_LABEL[behavior.asn_type] || '未知')}）</li>
<li>24h 请求 ${fmt(behavior.total)} 次 · 失败率 ${Number(behavior.pct4 || 0).toFixed(0)}% ·
不同路径 ${behavior.uri_card || 0} 个 · UA ${behavior.ua_card || 0} 种</li>
</ul>
${behavior.asn_org ? `<div class="hint">归属组织：${esc(behavior.asn_org)}</div>` : ''}
${behavior.share_note ? `<div class="hint">${esc(behavior.share_note)}</div>` : ''}
${(behavior.signals || []).length ? `<div class="hint">信号：${behavior.signals.map(esc).join('；')}</div>` : ''}
</div>` : '';
view.innerHTML = `
<a class="back" href="#/">← 返回名单</a>
<div class="detail-head">
<div>
<div class="detail-title">
<h2 class="copyable" data-copyip="${esc(item.ip)}" title="点击复制 IP">${esc(item.ip)}</h2>
<span class="badge ${src[1]}">${src[0]}</span>
</div>
<div class="geo-line">
<span class="badge badge-muted">
${flagOf(item.country_code)} ${esc(item.country_name || '未知地区')}${
item.region ? ' · ' + esc(item.region) : ''}${
item.city ? ' · ' + esc(item.city) : ''}
</span>
<span class="badge badge-muted" title="${esc(item.asn_org || '未知')} · ${esc(TYPE_LABEL[item.asn_type] || '未知')}">
${item.asn_brand ? esc(item.asn_icon + ' ' + item.asn_brand) : '❓ 未知'}${
item.asn ? ' · AS' + esc(item.asn) : ''}
</span>
</div>
<div class="muted" style="margin-top:6px">
发现时间 ${localTime(item.first_seen)} · 最近活动 ${localTime(item.last_seen)}
· 观测窗口 ${windowText}
${item.evidence_at ? ' · 证据生成 ' + esc(localTime(item.evidence_at)) : ''}
</div>
</div>
<div class="ring" style="--pct:${Math.min(100, item.score)}"><b>${Number(item.score).toFixed(0)}</b></div>
</div>
<div class="kpis">
<div class="kpi"><div class="n">${fmt(item.total)}</div><div class="k">请求总量</div></div>
<div class="kpi"><div class="n">${perMinute(item.total, window)}</div><div class="k">平均频率（次/分）</div></div>
<div class="kpi"><div class="n">${fmt(rules.reduce((a, r) => a + r.count, 0))}</div><div class="k">规则命中次数</div></div>
<div class="kpi"><div class="n">${fmt(item.hits)}</div><div class="k">被检测到轮次</div></div>
</div>
${advice(item)}
${behaviorPanel}
${ev.stale ? `<div class="advice warn"><b>注意：</b>${esc(ev.reason || 'Graylog 中该时段日志已过期，以下为判定时的快照')}</div>` : ''}
<div class="panel">
<h3>判定依据</h3>
${ruleRows}
${extraNotes.length ? `<div class="notes">${extraNotes.map(note =>
`<div>${esc(note)}</div>`).join('')}</div>` : ''}
<div class="hint">计数为该观测窗口内命中次数，不是全量历史。</div>
</div>
<div class="panel">
<h3>请求频率分布</h3>
${ev.available ? timeline(ev.timeline) : '<div class="muted">不可用</div>'}
<div class="hint">${esc(ev.interval || '')} 粒度
${ev.peak ? ' · 峰值 ' + esc(localTime(ev.peak.timestamp)) + '（' + fmt(ev.peak.count) + ' 次）' : ''}
${ev.active_buckets ? ' · ' + ev.active_buckets + ' 个活跃时段' : ''}
· 用于判断是持续扫描还是短时爆发。</div>
</div>
<div class="panel">
<h3>状态码分布</h3>
${statusRows}
</div>
<div class="panel">
<h3>请求方法分布</h3>
${methodRows}
</div>
<div class="panel">
<h3>访问域名分布</h3>
${hostRows}
</div>
<div class="panel">
<h3>证据样本</h3>
${sampleCards || '<div class="muted">暂无样本</div>'}
<div class="hint">${esc(sampleNote)} 点击虚线字段可在弹窗中查看完整内容，Esc 关闭。</div>
</div>`;
} catch (err) {
view.innerHTML = `<a class="back" href="#/">← 返回名单</a>
<div class="empty">加载失败：${esc(err.message)}</div>`;
}
}
function showList() {
state.inDetail = false;
document.title = '恶意 IP 清单';
$('#detailView').hidden = true;
$('#detailView').innerHTML = '';
$('#listView').hidden = false;
loadStatus();
loadStats();
loadList();
}
function route() {
const match = location.hash.match(/^#\/ip\/(.+)$/);
if (match) showDetail(decodeURIComponent(match[1]));
else showList();
}
$('#search').addEventListener('input', e => { state.q = e.target.value; render(); });
document.addEventListener('click', e => {
const target = e.target.closest('[data-copyip]');
if (!target) return;
e.stopPropagation();
copyText(target.dataset.copyip, '已复制 IP：' + target.dataset.copyip);
});
$('#copyBtn').addEventListener('click', e => {
e.stopPropagation();
toggleMenu('copyMenu');
});
$('#exportBtn').addEventListener('click', e => {
e.stopPropagation();
toggleMenu('exportMenu');
});
$('#copyMenu').addEventListener('click', e => {
const item = e.target.closest('[data-copy]');
if (!item) return;
closeMenus();
copyIps(item.dataset.copy);
});
$('#exportMenu').addEventListener('click', e => {
const item = e.target.closest('[data-export]');
if (!item) return;
closeMenus();
exportList(item.dataset.export);
});
$('#auto').addEventListener('change', e => {
if (e.target.checked) startTimer(); else clearInterval(state.timer);
});
function startTimer() {
clearInterval(state.timer);
state.timer = setInterval(() => {
loadStatus();
if (!state.inDetail) { loadStats(); loadList(); }
}, 30000);
}
window.addEventListener('hashchange', route);
startTimer();
route();
</script>
</body></html>"""
def make_handler(store, window, interval, detector=None, path_prefix=""):
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def _send(self, code, payload, content_type="application/json; charset=utf-8"):
if isinstance(payload, (dict, list)):
payload = json.dumps(payload, ensure_ascii=False)
raw = payload.encode("utf-8")
self.send_response(code)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(raw)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(raw)
def _send_raw(self, code, text, content_type, filename=None):
raw = text.encode("utf-8")
self.send_response(code)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(raw)))
if filename:
self.send_header("Content-Disposition",
'attachment; filename="%s"' % filename)
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(raw)
def do_GET(self):
parsed = urlparse(self.path)
path = parsed.path
query = {key: values[0] for key, values in parse_qs(parsed.query).items()}
if path_prefix:
if path == path_prefix:
self.send_response(302)
self.send_header("Location", path_prefix + "/")
self.end_headers()
return
if path.startswith(path_prefix + "/"):
path = path[len(path_prefix):] or "/"
elif path != "/healthz":
return self._send_raw(404, "Not Found",
"text/plain; charset=utf-8")
if path == "/":
return self._send(200, DASHBOARD.replace("__BASE__", path_prefix),
"text/html; charset=utf-8")
if path == "/healthz":
return self._send(200, {"ok": True})
if path == "/api/stats":
return self._send(200, store.stats())
if path == "/api/status":
status = {"graylog": "unknown", "last_run": None,
"last_error": None, "age_seconds": None}
if detector is not None:
age = (time.time() - detector.last_success
if getattr(detector, "last_success", None) else None)
status["last_run"] = detector.last_run
status["last_error"] = detector.last_error
status["age_seconds"] = round(age, 1) if age is not None else None
if detector.last_error and (age is None
or age > interval * 2):
status["graylog"] = "error"
elif age is not None and age <= max(120, interval * 2.5):
status["graylog"] = "ok"
elif age is not None:
status["graylog"] = "stale"
return self._send(200, status)
if path == "/api/ips":
limit = min(2000, int(query.get("limit", 500)))
fused = store.fused_items(limit)
window = detector.window if detector is not None else 3600
items = []
for row in fused:
try:
rules = json.loads(row.get("s_rules") or "{}")
except ValueError:
rules = {}
items.append({
"ip": row["ip"],
"sources": row["sources"],
"tier": row.get("tier") or "behavior",
"score": max(float(row.get("feature_score") or 0),
float(row.get("behavior_score") or 0)),
"feature_score": row.get("feature_score"),
"behavior_score": row.get("behavior_score"),
"total": row.get("total") or 0,
"rules": rules,
"reasons": [row.get("fused_reason") or ""],
"first_seen": row.get("first_banned_at"),
"last_seen": row.get("last_activity_at"),
"last_malicious_at": row.get("last_activity_at"),
"window": window,
"country_code": row.get("country_code") or "",
"country_name": row.get("country_name") or "",
"region": row.get("region") or "",
"city": row.get("city") or "",
"asn": row.get("asn") or "",
"asn_org": row.get("asn_org") or "",
"asn_brand": row.get("asn_brand") or "",
"asn_type": row.get("asn_type") or "",
"asn_icon": row.get("asn_icon") or "",
})
return self._send(200, {"count": len(items),
"updated_at": store.stats().get("updated_at"),
"items": items})
if path in ("/api/export.csv", "/api/export.txt"):
scope = query.get("scope", "core")
if scope not in ("core", "all"):
scope = "core"
sort = query.get("sort", "first_seen")
if sort not in Store.SORT_COLUMNS:
sort = "first_seen"
order = query.get("order", "desc")
if order not in ("asc", "desc"):
order = "desc"
items = store.list(limit=20000,
scope=None if scope == "all" else scope,
sort=sort, order=order)
if path.endswith(".csv"):
body = _csv_text(items)
ctype = "text/csv; charset=utf-8"
filename = "malicious-ips.csv"
else:
body = "\n".join(item["ip"] for item in items) + ("\n" if items else "")
ctype = "text/plain; charset=utf-8"
filename = "malicious-ips.txt"
return self._send_raw(200, body, ctype, filename)
if path.startswith("/api/ip/"):
ip = path[len("/api/ip/"):]
item = store.get(ip)
fused = store.fused_row(ip)
behavior = store.behavior_profile(ip)
if item is None and fused is None:
return self._send(404, {"error": "not found", "ip": ip})
if item is None:
item = {
"ip": ip, "tier": "behavior",
"score": (behavior or {}).get("score") or 0,
"total": (behavior or {}).get("total") or 0,
"rules": {},
"reasons": [(fused or {}).get("fused_reason")
or (behavior or {}).get("reason_text") or ""],
"first_seen": (fused or {}).get("first_banned_at"),
"last_seen": (fused or {}).get("last_activity_at"),
"last_malicious_at": (fused or {}).get("last_activity_at"),
"window": 86400,
"country_code": (behavior or {}).get("country_code", ""),
"country_name": (behavior or {}).get("country_name", ""),
"region": (behavior or {}).get("region", ""),
"city": (behavior or {}).get("city", ""),
"asn": (behavior or {}).get("asn", ""),
"asn_org": (behavior or {}).get("asn_org", ""),
"asn_brand": (behavior or {}).get("asn_brand", ""),
"asn_type": (behavior or {}).get("asn_type", ""),
"asn_icon": (behavior or {}).get("asn_icon", ""),
}
item["sources"] = (fused or {}).get("sources") or "feature"
item["behavior"] = behavior
item["fused"] = fused
if item.get("window") is None and detector is not None:
item["window"] = detector.window
# Evidence is fetched live from Graylog on click (memory
# cached) so the database stays small; a compact snapshot is
# kept as a fallback once the index has rotated away.
stored_raw = item.pop("evidence", None)
try:
evidence = collect_evidence(
detector, ip, item.get("rules"),
item.get("window") or (detector.window if detector else None))
except Exception as exc: # never kill the connection
evidence = {"available": False, "error": str(exc)}
live_ok = bool(evidence.get("available")
and (evidence.get("uris") or evidence.get("samples")))
if live_ok:
try:
stamp = now_iso()
store.set_evidence(ip, _compact_evidence(evidence), stamp)
item["evidence_at"] = stamp
except Exception:
pass
evidence["stale"] = False
else:
snapshot = None
if stored_raw:
try:
snapshot = json.loads(stored_raw)
except ValueError:
snapshot = None
if snapshot:
snapshot["available"] = True
snapshot["stale"] = True
snapshot["samples"] = []
snapshot["reason"] = "Graylog 中该时段日志已过期，以下为判定时的快照"
snapshot.setdefault("window", item.get("window"))
evidence = snapshot
return self._send(200, {"item": item, "evidence": evidence})
return self._send(404, {"error": "not found", "path": path})
return Handler
# --------------------------------------------------------------------- entry
def run(ctx, args):
fields = dict(DEFAULT_FIELDS)
profile_path = getattr(args, "profile", None)
if profile_path:
with open(profile_path, "r", encoding="utf-8") as handle:
profile = json.load(handle)
for key, value in (profile.get("fields") or {}).items():
if value and key in fields:
fields[key] = value
for attribute, key in (("field", "client_ip"), ("status_field", "status"),
("uri_field", "uri"), ("ua_field", "ua")):
value = getattr(args, attribute, None)
if value:
fields[key] = value
allow_entries = []
for path in (getattr(args, "allow", None) or []):
with open(path, "r", encoding="utf-8") as handle:
text = handle.read()
for line in text.splitlines():
line = line.split("#", 1)[0].strip()
if not line:
continue
try:
allow_entries.append(ipaddress.ip_network(line, strict=False))
except ValueError:
raise SystemExit("白名单 %s 不是合法 IP/CIDR: %r" % (path, line))
def allowlisted(ip):
try:
address = ipaddress.ip_address(ip)
except ValueError:
return False
return any(address.version == net.version and address in net
for net in allow_entries)
notifier = None
notify_path = getattr(args, "notify_config", "") or ""
if notify_path and os.path.exists(notify_path):
try:
notifier = load_notifier(notify_path)
print("推送已启用: %s" % notifier.kind, flush=True)
except Exception as exc:
print("推送配置无效: %s" % exc, flush=True)
if notifier is None:
try:
notifier = notifier_from_env()
if notifier is not None:
print("推送已启用（环境变量）: %s" % notifier.kind, flush=True)
except Exception as exc:
print("推送环境变量无效: %s" % exc, flush=True)
if getattr(args, "notify_test", False):
if notifier is None:
print("未配置推送：请创建 %s（参考 notify.example.json）"
% (notify_path or "notify.json"), flush=True)
return 1
try:
reply = notifier.send_test()
except Exception as exc:
print("测试推送失败: %s" % exc, flush=True)
return 1
print("测试推送成功: %s" % str(reply)[:300], flush=True)
return 0
store = Store(args.db)
detector = Detector(
ctx.client, store, fields,
window=int(args.window), size=int(args.size),
min_requests=int(args.min_requests), verify_limit=int(args.verify),
forget=int(args.forget), allowlist=allowlisted,
min_observe_score=float(args.min_observe_score),
warm_limit=int(getattr(args, "warm_limit", 5) or 0),
filter_str=("streams:" + ctx.stream_id) if ctx.stream_id else None,
geo_db=getattr(args, "geo_db", "") or None,
asn_db=getattr(args, "asn_db", "") or None,
)
if getattr(detector, "geo", None) is not None:
filled = detector.backfill_geo(3000)
if filled:
print("补充地理信息 %d 个 IP" % filled, flush=True)
if getattr(detector, "asn", None) is not None:
filled = detector.backfill_asn(3000)
if filled:
print("补充 ASN 信息 %d 个 IP" % filled, flush=True)
# Fused block list: existing feature detections first (historical rows keep
# their original discovery time), then the behaviour thread keeps adding.
store.sync_fused()
behavior_analyzer = None
if not getattr(args, "no_behavior", False):
behavior_analyzer = BehaviorAnalyzer(
ctx.client, store, fields,
filter_str=("streams:" + ctx.stream_id) if ctx.stream_id else None,
root=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
interval=int(getattr(args, "behavior_interval", 900) or 900),
notifier=notifier,
)
threading.Thread(target=behavior_analyzer.run_forever, daemon=True).start()
print("行为检测线程已启动（每 %d 秒一次）" % behavior_analyzer.interval,
flush=True)
def report(result):
line = ("[%s] 窗口 %ds: %d 个候选（恶意 %d / 可疑 %d / 自动化 %d），耗时 %.1fs"
% (now_iso(), result["window"], result["candidates"],
result["malicious"], result["suspicious"], result["automation"],
result["request_seconds"]))
print(line, flush=True)
if getattr(args, "fix_first_seen", False):
count = detector.backfill_first_seen(int(getattr(args, "warm_limit", 500) or 500))
print("已修正 %d 个 IP 的首次发现时间" % count, flush=True)
store.close()
return 0
if getattr(args, "warm_only", False):
count = detector.warm_snapshots(int(getattr(args, "warm_limit", 50) or 50))
print("已预热 %d 个 IP 的证据快照" % count, flush=True)
store.close()
return 0
if getattr(args, "once", False):
report(detector.run_once())
if getattr(args, "export", None):
chosen = export_blocklist(store, args.export)
print("已导出 %d 个恶意 IP -> %s" % (len(chosen), args.export))
for row in store.list(limit=int(args.top)):
print("%-16s %-11s %5.1f %6d %s" % (
row["ip"], TIER_LABELS.get(row["tier"], row["tier"]),
row["score"], row["total"], "；".join(row["reasons"][:3])))
store.close()
return 0
prefix = (getattr(args, "path_prefix", "") or "").strip()
if prefix:
prefix = "/" + prefix.strip("/")
server = ThreadingHTTPServer((args.host, int(args.port)),
make_handler(store, int(args.window),
int(args.interval), detector, prefix))
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
print("面板: http://%s:%s%s/ (Ctrl+C 退出)"
% (args.host, args.port, prefix), flush=True)
if getattr(args, "serve_only", False):
try:
thread.join()
except KeyboardInterrupt:
print("", flush=True)
finally:
server.shutdown()
store.close()
return 0
try:
while True:
try:
report(detector.run_once())
store.sync_fused()
if notifier is not None:
notify_fused(store, notifier)
if getattr(args, "export", None):
chosen = export_blocklist(store, args.export)
print(" 封禁名单: %d 个 -> %s" % (len(chosen), args.export), flush=True)
except GraylogError as exc:
detector.last_error = str(exc)
print("[%s] 检测失败: %s" % (now_iso(), exc), flush=True)
except Exception as exc:
detector.last_error = str(exc)
print("[%s] 检测异常: %s" % (now_iso(), exc), flush=True)
time.sleep(max(5, int(args.interval)))
except KeyboardInterrupt:
print("\n停止。数据库: %s" % os.path.abspath(args.db), flush=True)
finally:
server.shutdown()
store.close()
return 0
Reference in new issue View Git Blame Copy Permalink