3347 lines
149 KiB
Python
3347 lines
149 KiB
Python
"""Continuous malicious-IP detection, persistent list, local web dashboard.
|
||
|
||
Design
|
||
------
|
||
* Every ``interval`` seconds the detector runs a **fixed set of terms queries**
|
||
over the last ``window`` seconds, so the load on Elasticsearch does not grow
|
||
with the number of IPs.
|
||
* Rules were derived from an 8 hour exploration of the F5 access logs
|
||
(2026-10-17). The four observed patterns:
|
||
|
||
1. 漏洞探测 -- low volume, hits /actuator, /.env, /wp-login, /druid, /nacos,
|
||
/swagger, /ISAPI, /console ... (very high signal, few requests)
|
||
2. 登录爆破 -- repeated 401/403 on /login, /passport, /captcha endpoints
|
||
3. 爬虫/机器人 -- known bot UA (DotBot) + high 404 ratio + many content URIs
|
||
4. 自动化客户端 -- one single business API at a steady rate (a broken token,
|
||
not an attacker) -> downgraded, never bans
|
||
|
||
* Every candidate is persisted in SQLite (``data/ips.db``) so the list survives
|
||
restarts, and the same process serves a small read-only dashboard on
|
||
``http://127.0.0.1:8088``.
|
||
"""
|
||
|
||
import csv
|
||
import datetime
|
||
import html
|
||
import io
|
||
import ipaddress
|
||
import re
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import threading
|
||
import time
|
||
from datetime import datetime, timezone
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
from urllib.parse import parse_qs, urlparse
|
||
|
||
from . import detect
|
||
from .client import normalize_histogram, normalize_terms
|
||
from .http import GraylogError
|
||
from .notify import load_notifier, notifier_from_env
|
||
from .behavior import BehaviorAnalyzer, notify_fused, RULE_TITLES
|
||
from .asn import AsnDB
|
||
|
||
# --------------------------------------------------------------------- rules
|
||
# Queries run against Graylog's query_string. Field names are substituted from
|
||
# the profile (client_ip / status / uri / ua). Slashes must be escaped: an
|
||
# unescaped /wp-login* would be parsed as a Lucene regex.
|
||
PROBE_RULES = ("probe_java", "probe_config", "probe_cms", "probe_iot",
|
||
"probe_devops", "probe_webshell", "exploit_payload")
|
||
|
||
RULE_DEFS = (
|
||
{
|
||
"name": "probe_java",
|
||
"title": "Spring/中间件探测",
|
||
"weight": 60.0,
|
||
"saturate": 2.0,
|
||
"query": lambda f: (
|
||
"%s:(\\/actuator* OR \\/druid* OR \\/nacos* OR \\/swagger* "
|
||
"OR \\/v2\\/api-docs* OR \\/v3\\/api-docs* OR \\/openapi* OR \\/jmx* "
|
||
"OR \\/heapdump* OR \\/threaddump* OR \\/console* OR \\/manager\\/html* "
|
||
"OR \\/solr* OR \\/struts* OR \\/weblogic* OR \\/log4j*)"
|
||
) % f["uri"],
|
||
},
|
||
{
|
||
"name": "probe_config",
|
||
"title": "配置/凭证探测",
|
||
"weight": 60.0,
|
||
"saturate": 2.0,
|
||
"query": lambda f: (
|
||
"%s:(\\/.env* OR \\/.git* OR \\/.svn* OR \\/.hg* OR \\/.aws* OR \\/.ssh* "
|
||
"OR \\/.kube* OR \\/.docker* OR \\/appsettings* OR \\/web.config "
|
||
"OR \\/WEB-INF* OR \\/META-INF* OR \\/backup* OR \\/dump* "
|
||
"OR \\/db.sql* OR \\/.DS_Store)"
|
||
) % f["uri"],
|
||
},
|
||
{
|
||
"name": "probe_cms",
|
||
"title": "CMS 探测",
|
||
"weight": 60.0,
|
||
"saturate": 2.0,
|
||
"query": lambda f: (
|
||
"%s:(\\/wp-* OR \\/xmlrpc.php OR \\/phpmyadmin* OR \\/pma* "
|
||
"OR \\/administrator* OR \\/joomla* OR \\/drupal* OR \\/thinkphp* "
|
||
"OR \\/uc_server* OR \\/dede* OR \\/index.php?s=*)"
|
||
) % f["uri"],
|
||
},
|
||
{
|
||
"name": "probe_iot",
|
||
"title": "IoT/路由器探测",
|
||
"weight": 60.0,
|
||
"saturate": 2.0,
|
||
"query": lambda f: (
|
||
"%s:(\\/ISAPI* OR \\/LAPI* OR \\/HNAP1* OR \\/boaform* OR \\/GponForm* "
|
||
"OR \\/cgi-bin\\/luci* OR \\/goform* OR \\/setup.cgi* OR \\/login.cgi*)"
|
||
) % f["uri"],
|
||
},
|
||
{
|
||
"name": "probe_devops",
|
||
"title": "DevOps/监控探测",
|
||
"weight": 60.0,
|
||
"saturate": 2.0,
|
||
"query": lambda f: (
|
||
"%s:(\\/jenkins* OR \\/grafana* OR \\/kibana* OR \\/zabbix* "
|
||
"OR \\/prometheus* OR \\/consul* OR \\/eureka* OR \\/harbor* "
|
||
"OR \\/nexus* OR \\/sonarqube* OR \\/gitlab* OR \\/portainer*)"
|
||
) % f["uri"],
|
||
},
|
||
{
|
||
"name": "probe_webshell",
|
||
"title": "WebShell/上传探测",
|
||
"weight": 60.0,
|
||
"saturate": 2.0,
|
||
"query": lambda f: (
|
||
"%s:(\\/shell.php OR \\/shell.asp OR \\/shell.jsp OR \\/cmd.php "
|
||
"OR \\/cmd.asp OR \\/webshell* OR \\/phpinfo* OR \\/dbadmin* "
|
||
"OR \\/eval-stdin* OR \\/vendor\\/phpunit*)"
|
||
) % f["uri"],
|
||
},
|
||
{
|
||
# Regex: only over 1 hour max, a 25h regex aggregation kills the ES node.
|
||
"name": "exploit_payload",
|
||
"title": "注入/穿越载荷",
|
||
"weight": 65.0,
|
||
"saturate": 2.0,
|
||
"window_cap": 3600,
|
||
"query": lambda f: (
|
||
"%s:/.*(etc\\/passwd|proc\\/self|jndi:|union.*select|information_schema"
|
||
"|sleep\\(|benchmark\\(|\\.\\.%%2f|%%2e%%2e|%%252f|base64_decode|xp_cmdshell).*/"
|
||
) % f["uri"],
|
||
},
|
||
{
|
||
"name": "login_attack",
|
||
"title": "登录失败尝试",
|
||
"weight": 55.0,
|
||
"saturate": 8.0,
|
||
"query": lambda f: (
|
||
"(%s:(\\/login* OR \\/passport* OR \\/captcha* OR \\/jsxy\\/api\\/login*)) "
|
||
"AND %s:(401 OR 403)"
|
||
) % (f["uri"], f["status"]),
|
||
},
|
||
{
|
||
"name": "login_burst",
|
||
"title": "登录接口高频",
|
||
"weight": 15.0,
|
||
"saturate": 2000.0,
|
||
"query": lambda f: (
|
||
"%s:(\\/login* OR \\/passport* OR \\/captcha* OR \\/jsxy\\/api\\/login*)"
|
||
) % f["uri"],
|
||
},
|
||
{
|
||
"name": "bad_ua",
|
||
"title": "扫描器 UA",
|
||
"weight": 55.0,
|
||
"saturate": 1.0,
|
||
"query": lambda f: (
|
||
"%s:(sqlmap* OR nikto* OR nuclei* OR nmap* OR masscan* OR zgrab* "
|
||
"OR wpscan* OR gobuster* OR dirsearch* OR dirbuster* OR ffuf* "
|
||
"OR feroxbuster* OR wfuzz* OR joomscan* OR droopescan* OR whatweb* "
|
||
"OR xray* OR vulmap* OR afrog* OR hydra*)"
|
||
) % f["ua"],
|
||
},
|
||
{
|
||
"name": "bot_ua",
|
||
"title": "爬虫 UA",
|
||
"weight": 50.0,
|
||
"saturate": 20.0,
|
||
"query": lambda f: (
|
||
'%s:"Mozilla/5.0 (compatible; DotBot/1.2; +https://opensiteexplorer.org/dotbot; help@moz.com)"'
|
||
) % f["ua"],
|
||
},
|
||
{
|
||
"name": "not_found",
|
||
"title": "404",
|
||
"weight": 0.0, # scored via the 404 ratio, not the raw count
|
||
"saturate": 1.0,
|
||
"query": lambda f: "%s:404" % f["status"],
|
||
},
|
||
)
|
||
|
||
DEFAULT_FIELDS = {
|
||
"client_ip": "client_ip",
|
||
"status": "status",
|
||
"uri": "uri",
|
||
"ua": "ua",
|
||
"method": "method",
|
||
"host": "host",
|
||
"server_ip": "server_ip",
|
||
"bigip": "bigip",
|
||
}
|
||
|
||
AUTH_PATH_HINTS = ("login", "signin", "auth", "token", "captcha", "oauth", "password")
|
||
|
||
# How far back to look when dating the first attack of an IP.
|
||
BACKFILL_WINDOW = 129600 # ~36h, as far back as the index retention allows
|
||
|
||
TIER_MALICIOUS = "malicious"
|
||
TIER_SUSPICIOUS = "suspicious"
|
||
TIER_AUTOMATION = "automation"
|
||
TIER_OBSERVE = "observe"
|
||
|
||
TIER_LABELS = {
|
||
TIER_MALICIOUS: "恶意",
|
||
TIER_SUSPICIOUS: "可疑",
|
||
TIER_AUTOMATION: "自动化",
|
||
TIER_OBSERVE: "观察",
|
||
}
|
||
|
||
|
||
# ------------------------------------------------------------------ classify
|
||
RATE_RULES = ("login_attack", "login_burst")
|
||
|
||
|
||
def classify(total, hits, window_scale=1.0):
|
||
"""Pure scoring function: request counts per rule -> (tier, score, reasons).
|
||
|
||
``window_scale`` normalises rate-like rules to a 15 minute baseline, so an
|
||
8 hour analysis does not turn a student who mistyped a password six times
|
||
into a "suspicious IP". Behavioural rules (probing / bad UA / crawler) are
|
||
never scaled: any probing is bad regardless of window length.
|
||
"""
|
||
scale = max(1.0, float(window_scale))
|
||
score = 0.0
|
||
reasons = []
|
||
|
||
def add(rule, title, count):
|
||
nonlocal score
|
||
if count <= 0:
|
||
return
|
||
saturate = rule["saturate"]
|
||
if rule["name"] in RATE_RULES:
|
||
saturate *= scale
|
||
points = rule["weight"] * min(1.0, count / saturate)
|
||
score += points
|
||
reasons.append("%s %d 次" % (title, count))
|
||
|
||
for rule in RULE_DEFS:
|
||
if rule["weight"] <= 0:
|
||
continue
|
||
add(rule, rule["title"], hits.get(rule["name"], 0))
|
||
|
||
not_found = hits.get("not_found", 0)
|
||
if not_found and total:
|
||
ratio = not_found / float(total)
|
||
if ratio >= 0.5 and not_found >= 20:
|
||
score += 25.0 * min(1.0, ratio / 0.8)
|
||
reasons.append("404 占比 %.0f%%（%d/%d）" % (ratio * 100, not_found, total))
|
||
|
||
score = min(100.0, round(score, 1))
|
||
if score >= 60:
|
||
tier = TIER_MALICIOUS
|
||
elif score >= 30:
|
||
tier = TIER_SUSPICIOUS
|
||
elif score > 0:
|
||
tier = TIER_OBSERVE
|
||
else:
|
||
tier = TIER_OBSERVE
|
||
return tier, score, reasons
|
||
|
||
|
||
def is_public(text):
|
||
try:
|
||
return not detect.is_internal(detect.as_ip(text))
|
||
except (TypeError, ValueError):
|
||
return False
|
||
|
||
|
||
# --------------------------------------------------------------------- store
|
||
class Store:
|
||
def __init__(self, path):
|
||
directory = os.path.dirname(os.path.abspath(path))
|
||
if directory:
|
||
os.makedirs(directory, exist_ok=True)
|
||
self.path = path
|
||
self.lock = threading.Lock()
|
||
self.conn = sqlite3.connect(path, check_same_thread=False)
|
||
try:
|
||
self.conn.execute("PRAGMA journal_mode=WAL")
|
||
self.conn.execute("PRAGMA busy_timeout=5000")
|
||
except sqlite3.Error:
|
||
pass
|
||
self.conn.row_factory = sqlite3.Row
|
||
self._init()
|
||
|
||
def _init(self):
|
||
with self.conn:
|
||
self.conn.execute(
|
||
"""CREATE TABLE IF NOT EXISTS ip_stats (
|
||
ip TEXT PRIMARY KEY,
|
||
tier TEXT NOT NULL,
|
||
score REAL NOT NULL,
|
||
total INTEGER NOT NULL DEFAULT 0,
|
||
rules TEXT NOT NULL DEFAULT '{}',
|
||
reasons TEXT NOT NULL DEFAULT '[]',
|
||
first_seen TEXT,
|
||
last_seen TEXT,
|
||
risk TEXT,
|
||
window INTEGER,
|
||
evidence TEXT,
|
||
evidence_at TEXT,
|
||
ever_malicious INTEGER NOT NULL DEFAULT 0,
|
||
last_malicious_at TEXT,
|
||
hits INTEGER NOT NULL DEFAULT 1,
|
||
updated_at TEXT
|
||
)"""
|
||
)
|
||
for column in ("risk TEXT", "window INTEGER", "evidence TEXT",
|
||
"evidence_at TEXT", "ever_malicious INTEGER DEFAULT 0",
|
||
"last_malicious_at TEXT", "country_code TEXT",
|
||
"country_name TEXT", "region TEXT", "city TEXT",
|
||
"asn TEXT", "asn_org TEXT", "asn_brand TEXT",
|
||
"asn_type TEXT", "asn_icon TEXT"):
|
||
try:
|
||
self.conn.execute("ALTER TABLE ip_stats ADD COLUMN %s" % column)
|
||
except sqlite3.OperationalError:
|
||
pass # column already exists
|
||
try:
|
||
self.conn.execute(
|
||
|
||
"""UPDATE ip_stats SET ever_malicious = 1,
|
||
last_malicious_at = COALESCE(
|
||
last_malicious_at,
|
||
(SELECT MAX(e.ts) FROM events e
|
||
WHERE e.ip = ip_stats.ip AND e.tier = 'malicious'),
|
||
last_seen)
|
||
WHERE COALESCE(ever_malicious, 0) = 0 AND (
|
||
tier = 'malicious' OR EXISTS (
|
||
SELECT 1 FROM events e
|
||
WHERE e.ip = ip_stats.ip
|
||
AND (e.tier = 'malicious'
|
||
OR e.detail LIKE '%malicious%')))"""
|
||
)
|
||
except sqlite3.OperationalError:
|
||
pass # events table may not exist yet on a fresh db
|
||
self.conn.execute(
|
||
"""CREATE TABLE IF NOT EXISTS behavior_profiles (
|
||
ip TEXT PRIMARY KEY,
|
||
score REAL, verdict TEXT, signals TEXT, reason_text TEXT,
|
||
share_note TEXT, behavior_type TEXT,
|
||
asn TEXT, asn_org TEXT, shared_type TEXT,
|
||
asn_brand TEXT, asn_type TEXT, asn_icon TEXT,
|
||
total INTEGER, pct4 REAL, uri_card INTEGER,
|
||
uri_templates INTEGER, ua_card INTEGER, host_card INTEGER,
|
||
coverage REAL, peak_share REAL, top_uri TEXT, top_ua TEXT,
|
||
last_ts TEXT, updated_at TEXT,
|
||
country_code TEXT, country_name TEXT, region TEXT, city TEXT
|
||
)"""
|
||
)
|
||
for column in ("asn_brand TEXT", "asn_type TEXT", "asn_icon TEXT",
|
||
"historical INTEGER DEFAULT 0"):
|
||
try:
|
||
self.conn.execute(
|
||
"ALTER TABLE behavior_profiles ADD COLUMN %s" % column)
|
||
except sqlite3.OperationalError:
|
||
pass
|
||
self.conn.execute(
|
||
"""CREATE TABLE IF NOT EXISTS ip_fused (
|
||
ip TEXT PRIMARY KEY,
|
||
sources TEXT NOT NULL,
|
||
feature_score REAL, behavior_score REAL,
|
||
first_banned_at TEXT, last_activity_at TEXT,
|
||
last_pushed_at TEXT, push_count INTEGER NOT NULL DEFAULT 0,
|
||
fused_reason TEXT, updated_at TEXT
|
||
)"""
|
||
)
|
||
for column in ("push_fail_count INTEGER DEFAULT 0",
|
||
"push_next_at TEXT"):
|
||
try:
|
||
self.conn.execute(
|
||
"ALTER TABLE ip_fused ADD COLUMN %s" % column)
|
||
except sqlite3.OperationalError:
|
||
pass
|
||
self.conn.execute(
|
||
"CREATE INDEX IF NOT EXISTS idx_ip_fused_first "
|
||
"ON ip_fused(first_banned_at DESC, ip)")
|
||
self.conn.execute(
|
||
"""CREATE TABLE IF NOT EXISTS meta (
|
||
key TEXT PRIMARY KEY, value TEXT
|
||
)"""
|
||
)
|
||
self.conn.execute(
|
||
"""CREATE TABLE IF NOT EXISTS notifications (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
ip TEXT, channel TEXT, status TEXT, attempt INTEGER,
|
||
error TEXT, created_at TEXT
|
||
)"""
|
||
)
|
||
self.conn.execute(
|
||
"CREATE INDEX IF NOT EXISTS idx_notifications_created "
|
||
"ON notifications(created_at DESC)")
|
||
self.conn.execute(
|
||
"""CREATE TABLE IF NOT EXISTS events (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
ts TEXT, ip TEXT, tier TEXT, score REAL, detail TEXT
|
||
)"""
|
||
)
|
||
|
||
def upsert(self, record, now):
|
||
with self.lock, self.conn:
|
||
row = self.conn.execute(
|
||
"SELECT tier, score, ever_malicious, last_malicious_at, "
|
||
"last_seen, rules, total FROM ip_stats WHERE ip = ?",
|
||
(record["ip"],),
|
||
).fetchone()
|
||
malicious = record["tier"] == TIER_MALICIOUS
|
||
geo = record.get("geo") or {}
|
||
info = record.get("asn") or {}
|
||
if row is None:
|
||
last_seen = record.get("last_seen") or now
|
||
last_malicious_at = record.get("last_malicious_at") or (
|
||
last_seen if malicious else None)
|
||
self.conn.execute(
|
||
"""INSERT INTO ip_stats
|
||
(ip, tier, score, total, rules, reasons,
|
||
first_seen, last_seen, risk, window, hits, updated_at,
|
||
ever_malicious, last_malicious_at,
|
||
country_code, country_name, region, city,
|
||
asn, asn_org, asn_brand, asn_type, asn_icon)
|
||
VALUES (?,?,?,?,?,?,?,?,?,?,1,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||
(record["ip"], record["tier"], record["score"], record["total"],
|
||
json.dumps(record["rules"], ensure_ascii=False),
|
||
json.dumps(record["reasons"], ensure_ascii=False),
|
||
record.get("first_seen") or now, last_seen,
|
||
record.get("risk"), record.get("window"), now,
|
||
1 if malicious else 0, last_malicious_at,
|
||
geo.get("country_code"), geo.get("country_name"),
|
||
geo.get("region"), geo.get("city"),
|
||
info.get("asn"), info.get("org"), info.get("brand"),
|
||
info.get("type"), info.get("icon")),
|
||
)
|
||
self._event(now, record, "new")
|
||
return "new"
|
||
else:
|
||
ever = 1 if (malicious or row["ever_malicious"]) else 0
|
||
# Timestamps only move when the detector verified new evidence,
|
||
# otherwise a stale window would keep refreshing them to "now".
|
||
last_seen = record.get("last_seen") or row["last_seen"] or now
|
||
if record.get("last_malicious_at"):
|
||
last_malicious_at = record["last_malicious_at"]
|
||
elif malicious and row["tier"] != TIER_MALICIOUS:
|
||
last_malicious_at = last_seen
|
||
else:
|
||
last_malicious_at = row["last_malicious_at"]
|
||
self.conn.execute(
|
||
"""UPDATE ip_stats
|
||
SET tier=?, score=?, total=?, rules=?, reasons=?,
|
||
last_seen=?, risk=?, window=?, hits=hits+1, updated_at=?,
|
||
ever_malicious=?, last_malicious_at=?,
|
||
country_code=COALESCE(?, country_code),
|
||
country_name=COALESCE(?, country_name),
|
||
region=COALESCE(?, region),
|
||
city=COALESCE(?, city),
|
||
asn=COALESCE(?, asn), asn_org=COALESCE(?, asn_org),
|
||
asn_brand=COALESCE(?, asn_brand),
|
||
asn_type=COALESCE(?, asn_type),
|
||
asn_icon=COALESCE(?, asn_icon)
|
||
WHERE ip=?""",
|
||
(record["tier"], record["score"], record["total"],
|
||
json.dumps(record["rules"], ensure_ascii=False),
|
||
json.dumps(record["reasons"], ensure_ascii=False),
|
||
last_seen, record.get("risk"), record.get("window"), now,
|
||
ever, last_malicious_at,
|
||
geo.get("country_code"), geo.get("country_name"),
|
||
geo.get("region"), geo.get("city"),
|
||
info.get("asn"), info.get("org"), info.get("brand"),
|
||
info.get("type"), info.get("icon"), record["ip"]),
|
||
)
|
||
if row["tier"] != record["tier"]:
|
||
self._event(now, record, "%s->%s" % (row["tier"], record["tier"]))
|
||
if malicious and row["tier"] != TIER_MALICIOUS:
|
||
return "upgraded"
|
||
return "unchanged" if row["tier"] == record["tier"] else "changed"
|
||
|
||
def _event(self, now, record, detail):
|
||
self.conn.execute(
|
||
"INSERT INTO events (ts, ip, tier, score, detail) VALUES (?,?,?,?,?)",
|
||
(now, record["ip"], record["tier"], record["score"], detail),
|
||
)
|
||
|
||
SORT_COLUMNS = {"first_seen": "first_seen",
|
||
"last_malicious": "last_malicious_at",
|
||
"score": "score",
|
||
"total": "total"}
|
||
|
||
def list(self, limit=500, tier=None, scope=None, sort="first_seen", order="desc"):
|
||
query = ("SELECT ip, tier, score, total, rules, reasons, first_seen, "
|
||
"last_seen, risk, window, hits, updated_at, evidence_at, "
|
||
"ever_malicious, last_malicious_at, country_code, country_name, "
|
||
"region, city, asn, asn_org, asn_brand, asn_type, asn_icon "
|
||
"FROM ip_stats")
|
||
args = []
|
||
where = []
|
||
if tier:
|
||
where.append("tier = ?")
|
||
args.append(tier)
|
||
if scope == "core":
|
||
where.append("ever_malicious = 1")
|
||
if where:
|
||
query += " WHERE " + " AND ".join(where)
|
||
column = self.SORT_COLUMNS.get(sort, "first_seen")
|
||
direction = "DESC" if str(order).lower() == "desc" else "ASC"
|
||
query += (" ORDER BY (%s IS NULL) ASC, %s %s, "
|
||
"COALESCE(last_malicious_at, last_seen) DESC, ip ASC LIMIT ?"
|
||
% (column, column, direction))
|
||
args.append(limit)
|
||
with self.lock:
|
||
rows = self.conn.execute(query, args).fetchall()
|
||
result = []
|
||
for row in rows:
|
||
item = dict(row)
|
||
item["rules"] = json.loads(item["rules"] or "{}")
|
||
item["reasons"] = json.loads(item["reasons"] or "[]")
|
||
result.append(item)
|
||
return result
|
||
|
||
def get(self, ip):
|
||
with self.lock:
|
||
row = self.conn.execute("SELECT * FROM ip_stats WHERE ip = ?", (ip,)).fetchone()
|
||
if row is None:
|
||
return None
|
||
item = dict(row)
|
||
item["rules"] = json.loads(item["rules"] or "{}")
|
||
item["reasons"] = json.loads(item["reasons"] or "[]")
|
||
return item
|
||
|
||
# ---------------------------------------------------------- behaviour
|
||
def upsert_historical_profiles(self, items):
|
||
"""Insert backfilled behaviour profiles; live rows win on conflict."""
|
||
stamp = now_iso()
|
||
with self.lock, self.conn:
|
||
for item in items:
|
||
self.conn.execute(
|
||
"""INSERT OR IGNORE INTO behavior_profiles
|
||
(ip, score, verdict, signals, reason_text, share_note,
|
||
behavior_type, asn, asn_org, shared_type, asn_brand,
|
||
asn_type, asn_icon, total, pct4, uri_card, uri_templates,
|
||
ua_card, host_card, coverage, peak_share, top_uri, top_ua,
|
||
last_ts, updated_at, country_code, country_name, region,
|
||
city, historical)
|
||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,1)""",
|
||
(item["ip"], item.get("score"), item.get("verdict"),
|
||
json.dumps(item.get("signals") or [], ensure_ascii=False),
|
||
item.get("reason_text"), item.get("share_note"),
|
||
item.get("behavior_type"), item.get("asn"),
|
||
item.get("asn_org"), item.get("shared_type"),
|
||
item.get("asn_brand"), item.get("asn_type"),
|
||
item.get("asn_icon"), int(item.get("total") or 0),
|
||
item.get("pct4"), item.get("uri_card"),
|
||
item.get("uri_templates"), item.get("ua_card"),
|
||
item.get("host_card"), item.get("coverage"),
|
||
item.get("peak_share"), item.get("top_uri"),
|
||
item.get("top_ua"), item.get("last_ts") or stamp, stamp,
|
||
item.get("country_code"), item.get("country_name"),
|
||
item.get("region"), item.get("city")))
|
||
|
||
def replace_behavior_profiles(self, items):
|
||
stamp = now_iso()
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"DELETE FROM behavior_profiles WHERE COALESCE(historical,0)=0")
|
||
for item in items:
|
||
self.conn.execute(
|
||
"""INSERT OR REPLACE INTO behavior_profiles
|
||
(ip, score, verdict, signals, reason_text, share_note,
|
||
behavior_type, asn, asn_org, shared_type, asn_brand,
|
||
asn_type, asn_icon, total, pct4, uri_card, uri_templates,
|
||
ua_card, host_card, coverage, peak_share, top_uri, top_ua,
|
||
last_ts, updated_at, country_code, country_name, region,
|
||
city, historical)
|
||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,0)""",
|
||
(item["ip"], item.get("score"), item.get("verdict"),
|
||
json.dumps(item.get("signals") or [], ensure_ascii=False),
|
||
item.get("reason_text"), item.get("share_note"),
|
||
item.get("behavior_type"), item.get("asn"),
|
||
item.get("asn_org"), item.get("shared_type"),
|
||
item.get("asn_brand"), item.get("asn_type"),
|
||
item.get("asn_icon"),
|
||
int(item.get("total") or 0), item.get("pct4"),
|
||
int(item.get("uri_card") or 0),
|
||
int(item.get("uri_templates") or 0),
|
||
int(item.get("ua_card") or 0),
|
||
int(item.get("host_card") or 0), item.get("coverage"),
|
||
item.get("peak_share"), item.get("top_uri"),
|
||
item.get("top_ua"), stamp, stamp,
|
||
item.get("country_code"), item.get("country_name"),
|
||
item.get("region"), item.get("city")),
|
||
)
|
||
|
||
def behavior_profile(self, ip):
|
||
with self.lock:
|
||
row = self.conn.execute(
|
||
"SELECT * FROM behavior_profiles WHERE ip = ?", (ip,)).fetchone()
|
||
if row is None:
|
||
return None
|
||
item = dict(row)
|
||
try:
|
||
item["signals"] = json.loads(item.get("signals") or "[]")
|
||
except ValueError:
|
||
item["signals"] = []
|
||
return item
|
||
|
||
def fused_row(self, ip):
|
||
with self.lock:
|
||
row = self.conn.execute(
|
||
"SELECT * FROM ip_fused WHERE ip = ?", (ip,)).fetchone()
|
||
return dict(row) if row else None
|
||
|
||
@staticmethod
|
||
def _fused_filter(q):
|
||
if not q:
|
||
return "", []
|
||
like = "%" + q.replace("%", "").replace("_", "") + "%"
|
||
fields = ("f.ip", "f.fused_reason", "f.sources",
|
||
"s.country_name", "s.region", "s.asn_brand", "s.asn_org",
|
||
"b.country_name", "b.region", "b.asn_brand", "b.asn_org")
|
||
clause = " WHERE (" + " OR ".join("%s LIKE ?" % field
|
||
for field in fields) + ")"
|
||
return clause, [like] * len(fields)
|
||
|
||
def fused_count(self, q=None):
|
||
where, params = self._fused_filter(q)
|
||
with self.lock:
|
||
return self.conn.execute(
|
||
"SELECT COUNT(*) FROM ip_fused f "
|
||
"LEFT JOIN ip_stats s ON s.ip = f.ip "
|
||
"LEFT JOIN behavior_profiles b ON b.ip = f.ip" + where,
|
||
params).fetchone()[0]
|
||
|
||
def fused_items(self, limit=500, offset=0, q=None):
|
||
where, params = self._fused_filter(q)
|
||
with self.lock:
|
||
rows = self.conn.execute(
|
||
"""SELECT f.*, s.tier AS tier, s.total AS total, s.rules AS s_rules,
|
||
b.total AS b_total,
|
||
s.asn AS s_asn, s.asn_org AS s_asn_org,
|
||
s.asn_brand AS s_asn_brand, s.asn_type AS s_asn_type,
|
||
s.asn_icon AS s_asn_icon,
|
||
b.asn AS b_asn, b.asn_org AS b_asn_org,
|
||
b.asn_brand AS b_asn_brand, b.asn_type AS b_asn_type,
|
||
b.asn_icon AS b_asn_icon,
|
||
s.country_code AS s_cc, s.country_name AS s_cn,
|
||
s.region AS s_region, s.city AS s_city,
|
||
b.country_code AS b_cc, b.country_name AS b_cn,
|
||
b.region AS b_region, b.city AS b_city
|
||
FROM ip_fused f
|
||
LEFT JOIN ip_stats s ON s.ip = f.ip
|
||
LEFT JOIN behavior_profiles b ON b.ip = f.ip
|
||
""" + where + """
|
||
ORDER BY f.first_banned_at DESC, f.ip ASC
|
||
LIMIT ? OFFSET ?""",
|
||
params + [int(limit), int(offset)]).fetchall()
|
||
items = []
|
||
for row in rows:
|
||
item = dict(row)
|
||
item["country_code"] = item.get("s_cc") or item.get("b_cc") or ""
|
||
item["country_name"] = item.get("s_cn") or item.get("b_cn") or ""
|
||
item["region"] = item.get("s_region") or item.get("b_region") or ""
|
||
item["city"] = item.get("s_city") or item.get("b_city") or ""
|
||
item["asn"] = item.get("s_asn") or item.get("b_asn") or ""
|
||
item["asn_org"] = item.get("s_asn_org") or item.get("b_asn_org") or ""
|
||
item["asn_brand"] = item.get("s_asn_brand") or item.get("b_asn_brand") or ""
|
||
item["asn_type"] = item.get("s_asn_type") or item.get("b_asn_type") or ""
|
||
item["asn_icon"] = item.get("s_asn_icon") or item.get("b_asn_icon") or ""
|
||
item["total"] = item.get("total") or item.get("b_total") or 0
|
||
items.append(item)
|
||
return items
|
||
|
||
def _fused_reason(self, feature_rules, behavior_reason):
|
||
parts = []
|
||
if feature_rules:
|
||
try:
|
||
rules = json.loads(feature_rules or "{}")
|
||
except ValueError:
|
||
rules = {}
|
||
if rules:
|
||
ranked = sorted(rules.items(), key=lambda kv: -kv[1])[:3]
|
||
parts.append("特征命中：" + "、".join(
|
||
"%s %d 次" % (RULE_TITLES.get(name, name), count)
|
||
for name, count in ranked))
|
||
if behavior_reason:
|
||
parts.append("行为表现：" + behavior_reason)
|
||
return "；".join(parts)
|
||
|
||
def sync_fused(self, now=None):
|
||
"""Merge feature + behaviour verdicts into the fused block list.
|
||
|
||
Rule: feature-confirmed IPs always stay (option A, historical list);
|
||
behaviour adds IPs only when its verdict is "建议封禁" (shared/NAT
|
||
networks never enter the block list). first_banned_at is pinned so an
|
||
IP only reaches the top of the list once.
|
||
"""
|
||
now = now or now_iso()
|
||
with self.lock, self.conn:
|
||
feature_rows = self.conn.execute(
|
||
"SELECT ip, score, rules, first_seen, last_seen, "
|
||
"last_malicious_at, country_code, country_name, region, city "
|
||
"FROM ip_stats WHERE ever_malicious = 1").fetchall()
|
||
behavior_rows = self.conn.execute(
|
||
"SELECT * FROM behavior_profiles").fetchall()
|
||
existing = {row["ip"]: row for row in self.conn.execute(
|
||
"SELECT * FROM ip_fused").fetchall()}
|
||
initial = not existing
|
||
behavior = {row["ip"]: row for row in behavior_rows}
|
||
wanted = {}
|
||
|
||
def iso(value):
|
||
return value or now
|
||
|
||
for row in feature_rows:
|
||
ip = row["ip"]
|
||
beh = behavior.get(ip)
|
||
beh_ban = bool(beh and beh["verdict"] == "建议封禁")
|
||
prev = existing.get(ip)
|
||
first = (prev["first_banned_at"] if prev
|
||
else iso(row["first_seen"] or row["last_malicious_at"]))
|
||
activity = max(filter(None, [row["last_seen"],
|
||
beh["last_ts"] if beh else None]),
|
||
default=now)
|
||
wanted[ip] = {
|
||
"sources": "both" if beh_ban else "feature",
|
||
"feature_score": row["score"],
|
||
"behavior_score": beh["score"] if beh else None,
|
||
"first_banned_at": first,
|
||
"last_activity_at": iso(activity),
|
||
"fused_reason": self._fused_reason(
|
||
row["rules"], beh["reason_text"] if beh else None),
|
||
}
|
||
|
||
for ip, beh in behavior.items():
|
||
if beh["verdict"] != "建议封禁" or ip in wanted:
|
||
continue
|
||
prev = existing.get(ip)
|
||
historical = bool(beh["historical"]) if "historical" in beh.keys() else False
|
||
fallback = (beh["last_ts"] if historical and beh["last_ts"] else now)
|
||
wanted[ip] = {
|
||
"sources": "behavior",
|
||
"feature_score": None,
|
||
"behavior_score": beh["score"],
|
||
"first_banned_at": prev["first_banned_at"] if prev else fallback,
|
||
"last_activity_at": iso(beh["last_ts"]),
|
||
"fused_reason": "行为表现：" + (beh["reason_text"] or ""),
|
||
}
|
||
|
||
for ip, data in wanted.items():
|
||
prev = existing.get(ip)
|
||
if prev:
|
||
self.conn.execute(
|
||
"""UPDATE ip_fused
|
||
SET sources=?, feature_score=?, behavior_score=?,
|
||
last_activity_at=?, fused_reason=?, updated_at=?
|
||
WHERE ip=?""",
|
||
(data["sources"], data["feature_score"],
|
||
data["behavior_score"], data["last_activity_at"],
|
||
data["fused_reason"], now, ip))
|
||
else:
|
||
self.conn.execute(
|
||
"""INSERT INTO ip_fused
|
||
(ip, sources, feature_score, behavior_score,
|
||
first_banned_at, last_activity_at, last_pushed_at,
|
||
push_count, fused_reason, updated_at)
|
||
VALUES (?,?,?,?,?,?,?,0,?,?)""",
|
||
(ip, data["sources"], data["feature_score"],
|
||
data["behavior_score"], data["first_banned_at"],
|
||
data["last_activity_at"], now if initial else None,
|
||
data["fused_reason"], now))
|
||
|
||
stale = [ip for ip in existing if ip not in wanted]
|
||
for ip in stale:
|
||
self.conn.execute("DELETE FROM ip_fused WHERE ip = ?", (ip,))
|
||
return len(wanted)
|
||
|
||
# ------------------------------------------------------------- meta
|
||
def get_meta(self, key, default=None):
|
||
with self.lock:
|
||
row = self.conn.execute(
|
||
"SELECT value FROM meta WHERE key=?", (key,)).fetchone()
|
||
return row["value"] if row else default
|
||
|
||
def set_meta(self, key, value):
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"INSERT INTO meta(key, value) VALUES(?,?) "
|
||
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
||
(key, str(value)))
|
||
|
||
def incr_meta(self, key, amount=1):
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"INSERT INTO meta(key, value) VALUES(?,?) "
|
||
"ON CONFLICT(key) DO UPDATE SET value=CAST("
|
||
"CAST(value AS INTEGER)+? AS TEXT)",
|
||
(key, str(amount), int(amount)))
|
||
row = self.conn.execute(
|
||
"SELECT value FROM meta WHERE key=?", (key,)).fetchone()
|
||
return int(row["value"] or 0)
|
||
|
||
def meta_dict(self):
|
||
with self.lock:
|
||
rows = self.conn.execute("SELECT key, value FROM meta").fetchall()
|
||
return {row["key"]: row["value"] for row in rows}
|
||
|
||
# ------------------------------------------------------- notifications
|
||
def log_notification(self, ip, channel, status, attempt, error=None,
|
||
stamp=None):
|
||
stamp = stamp or now_iso()
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"INSERT INTO notifications(ip, channel, status, attempt, error, "
|
||
"created_at) VALUES(?,?,?,?,?,?)",
|
||
(ip, channel, status, int(attempt or 0), error, stamp))
|
||
|
||
def mark_push_failed(self, ips, error, stamp=None, base=60, cap=3600):
|
||
stamp = stamp or now_iso()
|
||
base_time = time.time()
|
||
with self.lock, self.conn:
|
||
for ip in ips:
|
||
row = self.conn.execute(
|
||
"SELECT COALESCE(push_fail_count,0) AS n FROM ip_fused "
|
||
"WHERE ip=?", (ip,)).fetchone()
|
||
fails = int(row["n"] if row else 0) + 1
|
||
delay = min(cap, base * (2 ** (fails - 1)))
|
||
next_at = time.strftime(
|
||
"%Y-%m-%dT%H:%M:%SZ",
|
||
time.gmtime(base_time + delay))
|
||
self.conn.execute(
|
||
"UPDATE ip_fused SET push_fail_count=?, push_next_at=? "
|
||
"WHERE ip=?", (fails, next_at, ip))
|
||
self.conn.execute(
|
||
"INSERT INTO notifications(ip, channel, status, attempt, "
|
||
"error, created_at) VALUES(?,?,?,?,?,?)",
|
||
(ip, "dingtalk", "failed", fails, error, stamp))
|
||
return len(ips)
|
||
|
||
def notification_stats(self, hours=24):
|
||
since = time.strftime("%Y-%m-%dT%H:%M:%SZ",
|
||
time.gmtime(time.time() - hours * 3600))
|
||
with self.lock:
|
||
rows = self.conn.execute(
|
||
"SELECT status, COUNT(*) AS n FROM notifications "
|
||
"WHERE created_at >= ? GROUP BY status", (since,)).fetchall()
|
||
recent = self.conn.execute(
|
||
"SELECT * FROM notifications ORDER BY id DESC LIMIT 30").fetchall()
|
||
counts = {row["status"]: int(row["n"]) for row in rows}
|
||
sent = counts.get("sent", 0)
|
||
failed = counts.get("failed", 0)
|
||
total = sent + failed
|
||
return {"sent": sent, "failed": failed,
|
||
"success_ratio": round(sent / total, 4) if total else None,
|
||
"recent": [dict(row) for row in recent]}
|
||
|
||
def due_pushes(self, active_hours=24, cooldown_hours=24):
|
||
now = time.time()
|
||
active_cut = time.strftime("%Y-%m-%dT%H:%M:%SZ",
|
||
time.gmtime(now - active_hours * 3600))
|
||
push_cut = time.strftime("%Y-%m-%dT%H:%M:%SZ",
|
||
time.gmtime(now - cooldown_hours * 3600))
|
||
with self.lock:
|
||
rows = self.conn.execute(
|
||
"""SELECT f.*, s.country_name AS s_cn, s.region AS s_region,
|
||
s.total AS s_total, s.asn AS s_asn,
|
||
s.asn_brand AS s_asn_brand, s.asn_type AS s_asn_type,
|
||
s.asn_icon AS s_asn_icon,
|
||
b.country_name AS b_cn, b.region AS b_region,
|
||
b.total AS b_total, b.asn AS b_asn,
|
||
b.asn_brand AS b_asn_brand, b.asn_type AS b_asn_type,
|
||
b.asn_icon AS b_asn_icon
|
||
FROM ip_fused f
|
||
LEFT JOIN ip_stats s ON s.ip = f.ip
|
||
LEFT JOIN behavior_profiles b ON b.ip = f.ip
|
||
WHERE f.last_activity_at >= ?
|
||
AND (f.last_pushed_at IS NULL OR f.last_pushed_at <= ?)
|
||
AND (f.push_next_at IS NULL OR f.push_next_at <= ?)
|
||
ORDER BY f.first_banned_at DESC LIMIT 50""",
|
||
(active_cut, push_cut, now_iso())).fetchall()
|
||
items = []
|
||
for row in rows:
|
||
item = dict(row)
|
||
item["country_name"] = item.get("s_cn") or item.get("b_cn") or ""
|
||
item["region"] = item.get("s_region") or item.get("b_region") or ""
|
||
item["total"] = item.get("s_total") or item.get("b_total") or 0
|
||
item["asn"] = item.get("s_asn") or item.get("b_asn") or ""
|
||
item["asn_brand"] = (item.get("s_asn_brand")
|
||
or item.get("b_asn_brand") or "")
|
||
item["asn_type"] = (item.get("s_asn_type")
|
||
or item.get("b_asn_type") or "")
|
||
item["asn_icon"] = (item.get("s_asn_icon")
|
||
or item.get("b_asn_icon") or "")
|
||
items.append(item)
|
||
return items
|
||
|
||
def mark_pushed(self, ips, stamp, channel="dingtalk"):
|
||
with self.lock, self.conn:
|
||
for ip in ips:
|
||
self.conn.execute(
|
||
"UPDATE ip_fused SET last_pushed_at=?, "
|
||
"push_count=push_count+1, push_fail_count=0, "
|
||
"push_next_at=NULL WHERE ip=?", (stamp, ip))
|
||
self.conn.execute(
|
||
"INSERT INTO notifications(ip, channel, status, attempt, "
|
||
"error, created_at) VALUES(?,?,?,?,?,?)",
|
||
(ip, channel, "sent", 1, None, stamp))
|
||
self.incr_meta("push_sent_total", len(ips))
|
||
|
||
def stats(self):
|
||
with self.lock:
|
||
rows = self.conn.execute(
|
||
"SELECT tier, COUNT(*) AS n FROM ip_stats GROUP BY tier"
|
||
).fetchall()
|
||
core = self.conn.execute(
|
||
"SELECT COUNT(*) AS n FROM ip_stats WHERE ever_malicious = 1"
|
||
).fetchone()["n"]
|
||
permanent = self.conn.execute(
|
||
"SELECT COUNT(*) AS n FROM ip_stats WHERE ever_malicious = 1"
|
||
).fetchone()["n"]
|
||
events = self.conn.execute("SELECT COUNT(*) AS n FROM events").fetchone()["n"]
|
||
updated = self.conn.execute(
|
||
"SELECT MAX(updated_at) AS ts FROM ip_stats"
|
||
).fetchone()["ts"]
|
||
counts = {row["tier"]: row["n"] for row in rows}
|
||
with self.lock:
|
||
fused_total = self.conn.execute(
|
||
"SELECT COUNT(*) AS n FROM ip_fused").fetchone()["n"]
|
||
fused_both = self.conn.execute(
|
||
"SELECT COUNT(*) AS n FROM ip_fused WHERE sources='both'"
|
||
).fetchone()["n"]
|
||
fused_behavior = self.conn.execute(
|
||
"SELECT COUNT(*) AS n FROM ip_fused WHERE sources='behavior'"
|
||
).fetchone()["n"]
|
||
fused_new = self.conn.execute(
|
||
"SELECT COUNT(*) AS n FROM ip_fused WHERE first_banned_at >= ?",
|
||
(time.strftime("%Y-%m-%dT%H:%M:%SZ",
|
||
time.gmtime(time.time() - 86400)),)
|
||
).fetchone()["n"]
|
||
pushed = self.conn.execute(
|
||
"SELECT COUNT(*) AS n FROM ip_fused WHERE last_pushed_at IS NOT NULL"
|
||
).fetchone()["n"]
|
||
return {
|
||
"total": sum(counts.values()),
|
||
"core": core,
|
||
"permanent": permanent,
|
||
"by_tier": counts,
|
||
"events": events,
|
||
"updated_at": updated,
|
||
"fused_total": fused_total,
|
||
"fused_new_24h": fused_new,
|
||
"fused_both": fused_both,
|
||
"fused_behavior_only": fused_behavior,
|
||
"fused_pushed": pushed,
|
||
}
|
||
|
||
def update_first_seen(self, ip, timestamp):
|
||
with self.lock, self.conn:
|
||
cursor = self.conn.execute(
|
||
"UPDATE ip_stats SET first_seen = ? "
|
||
"WHERE ip = ? AND (first_seen IS NULL OR first_seen > ?)",
|
||
(timestamp, ip, timestamp),
|
||
)
|
||
return cursor.rowcount
|
||
|
||
def update_asn(self, ip, info):
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"UPDATE ip_stats SET asn=?, asn_org=?, asn_brand=?, asn_type=?, "
|
||
"asn_icon=? WHERE ip=?",
|
||
(info.get("asn"), info.get("org"), info.get("brand"),
|
||
info.get("type"), info.get("icon"), ip))
|
||
|
||
def update_geo(self, ip, geo):
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"UPDATE ip_stats SET country_code=?, country_name=?, region=?, "
|
||
"city=? WHERE ip=?", (geo.get("country_code"), geo.get("country_name"),
|
||
geo.get("region"), geo.get("city"), ip))
|
||
|
||
def update_times(self, ip, last_seen, last_malicious_at=None):
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"UPDATE ip_stats SET last_seen = ?, "
|
||
"last_malicious_at = COALESCE(?, last_malicious_at) WHERE ip = ?",
|
||
(last_seen, last_malicious_at, ip),
|
||
)
|
||
|
||
def get_evidence(self, ip):
|
||
with self.lock:
|
||
row = self.conn.execute(
|
||
"SELECT evidence FROM ip_stats WHERE ip = ?", (ip,)
|
||
).fetchone()
|
||
return row["evidence"] if row else None
|
||
|
||
def set_evidence(self, ip, payload, now):
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"UPDATE ip_stats SET evidence = ?, evidence_at = ? WHERE ip = ?",
|
||
(json.dumps(payload, ensure_ascii=False), now, ip),
|
||
)
|
||
|
||
def forget(self, older_than_seconds, now_epoch):
|
||
cutoff = now_epoch - older_than_seconds
|
||
with self.lock, self.conn:
|
||
self.conn.execute(
|
||
"DELETE FROM ip_stats WHERE last_seen < ? "
|
||
"AND COALESCE(ever_malicious, 0) = 0",
|
||
(_iso(datetime.fromtimestamp(cutoff, timezone.utc)),),
|
||
)
|
||
|
||
def close(self):
|
||
self.conn.close()
|
||
|
||
|
||
def _iso(moment):
|
||
return moment.strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
|
||
def now_iso():
|
||
return _iso(datetime.now(timezone.utc))
|
||
|
||
|
||
# ------------------------------------------------------------------ detector
|
||
class Detector:
|
||
def __init__(self, client, store, fields, window=900, size=200,
|
||
min_requests=3, verify_limit=8, forget=7200, allowlist=None,
|
||
filter_str=None, min_observe_score=10.0, warm_limit=5, geo_db=None,
|
||
asn_db=None):
|
||
self.client = client
|
||
self.store = store
|
||
self.fields = fields
|
||
self.window = window
|
||
self.size = size
|
||
self.min_requests = min_requests
|
||
self.verify_limit = verify_limit
|
||
self.forget = forget
|
||
self.allowlist = allowlist
|
||
self.filter_str = filter_str
|
||
self.min_observe_score = float(min_observe_score)
|
||
self.warm_limit = int(warm_limit)
|
||
self.asn = None
|
||
if asn_db:
|
||
try:
|
||
self.asn = AsnDB(asn_db)
|
||
except (FileNotFoundError, sqlite3.Error):
|
||
self.asn = None
|
||
self.geo = None
|
||
if geo_db:
|
||
try:
|
||
from .geo import GeoIP
|
||
self.geo = GeoIP(geo_db)
|
||
except (FileNotFoundError, sqlite3.Error):
|
||
self.geo = None
|
||
self.rules = [
|
||
{"name": rule["name"], "title": rule["title"],
|
||
"window_cap": rule.get("window_cap"),
|
||
"query": rule["query"](fields)}
|
||
for rule in RULE_DEFS
|
||
]
|
||
self.last_run = None
|
||
self.last_success = None
|
||
self.last_error = None
|
||
self.last_duration = None
|
||
self.error_count = 0
|
||
self.runs = 0
|
||
|
||
def _terms(self, query, window_cap=None):
|
||
window = self.window
|
||
if window_cap:
|
||
window = min(window, window_cap)
|
||
data = self.client.terms(
|
||
field=self.fields["client_ip"], query=query, size=self.size,
|
||
range_seconds=window, filter_str=self.filter_str,
|
||
)
|
||
return {row["term"]: int(row["count"] or 0)
|
||
for row in normalize_terms(data)
|
||
if row["term"] not in (None, "")}
|
||
|
||
def _batch_totals(self, ips, chunk=40):
|
||
"""One terms query can return totals for many IPs at once."""
|
||
field = self.fields["client_ip"]
|
||
result = {}
|
||
for start in range(0, len(ips), chunk):
|
||
part = ips[start:start + chunk]
|
||
query = "%s:(%s)" % (field, " OR ".join('"%s"' % ip for ip in part))
|
||
try:
|
||
result.update(self._terms(query))
|
||
except GraylogError as exc:
|
||
self.last_error = "batch totals: %s" % exc
|
||
return result
|
||
|
||
def collect(self):
|
||
hits = {}
|
||
for rule in self.rules:
|
||
try:
|
||
hits[rule["name"]] = self._terms(
|
||
rule["query"], window_cap=rule.get("window_cap"))
|
||
except GraylogError as exc:
|
||
hits[rule["name"]] = {}
|
||
self.last_error = "%s: %s" % (rule["name"], exc)
|
||
|
||
# Low-volume scanners never make the "top requests" list, so the
|
||
# candidate pool must be the union of every rule hit plus the volume
|
||
# top. Totals for the missing ones are fetched in batches.
|
||
totals = self._terms("*")
|
||
candidates = set(totals)
|
||
for mapping in hits.values():
|
||
candidates.update(mapping)
|
||
missing = [ip for ip in candidates if ip not in totals]
|
||
if len(missing) > 1200: # keep the load bounded
|
||
missing = missing[:1200]
|
||
if missing:
|
||
totals.update(self._batch_totals(missing))
|
||
return totals, hits
|
||
|
||
def _verify(self, records):
|
||
"""Downgrade single-endpoint automation (broken token, not an attacker)."""
|
||
limit = self.verify_limit
|
||
for record in records[:limit]:
|
||
if record["tier"] not in (TIER_MALICIOUS, TIER_SUSPICIOUS):
|
||
continue
|
||
try:
|
||
data = self.client.terms(
|
||
field=self.fields["uri"],
|
||
query='%s:"%s"' % (self.fields["client_ip"], record["ip"]),
|
||
size=8, range_seconds=self.window, filter_str=self.filter_str,
|
||
)
|
||
except GraylogError as exc:
|
||
record["reasons"].append("核验失败: %s" % exc.message)
|
||
continue
|
||
rows = [(row["term"], row["count"]) for row in normalize_terms(data)
|
||
if row["term"] not in (None, "")]
|
||
total = sum(count for _, count in rows)
|
||
if not rows or not total:
|
||
continue
|
||
top_uri, top_count = rows[0]
|
||
ratio = top_count / float(total)
|
||
login_like = any(hint in str(top_uri).lower() for hint in AUTH_PATH_HINTS)
|
||
if ratio >= 0.9 and len(rows) <= 2 and not login_like:
|
||
record["tier"] = TIER_AUTOMATION
|
||
record["score"] = min(record["score"], 45.0)
|
||
record["rules"]["automation"] = {
|
||
"top_uri": top_uri, "ratio": round(ratio, 4)}
|
||
record["reasons"].append(
|
||
"流量集中在单一接口 %s（%.0f%%），疑似业务客户端" % (top_uri, ratio * 100))
|
||
|
||
def first_attack_at(self, ip, window=None):
|
||
"""Earliest message for this IP inside the window (probe-only first)."""
|
||
if getattr(self, "client", None) is None:
|
||
return None
|
||
window = int(window or self.window)
|
||
base = '%s:"%s"' % (self.fields["client_ip"], ip)
|
||
probe = _probe_query(self, None, window)
|
||
queries = []
|
||
if probe:
|
||
queries.append("(%s) AND %s" % (probe, base))
|
||
queries.append(base)
|
||
for query in queries:
|
||
try:
|
||
messages = self.client.search_views_messages(
|
||
query=query, range_seconds=window, limit=1,
|
||
filter_str=self.filter_str, sort_order="ASC")
|
||
except GraylogError:
|
||
continue
|
||
if messages and messages[0].get("timestamp"):
|
||
return messages[0]["timestamp"]
|
||
return None
|
||
|
||
def last_message_at(self, ip, window=None):
|
||
if getattr(self, "client", None) is None:
|
||
return None
|
||
window = int(window or self.window)
|
||
base = '%s:"%s"' % (self.fields["client_ip"], ip)
|
||
try:
|
||
messages = self.client.search_views_messages(
|
||
query=base, range_seconds=window, limit=1,
|
||
filter_str=self.filter_str, sort_order="DESC")
|
||
except GraylogError:
|
||
return None
|
||
return messages[0].get("timestamp") if messages else None
|
||
|
||
def last_attack_at(self, ip, window=None):
|
||
if getattr(self, "client", None) is None:
|
||
return None
|
||
window = int(window or self.window)
|
||
base = '%s:"%s"' % (self.fields["client_ip"], ip)
|
||
probe = _probe_query(self, None, window)
|
||
if probe:
|
||
try:
|
||
messages = self.client.search_views_messages(
|
||
query="(%s) AND %s" % (probe, base),
|
||
range_seconds=window, limit=1,
|
||
filter_str=self.filter_str, sort_order="DESC")
|
||
except GraylogError:
|
||
messages = []
|
||
if messages and messages[0].get("timestamp"):
|
||
return messages[0]["timestamp"]
|
||
return self.last_message_at(ip, window)
|
||
|
||
def backfill_asn(self, limit=5000):
|
||
if self.asn is None:
|
||
return 0
|
||
done = 0
|
||
for row in self.store.list(limit=limit, scope=None):
|
||
if row.get("asn"):
|
||
continue
|
||
info = self.asn.lookup(row["ip"])
|
||
if info:
|
||
self.store.update_asn(row["ip"], info)
|
||
done += 1
|
||
return done
|
||
|
||
def backfill_geo(self, limit=5000):
|
||
if self.geo is None:
|
||
return 0
|
||
done = 0
|
||
for row in self.store.list(limit=limit, scope=None):
|
||
if row.get("country_code"):
|
||
continue
|
||
geo = self.geo.lookup(row["ip"])
|
||
if geo:
|
||
self.store.update_geo(row["ip"], geo)
|
||
done += 1
|
||
return done
|
||
|
||
def backfill_times(self, limit=500):
|
||
"""Re-derive first-attack / last-attack / last-seen from the logs."""
|
||
if getattr(self, "client", None) is None:
|
||
return 0
|
||
rows = self.store.list(limit=20000, scope=None)
|
||
done = 0
|
||
for row in rows:
|
||
if limit and done >= limit:
|
||
break
|
||
window = max(row.get("window") or self.window, BACKFILL_WINDOW)
|
||
first = self.first_attack_at(row["ip"], window)
|
||
last = self.last_message_at(row["ip"], window)
|
||
attack = (self.last_attack_at(row["ip"], window)
|
||
if row.get("ever_malicious") else None)
|
||
if first:
|
||
self.store.update_first_seen(row["ip"], first)
|
||
if last:
|
||
self.store.update_times(row["ip"], last, attack)
|
||
done += 1
|
||
return done
|
||
|
||
# backwards-compatible alias
|
||
backfill_first_seen = backfill_times
|
||
|
||
def warm_snapshots(self, limit=10):
|
||
"""Pre-generate compact evidence for listed malicious IPs."""
|
||
if getattr(self, "client", None) is None:
|
||
return 0
|
||
rows = self.store.list(limit=10000, tier=TIER_MALICIOUS)
|
||
done = 0
|
||
for row in rows:
|
||
if limit and done >= limit:
|
||
break
|
||
if row.get("evidence_at"):
|
||
continue
|
||
try:
|
||
evidence = collect_evidence(self, row["ip"], row.get("rules"),
|
||
row.get("window") or self.window)
|
||
except Exception:
|
||
continue
|
||
if evidence.get("available"):
|
||
self.store.set_evidence(row["ip"], _compact_evidence(evidence),
|
||
now_iso())
|
||
done += 1
|
||
return done
|
||
|
||
def run_once(self):
|
||
started = time.time()
|
||
totals, hits = self.collect()
|
||
records = []
|
||
for ip, total in totals.items():
|
||
if total < self.min_requests or not is_public(ip):
|
||
continue
|
||
if self.allowlist and self.allowlist(ip):
|
||
continue
|
||
rule_hits = {name: mapping.get(ip, 0) for name, mapping in hits.items()}
|
||
tier, score, reasons = classify(
|
||
total, rule_hits, window_scale=max(1.0, self.window / 900.0))
|
||
|
||
risk = None
|
||
if tier == TIER_OBSERVE and (not reasons or score < self.min_observe_score):
|
||
continue
|
||
records.append({
|
||
"ip": ip,
|
||
"tier": tier,
|
||
"score": score,
|
||
"total": total,
|
||
"rules": {name: count for name, count in rule_hits.items() if count},
|
||
"reasons": reasons,
|
||
"risk": risk,
|
||
"window": self.window,
|
||
})
|
||
|
||
records.sort(key=lambda row: (-row["score"], -row["total"]))
|
||
for record in records:
|
||
if self.geo is not None:
|
||
record["geo"] = self.geo.lookup(record["ip"])
|
||
if self.asn is not None:
|
||
record["asn"] = self.asn.lookup(record["ip"])
|
||
existing = self.store.get(record["ip"])
|
||
if existing is None:
|
||
window = max(self.window, BACKFILL_WINDOW)
|
||
first = self.first_attack_at(record["ip"], window)
|
||
last = self.last_message_at(record["ip"], window)
|
||
if first:
|
||
record["first_seen"] = first
|
||
if last:
|
||
record["last_seen"] = last
|
||
if record["tier"] == TIER_MALICIOUS:
|
||
attack = self.last_attack_at(record["ip"], window)
|
||
if attack:
|
||
record["last_malicious_at"] = attack
|
||
elif last:
|
||
record["last_malicious_at"] = last
|
||
continue
|
||
changed = (existing.get("total") != record["total"]
|
||
or (existing.get("rules") or {}) != record["rules"])
|
||
if not changed:
|
||
# Same evidence as last round: do not move any timestamp.
|
||
continue
|
||
last = self.last_message_at(record["ip"], self.window)
|
||
if last:
|
||
record["last_seen"] = last
|
||
if record["tier"] == TIER_MALICIOUS:
|
||
attack = self.last_attack_at(record["ip"], self.window)
|
||
if attack:
|
||
record["last_malicious_at"] = attack
|
||
elif last:
|
||
record["last_malicious_at"] = last
|
||
self._verify(records)
|
||
|
||
stamp = now_iso()
|
||
# Notifications are sent only by the fused pipeline (notify_fused) so a
|
||
# new IP produces exactly one message, never one per detection source.
|
||
for record in records:
|
||
self.store.upsert(record, stamp)
|
||
self.store.forget(self.forget, time.time())
|
||
|
||
try:
|
||
self.warm_snapshots(int(getattr(self, "warm_limit", 0) or 0))
|
||
except Exception:
|
||
pass
|
||
|
||
self.last_run = stamp
|
||
self.last_success = time.time()
|
||
self.last_duration = round(time.time() - started, 2)
|
||
self.last_error = None
|
||
self.runs += 1
|
||
self.store.set_meta("detect_last_success", stamp)
|
||
self.store.set_meta("detect_last_duration", self.last_duration)
|
||
self.store.set_meta("detect_last_error", "")
|
||
self.store.incr_meta("detect_rounds_total")
|
||
return {
|
||
"window": self.window,
|
||
"request_seconds": round(time.time() - started, 2),
|
||
"candidates": len(records),
|
||
"malicious": sum(1 for row in records if row["tier"] == TIER_MALICIOUS),
|
||
"suspicious": sum(1 for row in records if row["tier"] == TIER_SUSPICIOUS),
|
||
"automation": sum(1 for row in records if row["tier"] == TIER_AUTOMATION),
|
||
}
|
||
|
||
|
||
def _csv_text(items):
|
||
buffer = io.StringIO()
|
||
writer = csv.writer(buffer)
|
||
writer.writerow(["ip", "tier", "score", "total", "first_seen",
|
||
"last_malicious_at", "risk", "reasons",
|
||
"country_code", "country_name", "region", "city",
|
||
"asn", "asn_org", "asn_brand", "asn_type"])
|
||
for item in items:
|
||
writer.writerow([
|
||
item.get("ip", ""),
|
||
item.get("tier", ""),
|
||
item.get("score", ""),
|
||
item.get("total", ""),
|
||
item.get("first_seen") or "",
|
||
item.get("last_malicious_at") or "",
|
||
item.get("risk") or "",
|
||
"; ".join(item.get("reasons") or []),
|
||
item.get("country_code") or "",
|
||
item.get("country_name") or "",
|
||
item.get("region") or "",
|
||
item.get("city") or "",
|
||
item.get("asn") or "",
|
||
item.get("asn_org") or "",
|
||
item.get("asn_brand") or "",
|
||
item.get("asn_type") or "",
|
||
])
|
||
# BOM so Excel opens the UTF-8 file without mojibake
|
||
return "\ufeff" + buffer.getvalue()
|
||
|
||
|
||
def export_blocklist(store, path):
|
||
"""Write every IP ever convicted as malicious."""
|
||
rows = store.list(limit=20000, scope="core", sort="first_seen", order="asc")
|
||
chosen = [row["ip"] for row in rows]
|
||
directory = os.path.dirname(os.path.abspath(path))
|
||
if directory:
|
||
os.makedirs(directory, exist_ok=True)
|
||
with open(path, "w", encoding="utf-8") as handle:
|
||
handle.write("\n".join(chosen) + ("\n" if chosen else ""))
|
||
return chosen
|
||
|
||
|
||
# --------------------------------------------------------------------- web
|
||
EVIDENCE_CACHE = {}
|
||
EVIDENCE_TTL = 600
|
||
|
||
SAMPLE_KEYS = ("timestamp", "client_ip", "client_port", "server_ip",
|
||
"server_port", "host", "bigip", "vs", "method", "uri", "query",
|
||
"status", "ua", "referer", "req_len", "rt",
|
||
"client_ip_country_code", "client_ip_city_name",
|
||
"client_ip_geolocation", "event_type", "source",
|
||
"gl2_remote_ip", "message")
|
||
|
||
|
||
HEADER_FIELDS = {"method", "status", "uri", "host", "client_ip", "server_ip",
|
||
"client_port", "server_port", "vs", "bigip", "event_type",
|
||
"source", "gl2_remote_ip"}
|
||
|
||
|
||
def _clean_text(value, limit=280):
|
||
"""Strip control chars / replacement chars that source logs sometimes carry."""
|
||
out = []
|
||
for char in str(value):
|
||
code = ord(char)
|
||
if char in "\n\r\t":
|
||
out.append(" ")
|
||
elif code < 32 or code == 127 or char == "\ufffd":
|
||
out.append("·")
|
||
else:
|
||
out.append(char)
|
||
text = re.sub(r"·{2,}", "·", "".join(out))
|
||
return _clip(text, limit)
|
||
|
||
|
||
def _scrub_binary(text):
|
||
"""Header-ish fields must be ASCII; a high non-ASCII ratio means garbage."""
|
||
if not text:
|
||
return text
|
||
non_ascii = sum(1 for char in text if ord(char) > 127)
|
||
if non_ascii / max(1, len(text)) > 0.2:
|
||
return "[非 UTF-8 数据]"
|
||
return text
|
||
|
||
|
||
def _clip(value, limit=280):
|
||
text = str(value)
|
||
return text[:limit] + ("…" if len(text) > limit else "")
|
||
|
||
|
||
def _sample(message):
|
||
payload = {}
|
||
for key in SAMPLE_KEYS:
|
||
value = message.get(key)
|
||
if value in (None, "", "N/A"):
|
||
continue
|
||
text = _clean_text(value, 4000 if key == "message" else 600)
|
||
if key in HEADER_FIELDS:
|
||
text = _scrub_binary(text)
|
||
payload[key] = text
|
||
return payload
|
||
|
||
|
||
def _compact_evidence(evidence):
|
||
"""Small summary persisted in SQLite: survives Graylog index rotation."""
|
||
return {
|
||
"compact": True,
|
||
"window": evidence.get("window"),
|
||
"captured_at": now_iso(),
|
||
"uris": (evidence.get("uris") or [])[:10],
|
||
"statuses": (evidence.get("statuses") or [])[:6],
|
||
"methods": (evidence.get("methods") or [])[:6],
|
||
"hosts": (evidence.get("hosts") or [])[:6],
|
||
"server_ips": (evidence.get("server_ips") or [])[:6],
|
||
"vips": (evidence.get("vips") or [])[:6],
|
||
"uas": (evidence.get("uas") or [])[:3],
|
||
"peak": evidence.get("peak"),
|
||
"interval": evidence.get("interval"),
|
||
"timeline": (evidence.get("timeline") or [])[-72:],
|
||
"sample_count": len(evidence.get("samples") or []),
|
||
}
|
||
|
||
|
||
def _probe_query(detector, item_rules=None, window=None):
|
||
"""Sample only the probe families that actually fired for this IP.
|
||
|
||
Combining all seven families (plus a regex) over a 24h window made the
|
||
Elasticsearch node drop the connection, so keep the sample query narrow and
|
||
skip rules whose window cap is smaller than the evidence window.
|
||
"""
|
||
names = {rule["name"] for rule in detector.rules if rule["name"] in PROBE_RULES}
|
||
if item_rules:
|
||
fired = sorted(((name, item_rules.get(name) or 0) for name in names
|
||
if item_rules.get(name)), key=lambda pair: -pair[1])
|
||
if fired:
|
||
# Only the strongest two families: keeps the sample query cheap for
|
||
# IPs that fired six or seven rules at once.
|
||
names = {name for name, _ in fired[:2]}
|
||
queries = []
|
||
window = window or detector.window
|
||
for rule in detector.rules:
|
||
if rule["name"] not in names or not rule.get("query"):
|
||
continue
|
||
cap = rule.get("window_cap")
|
||
if cap and window > cap:
|
||
continue
|
||
queries.append(rule["query"])
|
||
if not queries:
|
||
return None
|
||
return " OR ".join("(%s)" % query for query in queries)
|
||
|
||
|
||
def collect_evidence(detector, ip, item_rules=None, window=None):
|
||
"""Fetch a handful of typical requests for one IP (never the whole set).
|
||
|
||
All Graylog calls run concurrently and the result is persisted by the
|
||
caller, so opening the same IP a second time is instant.
|
||
"""
|
||
if detector is None or getattr(detector, "client", None) is None:
|
||
return {"available": False, "reason": "未连接 Graylog，仅展示入库统计"}
|
||
|
||
window = int(window or detector.window)
|
||
cache_key = (ip, window)
|
||
cached = EVIDENCE_CACHE.get(cache_key)
|
||
if cached and time.time() - cached[0] < EVIDENCE_TTL:
|
||
return cached[1]
|
||
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
|
||
fields = detector.fields
|
||
client = detector.client
|
||
query = '%s:"%s"' % (fields["client_ip"], ip)
|
||
result = {"available": True, "window": window, "samples": [],
|
||
"uris": [], "uas": [], "statuses": [], "methods": [], "hosts": [],
|
||
"timeline": [],
|
||
"rule_titles": {rule["name"]: rule.get("title") or rule["name"]
|
||
for rule in detector.rules}}
|
||
|
||
def terms_call(field, size):
|
||
data = client.terms(field=field, query=query, size=size,
|
||
range_seconds=window, filter_str=detector.filter_str)
|
||
rows = [{"term": _scrub_binary(_clean_text(row["term"], 240)),
|
||
"count": row["count"]}
|
||
for row in normalize_terms(data) if row["term"] not in (None, "")]
|
||
if field == fields.get("status"):
|
||
# "200 OK" and "200" are the same code; merge them for the chart.
|
||
merged = {}
|
||
for row in rows:
|
||
match = re.match(r"\s*(\d{3})", str(row["term"]))
|
||
key = match.group(1) if match else str(row["term"])
|
||
merged[key] = merged.get(key, 0) + row["count"]
|
||
rows = [{"term": key, "count": count}
|
||
for key, count in sorted(merged.items(),
|
||
key=lambda pair: (-pair[1], pair[0]))]
|
||
return rows
|
||
|
||
def samples_call():
|
||
"""Latest 3 messages per fired rule, so each attack type keeps evidence."""
|
||
fired = []
|
||
for rule in detector.rules:
|
||
name = rule["name"]
|
||
if not item_rules or not int(item_rules.get(name) or 0):
|
||
continue
|
||
if not rule.get("query"):
|
||
continue
|
||
cap = rule.get("window_cap")
|
||
rule_window = min(window, int(cap)) if cap else window
|
||
fired.append((name, rule["query"], rule_window))
|
||
if not fired:
|
||
try:
|
||
messages = client.search_views_messages(
|
||
query=query, range_seconds=window, limit=5,
|
||
filter_str=detector.filter_str)
|
||
except GraylogError:
|
||
messages = []
|
||
return [(None, messages)]
|
||
|
||
def fetch(job):
|
||
rule_name, rule_query, rule_window = job
|
||
try:
|
||
messages = client.search_views_messages(
|
||
query="(%s) AND %s" % (rule_query, query),
|
||
range_seconds=rule_window, limit=3,
|
||
filter_str=detector.filter_str)
|
||
except GraylogError:
|
||
messages = []
|
||
return rule_name, messages
|
||
|
||
results = []
|
||
with ThreadPoolExecutor(max_workers=6) as pool:
|
||
for item in pool.map(fetch, fired):
|
||
results.append(item)
|
||
return results
|
||
|
||
def timeline_call():
|
||
interval = "5m" if window <= 3600 else "1h"
|
||
history = client.histogram(query=query, range_seconds=window,
|
||
interval=interval, filter_str=detector.filter_str)
|
||
return interval, normalize_histogram(history)
|
||
|
||
dimensions = (("uris", "uri", 10), ("uas", "ua", 5), ("statuses", "status", 6),
|
||
("methods", "method", 6), ("hosts", "host", 6),
|
||
("server_ips", "server_ip", 6), ("vips", "bigip", 6))
|
||
|
||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||
futures = {}
|
||
for name, field_key, size in dimensions:
|
||
field = fields.get(field_key)
|
||
if field:
|
||
futures[name] = pool.submit(terms_call, field, size)
|
||
futures["samples"] = pool.submit(samples_call)
|
||
futures["timeline"] = pool.submit(timeline_call)
|
||
|
||
for name, future in futures.items():
|
||
try:
|
||
value = future.result()
|
||
except GraylogError as exc:
|
||
result[name + "_error"] = str(exc)
|
||
continue
|
||
if name == "samples":
|
||
merged = []
|
||
seen = {}
|
||
for rule_name, messages in value:
|
||
for message in messages:
|
||
# second precision: the same request is often logged
|
||
# twice (two client ports) within the same second.
|
||
fingerprint = ((message.get("timestamp") or "")[:19],
|
||
message.get("uri"), message.get("status"),
|
||
message.get("method"))
|
||
if fingerprint in seen:
|
||
entry = seen[fingerprint]
|
||
if rule_name and rule_name not in entry["rules"]:
|
||
entry["rules"].append(rule_name)
|
||
continue
|
||
payload = _sample(message)
|
||
payload["rules"] = [rule_name] if rule_name else []
|
||
seen[fingerprint] = payload
|
||
merged.append(payload)
|
||
merged.sort(key=lambda entry: entry.get("timestamp") or "",
|
||
reverse=True)
|
||
result["samples"] = merged[:24]
|
||
elif name == "timeline":
|
||
interval, timeline = value
|
||
result["timeline"] = timeline
|
||
result["interval"] = interval
|
||
if timeline:
|
||
peak = max(timeline, key=lambda row: row["count"])
|
||
result["peak"] = {"timestamp": peak["timestamp"],
|
||
"count": peak["count"]}
|
||
result["active_buckets"] = sum(1 for row in timeline if row["count"])
|
||
else:
|
||
result[name] = value
|
||
|
||
result["distinct_uris_shown"] = len(result.get("uris") or [])
|
||
if not result.get("samples") and not result.get("samples_error"):
|
||
result["samples_error"] = "该窗口内没有可抽取的样本"
|
||
if not result.get("uris") and not result.get("samples"):
|
||
result["available"] = False
|
||
if not result.get("error") and result.get("uris_error"):
|
||
result["error"] = result["uris_error"]
|
||
|
||
EVIDENCE_CACHE[cache_key] = (time.time(), result)
|
||
return result
|
||
|
||
|
||
DASHBOARD = r"""<!doctype html>
|
||
<html lang="zh"><head>
|
||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<title>恶意 IP 清单</title>
|
||
<style>
|
||
:root{
|
||
--bg:#F6F4EE;
|
||
--surface:#FFFFFF;
|
||
--surface-2:#FBFAF6;
|
||
--ink:#242119;
|
||
--ink-2:#4B463B;
|
||
--muted:#8B8577;
|
||
--line:#E8E3D9;
|
||
--line-strong:#D9D2C4;
|
||
--accent:#C15F3C;
|
||
--accent-deep:#A94E30;
|
||
--accent-soft:#F4E6DE;
|
||
--green:#2E6B4F;
|
||
--green-soft:#E3EEE7;
|
||
--amber:#8F6418;
|
||
--amber-soft:#F6EBD2;
|
||
--red:#A63A28;
|
||
--red-soft:#F6E1DB;
|
||
--blue:#34557F;
|
||
--blue-soft:#E6ECF5;
|
||
--radius:14px;
|
||
--shadow:0 1px 2px rgba(36,33,25,.05), 0 10px 30px rgba(36,33,25,.045);
|
||
}
|
||
*{box-sizing:border-box}
|
||
html,body{margin:0}
|
||
body{
|
||
background:var(--bg);
|
||
color:var(--ink);
|
||
font:14px/1.62 -apple-system,BlinkMacSystemFont,"PingFang SC","Hiragino Sans GB",
|
||
"Microsoft YaHei",Segoe UI,sans-serif;
|
||
min-height:100vh;
|
||
-webkit-font-smoothing:antialiased;
|
||
}
|
||
header.top{
|
||
position:sticky;top:0;z-index:30;display:flex;justify-content:space-between;
|
||
align-items:center;gap:16px;padding:18px 32px;
|
||
border-bottom:1px solid var(--line);
|
||
background:rgba(246,244,238,.88);backdrop-filter:blur(12px);
|
||
}
|
||
.brand{display:flex;align-items:center;gap:14px}
|
||
.logo{width:34px;height:34px;border-radius:11px;display:grid;place-items:center;
|
||
background:linear-gradient(145deg,#D97757,#B4532F);color:#fff;font-weight:700;
|
||
font-size:13px;letter-spacing:.5px;box-shadow:0 2px 8px rgba(193,95,60,.28)}
|
||
h1{font-family:"Iowan Old Style","Palatino Linotype",Palatino,Georgia,"Songti SC",serif;
|
||
font-size:19px;font-weight:600;margin:0;letter-spacing:.2px;color:var(--ink)}
|
||
.sub{color:var(--muted);font-size:12px;margin-top:1px}
|
||
.head-right{display:flex;align-items:center;gap:10px}
|
||
.status-dot{width:10px;height:10px;border-radius:50%;background:#C9C2B4;
|
||
transition:background .2s,box-shadow .2s;flex:none}
|
||
.status-dot.ok{background:#3E8E63;box-shadow:0 0 0 3px rgba(62,142,99,.18)}
|
||
.status-dot.error{background:#C0503A;box-shadow:0 0 0 3px rgba(192,80,58,.2)}
|
||
.status-dot.stale{background:#C08A2E;box-shadow:0 0 0 3px rgba(192,138,46,.2)}
|
||
.meta{color:var(--muted);font-size:12px;text-align:right;line-height:1.9}
|
||
.meta b{color:var(--ink-2);font-weight:600}
|
||
.meta a{color:var(--accent);text-decoration:none;border-bottom:1px solid transparent}
|
||
.meta a:hover{border-bottom-color:var(--accent)}
|
||
main{padding:26px 32px 80px;max-width:1500px;margin:0 auto}
|
||
.stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(168px,1fr));
|
||
gap:14px;margin-bottom:22px}
|
||
.stat{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);
|
||
padding:16px 18px;box-shadow:var(--shadow)}
|
||
.stat .n{font-family:"Iowan Old Style",Palatino,Georgia,serif;font-size:30px;
|
||
font-weight:600;line-height:1.15;letter-spacing:.4px;color:var(--ink)}
|
||
.stat .k{color:var(--muted);font-size:12px;margin-top:3px}
|
||
.stat.hot .n{color:var(--accent)}
|
||
.stat.warm .n{color:var(--amber)}
|
||
.stat.cool .n{color:var(--blue)}
|
||
.toolbar{display:flex;align-items:center;gap:10px;margin:2px 0 14px;flex-wrap:wrap}
|
||
input[type=search]{
|
||
flex:1;min-width:260px;background:var(--surface);border:1px solid var(--line);
|
||
color:var(--ink);border-radius:11px;padding:11px 15px;outline:none;font-size:13px;
|
||
box-shadow:0 1px 2px rgba(36,33,25,.03);transition:.15s;
|
||
}
|
||
input[type=search]::placeholder{color:#AAA392}
|
||
input[type=search]:focus{border-color:#D3B39F;box-shadow:0 0 0 3px rgba(193,95,60,.1)}
|
||
.chips{display:flex;gap:2px;background:#EFEAE0;border:1px solid var(--line);
|
||
padding:3px;border-radius:11px}
|
||
.chip{background:transparent;border:0;color:var(--ink-2);padding:7px 14px;
|
||
border-radius:8px;cursor:pointer;font-size:13px;transition:.15s;white-space:nowrap}
|
||
.chip:hover{color:var(--ink)}
|
||
.chip.active{background:var(--surface);color:var(--accent-deep);font-weight:600;
|
||
box-shadow:0 1px 3px rgba(36,33,25,.08)}
|
||
.sortgroup{display:flex;gap:8px}
|
||
.select{background:var(--surface);border:1px solid var(--line);color:var(--ink-2);
|
||
border-radius:11px;padding:10px 13px;font-size:13px;outline:none;cursor:pointer;
|
||
box-shadow:0 1px 2px rgba(36,33,25,.03);transition:.15s;appearance:none;
|
||
padding-right:32px;
|
||
background-image:url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='10' height='6' viewBox='0 0 10 6'%3E%3Cpath d='M1 1l4 4 4-4' stroke='%238B8577' stroke-width='1.6' fill='none' stroke-linecap='round'/%3E%3C/svg%3E");
|
||
background-repeat:no-repeat;background-position:right 12px center}
|
||
.select:focus{border-color:#D3B39F;box-shadow:0 0 0 3px rgba(193,95,60,.1)}
|
||
.switch{display:flex;align-items:center;gap:7px;color:var(--muted);font-size:12px;
|
||
cursor:pointer;user-select:none;white-space:nowrap}
|
||
.switch input{accent-color:var(--accent);width:15px;height:15px}
|
||
.actions{display:flex;gap:8px;margin-left:auto}
|
||
.btn{background:var(--surface);border:1px solid var(--line);color:var(--ink-2);
|
||
border-radius:11px;padding:10px 15px;font-size:13px;cursor:pointer;transition:.15s;
|
||
box-shadow:0 1px 2px rgba(36,33,25,.03);white-space:nowrap;font-weight:500}
|
||
.btn:hover{border-color:#D3B39F;color:var(--accent-deep);background:#FBF7F2}
|
||
.btn:active{transform:translateY(1px)}
|
||
.btn.primary{background:var(--accent);border-color:var(--accent);color:#fff}
|
||
.btn.primary:hover{background:var(--accent-deep);color:#fff;border-color:var(--accent-deep)}
|
||
.menu-wrap{position:relative}
|
||
.caret{font-size:10px;opacity:.65;margin-left:5px}
|
||
.menu{position:absolute;right:0;top:calc(100% + 7px);min-width:200px;
|
||
background:var(--surface);border:1px solid var(--line-strong);border-radius:12px;
|
||
box-shadow:0 18px 44px rgba(36,33,25,.16);padding:6px;display:none;z-index:60}
|
||
.menu.open{display:block}
|
||
.menu button{display:block;width:100%;text-align:left;background:transparent;border:0;
|
||
color:var(--ink-2);padding:9px 12px;border-radius:8px;font-size:13px;cursor:pointer;
|
||
transition:.12s;white-space:nowrap}
|
||
.menu button:hover{background:var(--accent-soft);color:var(--accent-deep)}
|
||
.toast{position:fixed;left:50%;bottom:30px;transform:translateX(-50%) translateY(12px);
|
||
background:#242119;color:#F6F4EE;padding:11px 20px;border-radius:12px;font-size:13px;
|
||
opacity:0;pointer-events:none;transition:.22s;z-index:300;
|
||
box-shadow:0 12px 32px rgba(36,33,25,.28)}
|
||
.toast.show{opacity:1;transform:translateX(-50%) translateY(0)}
|
||
.tablewrap{overflow:auto;border:1px solid var(--line);border-radius:var(--radius);
|
||
background:var(--surface);box-shadow:var(--shadow)}
|
||
table{width:100%;border-collapse:collapse}
|
||
thead th{position:sticky;top:0;background:var(--surface-2);color:var(--muted);
|
||
font-weight:600;font-size:11.5px;letter-spacing:.4px;text-transform:uppercase;
|
||
text-align:left;padding:12px 16px;border-bottom:1px solid var(--line);
|
||
white-space:nowrap;z-index:5}
|
||
tbody td{padding:13px 16px;border-bottom:1px solid #F0ECE3;vertical-align:middle;
|
||
font-size:13px}
|
||
tbody tr{cursor:pointer;transition:background .12s}
|
||
tbody tr:hover{background:#FBF9F4}
|
||
tbody tr:last-child td{border-bottom:0}
|
||
.ip{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:var(--ink);
|
||
font-weight:500;white-space:nowrap}
|
||
.copyable{cursor:pointer;transition:.12s}
|
||
.ip.copyable:hover{color:var(--accent-deep);text-decoration:underline dotted}
|
||
h2.copyable:hover{color:var(--accent-deep)}
|
||
.num{font-variant-numeric:tabular-nums;white-space:nowrap;color:var(--ink-2)}
|
||
.muted{color:var(--muted);font-size:12px}
|
||
tbody td.muted{white-space:nowrap}
|
||
.geo{white-space:nowrap;font-size:12.5px;color:var(--ink-2);line-height:1.55}
|
||
.geo-l1{font-size:12.5px;color:var(--ink-2)}
|
||
.geo-l2{font-size:11.5px;color:var(--muted)}
|
||
|
||
.badge{display:inline-block;padding:3px 10px;border-radius:20px;font-size:11.5px;
|
||
line-height:1.55;white-space:nowrap;vertical-align:middle;font-weight:500}
|
||
.badge-danger{background:var(--red-soft);color:var(--red)}
|
||
.badge-warn{background:var(--amber-soft);color:var(--amber)}
|
||
.badge-info{background:var(--blue-soft);color:var(--blue)}
|
||
.badge-muted{background:#EFEBE2;color:var(--muted)}
|
||
.badge-ok{background:var(--green-soft);color:var(--green)}
|
||
.badge-risk{background:var(--amber-soft);color:var(--amber);
|
||
border:1px dashed #D8B77E}
|
||
.scorecell{display:flex;align-items:center;gap:9px}
|
||
.scorecell span{font-variant-numeric:tabular-nums;color:var(--ink-2);
|
||
min-width:34px;font-weight:600}
|
||
.bar{height:6px;width:66px;border-radius:4px;background:#EDE8DE;overflow:hidden;flex:none}
|
||
.bar i{display:block;height:100%;border-radius:4px;
|
||
background:linear-gradient(90deg,#D98A6B,#B4532F)}
|
||
.reason{color:var(--muted);font-size:12px;max-width:520px}
|
||
.empty{padding:64px 20px;text-align:center;color:var(--muted)}
|
||
a{color:var(--accent);text-decoration:none}
|
||
a:hover{text-decoration:underline}
|
||
.detail-head{display:flex;justify-content:space-between;align-items:flex-start;
|
||
gap:20px;flex-wrap:wrap;margin-bottom:4px}
|
||
.back{display:inline-flex;align-items:center;gap:6px;color:var(--muted);
|
||
font-size:13px;margin-bottom:18px}
|
||
.back:hover{color:var(--accent-deep);text-decoration:none}
|
||
.detail-title{display:flex;align-items:center;gap:12px;flex-wrap:wrap}
|
||
.geo-line{display:flex;gap:6px;flex-wrap:wrap;margin-top:8px}
|
||
.detail-title h2{margin:0;font-size:26px;font-weight:600;letter-spacing:.2px;
|
||
font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:var(--ink)}
|
||
.ring{--pct:0;width:82px;height:82px;border-radius:50%;display:grid;
|
||
place-items:center;flex:none;position:relative;
|
||
background:conic-gradient(var(--accent) calc(var(--pct)*1%),#EDE8DE 0)}
|
||
.ring::after{content:"";position:absolute;inset:8px;border-radius:50%;
|
||
background:var(--surface);box-shadow:inset 0 1px 2px rgba(36,33,25,.05)}
|
||
.ring b{position:relative;z-index:1;font-size:20px;font-variant-numeric:tabular-nums;
|
||
font-family:"Iowan Old Style",Palatino,Georgia,serif;color:var(--ink)}
|
||
.kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(172px,1fr));
|
||
gap:14px;margin:20px 0 18px}
|
||
.kpi{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);
|
||
padding:15px 18px;box-shadow:var(--shadow)}
|
||
.kpi .n{font-family:"Iowan Old Style",Palatino,Georgia,serif;font-size:23px;
|
||
font-weight:600;font-variant-numeric:tabular-nums;line-height:1.2}
|
||
.kpi .k{color:var(--muted);font-size:12px;margin-top:3px}
|
||
.panel{background:var(--surface);border:1px solid var(--line);border-radius:var(--radius);
|
||
padding:18px 20px;margin-top:16px;box-shadow:var(--shadow)}
|
||
.panel h3{margin:0 0 14px;font-size:14px;font-weight:600;color:var(--ink);
|
||
letter-spacing:.2px;display:flex;align-items:center;gap:8px}
|
||
.panel h3::before{content:"";width:3px;height:14px;border-radius:2px;
|
||
background:var(--accent);display:inline-block}
|
||
.reasons{display:grid;gap:9px;margin:0;padding:0;list-style:none}
|
||
.reasons li{display:flex;gap:10px;align-items:flex-start;font-size:13px;color:var(--ink-2)}
|
||
.reasons li::before{content:"";width:6px;height:6px;border-radius:50%;
|
||
background:var(--accent);margin-top:8px;flex:none}
|
||
.rows{display:grid;gap:9px}
|
||
.row{display:grid;grid-template-columns:170px 1fr 66px;gap:12px;align-items:center;
|
||
font-size:12px}
|
||
.row .t{color:var(--ink-2);overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||
.row .b{height:8px;background:#EFEAE0;border-radius:5px;overflow:hidden}
|
||
.row .b i{display:block;height:100%;border-radius:5px;
|
||
background:linear-gradient(90deg,#E3A183,#C15F3C)}
|
||
.row .v{text-align:right;color:var(--ink-2);font-variant-numeric:tabular-nums;
|
||
font-weight:500}
|
||
.tl{display:flex;align-items:flex-end;justify-content:center;gap:4px;
|
||
height:132px;padding-top:8px}
|
||
.tl div{flex:1 1 0;max-width:64px;min-height:2px;border-radius:4px 4px 0 0;
|
||
background:linear-gradient(180deg,#E3A183,#C15F3C);transition:.15s}
|
||
.tl div:hover{background:#D98A6B}
|
||
.tl-axis{display:flex;justify-content:space-between;color:var(--muted);
|
||
font-size:11px;margin-top:7px}
|
||
.s-card{background:var(--surface-2);border:1px solid var(--line);border-radius:12px;
|
||
padding:14px 16px;margin-bottom:10px}
|
||
.s-head{display:flex;align-items:center;gap:8px;color:var(--muted);font-size:12px}
|
||
.s-head .badge{font-size:11px}
|
||
.s-uri{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:13px;
|
||
color:var(--ink);margin:8px 0 10px;word-break:break-all;line-height:1.55}
|
||
.s-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));
|
||
gap:7px 20px;font-size:12px;color:var(--ink-2)}
|
||
.s-grid b,.s-ua b{color:var(--muted);font-weight:500;margin-right:7px}
|
||
.s-ua{font-size:12px;color:var(--ink-2);margin-top:10px;word-break:break-all;line-height:1.5}
|
||
.s-card details{margin-top:10px}
|
||
.s-card summary{cursor:pointer;color:var(--muted);font-size:12px}
|
||
.s-card pre{white-space:pre-wrap;word-break:break-all;background:#F4F1E9;
|
||
border:1px solid var(--line);border-radius:9px;padding:10px 12px;font-size:11.5px;
|
||
color:#5B5648;margin:8px 0 0;line-height:1.7}
|
||
.notes{margin-top:13px;display:grid;gap:7px}
|
||
.notes div{font-size:12.5px;color:var(--ink-2);background:var(--surface-2);
|
||
border:1px solid var(--line);border-radius:9px;padding:9px 12px;line-height:1.55}
|
||
.hint{color:var(--muted);font-size:12px;margin-top:11px;line-height:1.7}
|
||
.monitor-head{display:flex;align-items:center;gap:14px;margin:6px 0 18px}
|
||
.monitor-title{font-size:21px;font-weight:600;color:var(--ink);margin-right:auto}
|
||
.mgrid{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));
|
||
gap:12px;margin-bottom:18px}
|
||
@media (max-width:1100px){.mgrid{grid-template-columns:repeat(2,minmax(0,1fr))}}
|
||
@media (max-width:700px){.mgrid{grid-template-columns:1fr}}
|
||
.mcard{background:var(--surface);border:1px solid var(--line);
|
||
border-radius:14px;padding:15px 17px;box-shadow:0 1px 2px rgba(36,33,25,.03)}
|
||
.mcard h3{font-size:11.5px;color:var(--muted);font-weight:600;
|
||
letter-spacing:.06em;margin:0 0 7px}
|
||
.mbig{font-family:"Iowan Old Style",Palatino,Georgia,serif;font-size:27px;
|
||
color:var(--ink);line-height:1.15;margin-bottom:6px}
|
||
.mbig.mok{color:#3E8E63}.mbig.mbad{color:#C0503A}.mbig.mwarn{color:var(--amber)}
|
||
.mrow{display:flex;justify-content:space-between;gap:10px;font-size:12px;
|
||
color:var(--ink-2);margin-top:5px}
|
||
.mrow span:last-child{color:var(--muted);text-align:right}
|
||
.mtablewrap{overflow:auto;border:1px solid var(--line);border-radius:14px;
|
||
background:var(--surface);box-shadow:0 1px 2px rgba(36,33,25,.03)}
|
||
.mtable{width:100%;border-collapse:collapse;font-size:12.5px}
|
||
.mtable th{color:var(--muted);font-size:11.5px;font-weight:600;text-align:left;
|
||
padding:10px 14px;border-bottom:1px solid var(--line);background:var(--surface-2)}
|
||
.mtable td{padding:9px 14px;border-bottom:1px solid #EFE9DD;color:var(--ink-2)}
|
||
.mtable tr:last-child td{border-bottom:0}
|
||
.nbadge{display:inline-block;padding:2px 9px;border-radius:99px;font-size:11px}
|
||
.nbadge.ok{background:var(--green-soft,#E4EFE7);color:#2E6B4F}
|
||
.nbadge.bad{background:var(--red-soft);color:#93402D}
|
||
.pager{display:flex;align-items:center;justify-content:center;gap:18px;
|
||
margin:18px 0 2px;color:var(--muted);font-size:12.5px;flex-wrap:wrap}
|
||
.pg-total b{color:var(--ink);font-weight:600}
|
||
.pg-nav{display:flex;align-items:center;gap:4px}
|
||
.pg-num,.pg-arrow{min-width:32px;height:32px;padding:0 9px;border:1px solid transparent;
|
||
border-radius:10px;background:transparent;color:var(--ink-2);font-size:13px;
|
||
font-family:inherit;line-height:1;cursor:pointer;transition:all .16s ease}
|
||
.pg-num:hover,.pg-arrow:hover:not(:disabled){background:#F3EAE0;color:var(--ink)}
|
||
.pg-num.active{background:var(--accent);border-color:var(--accent);color:#fff;
|
||
font-weight:600;box-shadow:0 3px 10px rgba(193,95,60,.28)}
|
||
.pg-arrow{border-color:var(--line);background:var(--card);font-size:16px}
|
||
.pg-arrow:hover:not(:disabled){border-color:var(--accent);color:var(--accent);
|
||
background:var(--card)}
|
||
.pg-arrow:disabled{opacity:.32;cursor:default}
|
||
.pg-gap{padding:0 3px;color:#C3B9AA;user-select:none}
|
||
.pg-size{display:flex;align-items:center;gap:6px;color:var(--muted)}
|
||
.pg-size select{appearance:none;-webkit-appearance:none;padding:6px 26px 6px 10px;
|
||
border:1px solid var(--line);border-radius:9px;background:var(--card)
|
||
url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='10' height='6'%3E%3Cpath d='M1 1l4 4 4-4' stroke='%238A8175' fill='none' stroke-width='1.5' stroke-linecap='round'/%3E%3C/svg%3E")
|
||
no-repeat right 8px center;font-size:12px;color:var(--ink-2);cursor:pointer;
|
||
transition:border-color .16s}
|
||
.pg-size select:hover,.pg-size select:focus{border-color:#D3B39F;outline:none}
|
||
@media (max-width:640px){
|
||
.pager{gap:10px;padding:0 2px}
|
||
.pg-total{order:3;width:100%;text-align:center}
|
||
}
|
||
.advice{display:flex;gap:12px;align-items:flex-start;padding:15px 18px;
|
||
border-radius:12px;border:1px solid #EBC9BD;background:var(--red-soft);
|
||
color:#7E3020;font-size:13px;line-height:1.6}
|
||
.advice.warn{border-color:#E7D2A9;background:var(--amber-soft);color:#6E4E14}
|
||
.expandable{cursor:pointer;border-bottom:1px dashed #C9BFA9;transition:.12s}
|
||
.expandable:hover{color:var(--accent-deep);border-bottom-color:#C98C72}
|
||
.linkbtn{background:var(--surface);border:1px solid var(--line);color:var(--accent-deep);
|
||
border-radius:9px;padding:6px 12px;font-size:12px;cursor:pointer;margin-top:10px;
|
||
transition:.15s;box-shadow:0 1px 2px rgba(36,33,25,.04)}
|
||
.linkbtn:hover{background:var(--accent-soft);border-color:#DCB7A5}
|
||
.modal{position:fixed;inset:0;background:rgba(36,33,25,.34);display:none;z-index:100;
|
||
align-items:center;justify-content:center;padding:26px;backdrop-filter:blur(3px)}
|
||
.modal.open{display:flex}
|
||
.modal-box{width:min(980px,100%);max-height:82vh;display:flex;flex-direction:column;
|
||
background:var(--surface);border:1px solid var(--line-strong);border-radius:16px;
|
||
box-shadow:0 30px 80px rgba(36,33,25,.22)}
|
||
.modal-head{display:flex;justify-content:space-between;align-items:center;
|
||
padding:15px 20px;border-bottom:1px solid var(--line)}
|
||
.modal-head h4{margin:0;font-size:14px;color:var(--ink);font-weight:600}
|
||
.modal-close{background:transparent;border:0;color:var(--muted);font-size:22px;
|
||
cursor:pointer;line-height:1;padding:0 4px}
|
||
.modal-close:hover{color:var(--ink)}
|
||
.modal-body{padding:16px 20px;overflow:auto}
|
||
.modal-body pre{margin:0;white-space:pre-wrap;word-break:break-all;font-size:12.5px;
|
||
line-height:1.8;color:var(--ink-2);font-family:ui-monospace,Menlo,monospace}
|
||
.skeleton{padding:48px;text-align:center;color:var(--muted)}
|
||
@media (max-width:860px){
|
||
main{padding:18px 16px}
|
||
header.top{padding:14px 18px}
|
||
.reason{display:none}
|
||
.stats{grid-template-columns:repeat(2,1fr)}
|
||
.sortgroup{width:100%}
|
||
.select{flex:1}
|
||
}
|
||
@media (max-width:560px){
|
||
.chips{width:100%;justify-content:space-between}
|
||
.chip{flex:1;text-align:center}
|
||
.stats{grid-template-columns:1fr}
|
||
}
|
||
</style></head><body>
|
||
|
||
<header class="top">
|
||
<div class="brand">
|
||
<div class="logo">IP</div>
|
||
<div><h1>恶意 IP 清单</h1><div class="sub">特征 + 行为融合检测 · Graylog · F5 访问日志</div></div>
|
||
</div>
|
||
<div class="head-right">
|
||
<span class="status-dot" id="statusDot" title="Graylog 连接状态"></span>
|
||
<div class="meta" id="meta">加载中…</div>
|
||
</div>
|
||
</header>
|
||
|
||
<main>
|
||
<section id="listView">
|
||
<div class="stats" id="stats"></div>
|
||
<div class="toolbar">
|
||
<input type="search" id="search" placeholder="搜索 IP 或判定原因，例如 42.92 或 actuator">
|
||
<label class="switch"><input type="checkbox" id="auto" checked> 自动刷新</label>
|
||
<div class="actions">
|
||
<div class="menu-wrap">
|
||
<button class="btn" id="copyBtn" aria-haspopup="true">复制 IP <span class="caret">▾</span></button>
|
||
<div class="menu" id="copyMenu">
|
||
<button data-copy="comma">逗号分隔</button>
|
||
<button data-copy="newline">换行分隔</button>
|
||
</div>
|
||
</div>
|
||
<div class="menu-wrap">
|
||
<button class="btn primary" id="exportBtn" aria-haspopup="true">导出 <span class="caret">▾</span></button>
|
||
<div class="menu" id="exportMenu">
|
||
<button data-export="csv">CSV（完整字段）</button>
|
||
<button data-export="txt">仅 IP 列表（换行）</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="tablewrap">
|
||
<table>
|
||
<thead><tr>
|
||
<th>IP 地址</th><th>位置 / 归属</th><th>来源</th><th>评分</th><th>请求数</th><th>频率</th>
|
||
<th>发现时间</th><th>最近活动</th><th>判定理由</th><th></th>
|
||
</tr></thead>
|
||
<tbody id="rows"></tbody>
|
||
</table>
|
||
<div class="empty" id="empty" hidden>没有匹配的记录</div>
|
||
</div>
|
||
<div class="pager" id="pager"></div>
|
||
<div class="hint" id="listHint"></div>
|
||
</section>
|
||
|
||
<section id="detailView" hidden></section>
|
||
<section id="monitorView" hidden></section>
|
||
</main>
|
||
|
||
<div class="toast" id="toast"></div>
|
||
<div class="modal" id="modal">
|
||
<div class="modal-box">
|
||
<div class="modal-head"><h4 id="modalTitle">详情</h4>
|
||
<button class="modal-close" id="modalClose" title="关闭">×</button></div>
|
||
<div class="modal-body"><pre id="modalBody"></pre></div>
|
||
</div>
|
||
</div>
|
||
|
||
<script>
|
||
window.addEventListener('error', function (event) {
|
||
var meta = document.getElementById('meta');
|
||
if (meta) meta.textContent = '前端脚本错误: ' + (event.message || 'unknown');
|
||
});
|
||
window.addEventListener('unhandledrejection', function (event) {
|
||
var meta = document.getElementById('meta');
|
||
if (meta) meta.textContent = '请求失败: ' + ((event.reason && event.reason.message) || event.reason || 'unknown');
|
||
});
|
||
const BASE = "__BASE__";
|
||
const $ = (sel, root = document) => root.querySelector(sel);
|
||
const flagOf = code => {
|
||
if (!code || code.length !== 2 || code === 'ZZ') return '';
|
||
return String.fromCodePoint(...[...code.toUpperCase()]
|
||
.map(ch => 0x1F1E6 + ch.charCodeAt(0) - 65));
|
||
};
|
||
const esc = v => String(v == null ? '' : v).replace(/[&<>"']/g, c => (
|
||
{'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
||
const fmt = n => Number(n || 0).toLocaleString('en-US');
|
||
const truncate = (text, limit) => {
|
||
const value = String(text == null ? '' : text);
|
||
return value.length > limit ? value.slice(0, limit - 1) + '…' : value;
|
||
};
|
||
const TIER = {
|
||
malicious: ['恶意', 'badge-danger'],
|
||
suspicious: ['可疑', 'badge-warn'],
|
||
automation: ['自动化', 'badge-info'],
|
||
observe: ['观察', 'badge-muted']
|
||
};
|
||
const SOURCE = {
|
||
feature: ['特征', 'badge-danger'],
|
||
behavior: ['行为', 'badge-info'],
|
||
both: ['双来源', 'badge-warn'],
|
||
};
|
||
const TYPE_LABEL = {
|
||
cloud: '云服务商', idc: 'IDC/机房', mobile: '移动网络',
|
||
broadband: '宽带', education: '教育网', enterprise: '企业/政府',
|
||
unknown: '未知',
|
||
};
|
||
const SHARE_LABEL = {
|
||
mobile: '移动网络', broadband: '宽带 ISP', education: '教育网',
|
||
hosting: '云主机/机房', unknown: '未知网络',
|
||
};
|
||
const RULE_TITLES = {
|
||
probe_java: 'Spring/中间件探测', probe_config: '配置/凭证探测', probe_cms: 'CMS 探测',
|
||
probe_iot: 'IoT/路由器探测', probe_devops: 'DevOps/监控探测', probe_webshell: 'WebShell/上传探测',
|
||
exploit_payload: '注入/穿越载荷', login_attack: '登录失败尝试', login_burst: '登录接口高频',
|
||
bad_ua: '扫描器 UA', bot_ua: '爬虫 UA', not_found: '404 探测'
|
||
};
|
||
const state = { q: '', items: [], page: 1, pageSize: 20, total: 0, pages: 1,
|
||
timer: null, inDetail: false, inMonitor: false, detailSamples: [] };
|
||
|
||
const modal = $('#modal');
|
||
function openModal(title, text) {
|
||
if (!modal) return;
|
||
$('#modalTitle').textContent = title;
|
||
$('#modalBody').textContent = text || '(空)';
|
||
modal.classList.add('open');
|
||
}
|
||
function closeModal() { if (modal) modal.classList.remove('open'); }
|
||
if (modal) {
|
||
$('#modalClose').addEventListener('click', closeModal);
|
||
modal.addEventListener('click', e => { if (e.target === modal) closeModal(); });
|
||
}
|
||
document.addEventListener('keydown', e => { if (e.key === 'Escape') closeModal(); });
|
||
document.addEventListener('click', e => {
|
||
const el = e.target.closest('[data-mfield]');
|
||
if (!el) return;
|
||
const sample = (state.detailSamples || [])[Number(el.dataset.idx)] || {};
|
||
const field = el.dataset.mfield;
|
||
openModal(field === 'message' ? '原始日志行' : field.toUpperCase(), sample[field] || '');
|
||
});
|
||
|
||
function localTime(iso) {
|
||
if (!iso) return '-';
|
||
const d = new Date(iso);
|
||
if (isNaN(d)) return iso;
|
||
const p = n => String(n).padStart(2, '0');
|
||
return `${p(d.getMonth()+1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}`;
|
||
}
|
||
function perMinute(total, window) {
|
||
if (!window) return '-';
|
||
return (Number(total) / window * 60).toFixed(1);
|
||
}
|
||
async function jget(url) {
|
||
const r = await fetch(url, { cache: 'no-store' });
|
||
if (!r.ok) throw new Error('HTTP ' + r.status);
|
||
return r.json();
|
||
}
|
||
|
||
async function loadStatus() {
|
||
const dot = $('#statusDot');
|
||
try {
|
||
const s = await jget(BASE + '/api/status');
|
||
dot.className = 'status-dot ' + (s.graylog || 'unknown');
|
||
dot.title = s.graylog === 'ok' ? 'Graylog 已连接'
|
||
: s.graylog === 'error' ? 'Graylog 连接失败'
|
||
: s.graylog === 'stale' ? 'Graylog 状态过期'
|
||
: 'Graylog 状态未知';
|
||
} catch (err) {
|
||
dot.className = 'status-dot error';
|
||
dot.title = '后端状态接口不可用';
|
||
}
|
||
}
|
||
|
||
async function loadStats() {
|
||
try {
|
||
const s = await jget(BASE + '/api/stats');
|
||
$('#stats').innerHTML = `
|
||
<div class="stat hot"><div class="n">${fmt(s.fused_total)}</div><div class="k">应封禁（融合）</div></div>
|
||
<div class="stat warm"><div class="n">${fmt(s.fused_new_24h)}</div><div class="k">今日新增</div></div>
|
||
<div class="stat cool"><div class="n">${fmt(s.fused_both)}</div><div class="k">双来源</div></div>
|
||
<div class="stat"><div class="n">${fmt(s.fused_behavior_only)}</div><div class="k">行为新发现</div></div>`;
|
||
$('#meta').innerHTML = `最近更新 <b>${localTime(s.updated_at)}</b><br>
|
||
<a href="#/monitor">监控</a> ·
|
||
<a href="${BASE}/api/ips" target="_blank">JSON</a> ·
|
||
<a href="${BASE}/api/metrics" target="_blank">metrics</a>`;
|
||
} catch (err) {
|
||
$('#meta').textContent = '统计加载失败: ' + err.message;
|
||
}
|
||
}
|
||
|
||
async function loadList() {
|
||
try {
|
||
const params = new URLSearchParams({
|
||
page: state.page, page_size: state.pageSize,
|
||
});
|
||
if (state.q.trim()) params.set('q', state.q.trim());
|
||
const data = await jget(BASE + '/api/ips?' + params.toString());
|
||
state.total = data.total || 0;
|
||
state.pages = data.pages || 1;
|
||
if (state.page > state.pages) { state.page = state.pages; return loadList(); }
|
||
state.items = data.items || [];
|
||
render();
|
||
renderPager();
|
||
$('#listHint').textContent =
|
||
`按发现时间倒序（最新在前）· 每 30 秒更新`;
|
||
} catch (err) {
|
||
$('#rows').innerHTML = `<tr><td colspan="10" class="empty">加载失败：${esc(err.message)}</td></tr>`;
|
||
}
|
||
}
|
||
|
||
async function fetchAllItems() {
|
||
const params = new URLSearchParams({ limit: 2000 });
|
||
if (state.q.trim()) params.set('q', state.q.trim());
|
||
const data = await jget(BASE + '/api/ips?' + params.toString());
|
||
return data.items || [];
|
||
}
|
||
|
||
function filteredItems() {
|
||
return state.items;
|
||
}
|
||
|
||
function pageWindow(current, total) {
|
||
const picked = [];
|
||
const add = n => { if (n >= 1 && n <= total && !picked.includes(n)) picked.push(n); };
|
||
add(1);
|
||
for (let n = current - 2; n <= current + 2; n += 1) add(n);
|
||
add(total);
|
||
picked.sort((a, b) => a - b);
|
||
const out = [];
|
||
picked.forEach((n, index) => {
|
||
if (index > 0 && n - picked[index - 1] > 1) out.push('gap');
|
||
out.push(n);
|
||
});
|
||
return out;
|
||
}
|
||
|
||
function renderPager() {
|
||
const pager = $('#pager');
|
||
if (!pager) return;
|
||
if (state.pages <= 1) { pager.innerHTML = ''; return; }
|
||
const nums = pageWindow(state.page, state.pages).map(n =>
|
||
n === 'gap'
|
||
? '<span class="pg-gap">…</span>'
|
||
: `<button class="pg-num${n === state.page ? ' active' : ''}" data-page="${n}">${n}</button>`
|
||
).join('');
|
||
const sizes = [20, 50, 100].map(n =>
|
||
`<option value="${n}" ${n === state.pageSize ? 'selected' : ''}>${n}</option>`
|
||
).join('');
|
||
pager.innerHTML = `
|
||
<div class="pg-total">共 <b>${fmt(state.total)}</b> 条</div>
|
||
<div class="pg-nav">
|
||
<button class="pg-arrow" data-page="${state.page - 1}" ${state.page <= 1 ? 'disabled' : ''} aria-label="上一页">‹</button>
|
||
${nums}
|
||
<button class="pg-arrow" data-page="${state.page + 1}" ${state.page >= state.pages ? 'disabled' : ''} aria-label="下一页">›</button>
|
||
</div>
|
||
<label class="pg-size">每页
|
||
<select id="pageSize">${sizes}</select> 条
|
||
</label>`;
|
||
}
|
||
|
||
let toastTimer = null;
|
||
function toast(message) {
|
||
const el = $('#toast');
|
||
el.textContent = message;
|
||
el.classList.add('show');
|
||
clearTimeout(toastTimer);
|
||
toastTimer = setTimeout(() => el.classList.remove('show'), 2200);
|
||
}
|
||
|
||
function legacyCopy(text) {
|
||
const area = document.createElement('textarea');
|
||
area.value = text;
|
||
area.style.position = 'fixed';
|
||
area.style.opacity = '0';
|
||
area.style.left = '-9999px';
|
||
document.body.appendChild(area);
|
||
area.select();
|
||
document.execCommand('copy');
|
||
document.body.removeChild(area);
|
||
}
|
||
|
||
async function copyText(text, message) {
|
||
try {
|
||
if (navigator.clipboard && window.isSecureContext) {
|
||
await navigator.clipboard.writeText(text);
|
||
} else {
|
||
legacyCopy(text);
|
||
}
|
||
toast(message);
|
||
} catch (err) {
|
||
legacyCopy(text);
|
||
toast(message + '（兼容模式）');
|
||
}
|
||
}
|
||
|
||
async function copyIps(separator) {
|
||
const rows = await fetchAllItems();
|
||
if (!rows.length) { toast('当前列表为空'); return; }
|
||
const newline = separator === 'newline';
|
||
const text = rows.map(item => item.ip).join(newline ? '\n' : ',');
|
||
const label = newline ? '换行分隔' : '逗号分隔';
|
||
try {
|
||
if (navigator.clipboard && window.isSecureContext) {
|
||
await navigator.clipboard.writeText(text);
|
||
} else {
|
||
legacyCopy(text);
|
||
}
|
||
toast(`已复制 ${rows.length} 个 IP（${label}）`);
|
||
} catch (err) {
|
||
legacyCopy(text);
|
||
toast(`已复制 ${rows.length} 个 IP（${label}，兼容模式）`);
|
||
}
|
||
}
|
||
|
||
function downloadFile(content, filename, mime) {
|
||
const blob = new Blob([content], { type: mime });
|
||
const link = document.createElement('a');
|
||
link.href = URL.createObjectURL(blob);
|
||
link.download = filename;
|
||
document.body.appendChild(link);
|
||
link.click();
|
||
document.body.removeChild(link);
|
||
URL.revokeObjectURL(link.href);
|
||
}
|
||
|
||
async function exportList(format) {
|
||
const rows = await fetchAllItems();
|
||
if (!rows.length) { toast('当前列表为空'); return; }
|
||
const stamp = new Date().toISOString().slice(0, 10);
|
||
if (format === 'txt') {
|
||
const text = rows.map(item => item.ip).join('\n') + '\n';
|
||
downloadFile(text, 'malicious-ips-' + stamp + '.txt',
|
||
'text/plain;charset=utf-8');
|
||
toast(`已导出 ${rows.length} 个 IP（换行分隔）`);
|
||
return;
|
||
}
|
||
const quote = value => '"' + String(value == null ? '' : value)
|
||
.replace(/"/g, '""') + '"';
|
||
const header = ['ip', 'total', 'first_seen', 'last_malicious_at',
|
||
'feature_score', 'behavior_score', 'sources', 'asn', 'asn_brand',
|
||
'asn_type', 'reasons'];
|
||
const lines = [header.join(',')];
|
||
rows.forEach(item => {
|
||
lines.push([
|
||
item.ip, item.total, item.first_seen || '', item.last_malicious_at || '',
|
||
item.feature_score ?? '', item.behavior_score ?? '', item.sources || '',
|
||
item.asn || '', item.asn_brand || '', item.asn_type || '',
|
||
(item.reasons || []).join('; ')
|
||
].map(quote).join(','));
|
||
});
|
||
downloadFile('\ufeff' + lines.join('\n'), 'malicious-ips-' + stamp + '.csv',
|
||
'text/csv;charset=utf-8');
|
||
toast(`已导出 ${rows.length} 条到 CSV`);
|
||
}
|
||
|
||
function closeMenus() {
|
||
document.querySelectorAll('.menu.open').forEach(menu => menu.classList.remove('open'));
|
||
}
|
||
function toggleMenu(id) {
|
||
const menu = $('#' + id);
|
||
const willOpen = !menu.classList.contains('open');
|
||
closeMenus();
|
||
if (willOpen) menu.classList.add('open');
|
||
}
|
||
document.addEventListener('click', e => {
|
||
if (!e.target.closest('.menu-wrap')) closeMenus();
|
||
});
|
||
document.addEventListener('keydown', e => {
|
||
if (e.key === 'Escape') closeMenus();
|
||
});
|
||
|
||
function render() {
|
||
const rows = filteredItems();
|
||
$('#empty').hidden = rows.length > 0;
|
||
$('#rows').innerHTML = rows.map(item => {
|
||
const src = SOURCE[item.sources] || ['特征', 'badge-danger'];
|
||
const reasons = (item.reasons || []).slice(0, 2).join('；') || '-';
|
||
const scoreTitle = `特征分 ${item.feature_score ?? '-'} / 行为分 ${item.behavior_score ?? '-'}`;
|
||
const geoFlag = flagOf(item.country_code);
|
||
const geoName = [item.country_name, item.region].filter(Boolean).join(' · ');
|
||
const geoTitle = [item.country_name, item.region, item.city]
|
||
.filter(Boolean).join(' · ') || '未知';
|
||
return `<tr data-ip="${esc(item.ip)}">
|
||
<td class="ip copyable" data-copyip="${esc(item.ip)}" title="点击复制 IP">${esc(item.ip)}</td>
|
||
<td class="geo">
|
||
<div class="geo-l1" title="${esc(geoTitle)}">${geoFlag ? geoFlag + ' ' : ''}${
|
||
esc(geoName || '-')}</div>
|
||
<div class="geo-l2" title="AS${esc(item.asn || '-')} · ${esc(item.asn_org || '-')} · ${esc(TYPE_LABEL[item.asn_type] || '未知')}">${
|
||
item.asn_brand ? esc(item.asn_icon + ' ' + item.asn_brand) : '❓ 未知'}</div>
|
||
</td>
|
||
<td><span class="badge ${src[1]}">${src[0]}</span></td>
|
||
<td title="${esc(scoreTitle)}"><div class="scorecell"><span>${Number(item.score).toFixed(0)}</span>
|
||
<div class="bar"><i style="width:${Math.min(100, item.score)}%"></i></div></div></td>
|
||
<td class="num">${fmt(item.total)}</td>
|
||
<td class="num">${perMinute(item.total, item.window)} /分</td>
|
||
<td class="muted">${localTime(item.first_seen)}</td>
|
||
<td class="muted">${item.last_malicious_at ? localTime(item.last_malicious_at) : '-'}</td>
|
||
<td class="reason" title="${esc(reasons)}">${esc(reasons)}</td>
|
||
<td class="muted">›</td>
|
||
</tr>`;
|
||
}).join('');
|
||
|
||
$('#rows').querySelectorAll('tr[data-ip]').forEach(tr => {
|
||
tr.addEventListener('click', e => {
|
||
if (e.target.closest('[data-copyip]')) return;
|
||
location.hash = '#/ip/' + encodeURIComponent(tr.dataset.ip);
|
||
});
|
||
});
|
||
}
|
||
|
||
function bars(items, valueKey, labelKey) {
|
||
const max = Math.max(1, ...items.map(x => Number(x[valueKey] || 0)));
|
||
return items.map(x => `<div class="row">
|
||
<div class="t" title="${esc(x[labelKey])}">${esc(x[labelKey])}</div>
|
||
<div class="b"><i style="width:${Math.round(100 * x[valueKey] / max)}%"></i></div>
|
||
<div class="v">${fmt(x[valueKey])}</div>
|
||
</div>`).join('');
|
||
}
|
||
|
||
function timeline(items) {
|
||
if (!items || !items.length) return '<div class="muted">无时间分布数据</div>';
|
||
const max = Math.max(1, ...items.map(x => Number(x.count || 0)));
|
||
const barsHtml = items.map(x => `<div style="height:${Math.max(2, Math.round(100 * x.count / max))}%"
|
||
title="${esc(localTime(x.timestamp))} · ${fmt(x.count)} 次"></div>`).join('');
|
||
return `<div class="tl">${barsHtml}</div>
|
||
<div class="tl-axis"><span>${esc(localTime(items[0].timestamp))}</span>
|
||
<span>${esc(localTime(items[items.length - 1].timestamp))}</span></div>`;
|
||
}
|
||
|
||
function advice(item) {
|
||
if (item.sources === 'both') {
|
||
return `<div class="advice"><b>建议：封禁（特征 + 行为双来源）。</b>
|
||
该 IP 命中明确攻击特征，且行为模式与攻击一致，置信度最高，可直接下发封禁。</div>`;
|
||
}
|
||
if (item.sources === 'behavior') {
|
||
return `<div class="advice"><b>建议：封禁（行为检测判定）。</b>
|
||
行为特征高度符合自动化扫描/攻击模式，已进入融合封禁名单。</div>`;
|
||
}
|
||
return `<div class="advice"><b>建议：封禁（特征检测）。</b>
|
||
该 IP 命中明确攻击特征，可直接下发到 F5 / AD 黑名单。</div>`;
|
||
}
|
||
|
||
async function showDetail(ip) {
|
||
state.inDetail = true;
|
||
state.inMonitor = false;
|
||
$('#monitorView').hidden = true;
|
||
$('#listView').hidden = true;
|
||
const view = $('#detailView');
|
||
view.hidden = false;
|
||
view.innerHTML = '<div class="skeleton">正在加载证据…</div>';
|
||
document.title = ip + ' · 恶意 IP 清单';
|
||
loadStats();
|
||
try {
|
||
const data = await jget(BASE + '/api/ip/' + encodeURIComponent(ip));
|
||
const item = data.item;
|
||
const ev = data.evidence || {};
|
||
const tier = TIER[item.tier] || ['未知', 'badge-muted'];
|
||
const src = SOURCE[item.sources] || ['特征', 'badge-danger'];
|
||
const window = item.window || ev.window || 0;
|
||
const windowText = window >= 86400 ? (window / 86400).toFixed(1) + ' 天'
|
||
: window >= 3600 ? (window / 3600).toFixed(1) + ' 小时'
|
||
: Math.round(window / 60) + ' 分钟';
|
||
const rules = Object.entries(item.rules || {})
|
||
.map(([name, count]) => ({ name: RULE_TITLES[name] || name, count }));
|
||
const ruleRows = rules.length
|
||
? `<div class="rows">${bars(rules, 'count', 'name')}</div>`
|
||
: '<div class="muted">无规则命中明细</div>';
|
||
const extraNotes = (item.reasons || []).filter(reason =>
|
||
!rules.some(rule => String(reason).startsWith(rule.name)));
|
||
const dimension = (items, empty) => (items || []).length
|
||
? `<div class="rows">${bars(items.map(x => ({name: x.term, count: x.count})), 'count', 'name')}</div>`
|
||
: `<div class="muted">${empty}</div>`;
|
||
const statusRows = dimension(ev.statuses, '无状态码分布');
|
||
const methodRows = dimension(ev.methods, '无方法分布');
|
||
const hostRows = dimension(ev.hosts, '无域名分布');
|
||
const statusBadge = value => {
|
||
const text = String(value || '').trim();
|
||
if (!text) return '<span class="badge badge-muted">无状态</span>';
|
||
const cls = text.startsWith('2') ? 'badge-ok'
|
||
: text.startsWith('3') ? 'badge-info' : 'badge-danger';
|
||
return `<span class="badge ${cls}">${esc(text)}</span>`;
|
||
};
|
||
state.detailSamples = ev.samples || [];
|
||
const ruleTitle = name => (ev.rule_titles || {})[name] || name;
|
||
const sampleCards = (ev.samples || []).map((s, idx) => {
|
||
const ruleBadges = (s.rules || []).map(name =>
|
||
`<span class="badge badge-warn" title="${esc(name)}">${esc(ruleTitle(name))}</span>`
|
||
).join('');
|
||
const client = [s.client_ip, s.client_port].filter(Boolean).join(':');
|
||
const server = [s.server_ip, s.server_port].filter(Boolean).join(':');
|
||
const geo = [s.client_ip_country_code, s.client_ip_city_name]
|
||
.filter(x => x && x !== 'N/A').join(' / ');
|
||
const facts = [
|
||
`<div><b>客户端</b><span class="mono">${esc(client || '-')}</span></div>`,
|
||
`<div><b>服务端</b><span class="mono">${esc(server || '-')}</span></div>`,
|
||
s.host ? `<div><b>Host</b><span class="mono expandable" data-idx="${idx}" data-mfield="host">${esc(s.host)}</span></div>` : '',
|
||
s.bigip ? `<div><b>VIP</b><span class="mono">${esc(s.bigip)}${s.vs ? ' · ' + esc(s.vs) : ''}</span></div>` : '',
|
||
geo ? `<div><b>地区</b>${esc(geo)}</div>` : '',
|
||
s.rt ? `<div><b>响应</b>${esc(s.rt)} s</div>` : '',
|
||
s.req_len ? `<div><b>请求长度</b>${esc(s.req_len)} B</div>` : '',
|
||
s.referer ? `<div><b>Referer</b><span class="mono expandable" data-idx="${idx}" data-mfield="referer">${esc(truncate(s.referer, 66))}</span></div>` : '',
|
||
s.source ? `<div><b>采集来源</b><span class="mono">${esc(s.source)}</span></div>` : '',
|
||
].join('');
|
||
return `<div class="s-card">
|
||
<div class="s-head">
|
||
<span class="mono">${esc(localTime(s.timestamp))}</span>
|
||
<span class="badge badge-info">${esc(s.method || '?')}</span>
|
||
${statusBadge(s.status)}
|
||
${ruleBadges}
|
||
</div>
|
||
<div class="s-uri mono"><span class="expandable" data-idx="${idx}" data-mfield="uri">${esc(s.uri || '-')}</span>${
|
||
s.query ? '<span class="muted expandable" data-idx="' + idx + '" data-mfield="query"> ?'
|
||
+ esc(truncate(s.query, 60)) + '</span>' : ''}</div>
|
||
<div class="s-grid">${facts}</div>
|
||
<div class="s-ua"><b>UA</b><span class="mono expandable" data-idx="${idx}" data-mfield="ua">${
|
||
esc(truncate(s.ua || '-', 90))}</span></div>
|
||
${s.message ? `<button class="linkbtn" data-idx="${idx}" data-mfield="message">查看完整原始日志行</button>` : ''}
|
||
</div>`;
|
||
}).join('');
|
||
const sampleNote = (ev.samples || []).length
|
||
? `按攻击类型抽样：每种类型保留最近 3 条，共 ${(ev.samples || []).length} 条，不是该 IP 的全部日志。`
|
||
: `暂无样本${ev.samples_error ? '：' + esc(ev.samples_error) : ''}${
|
||
ev.available ? '' : '（证据服务不可用：' + esc(ev.reason || ev.error || '未知原因') + '）'}`;
|
||
|
||
const behavior = item.behavior;
|
||
const behaviorPanel = behavior ? `<div class="panel">
|
||
<h3>行为检测画像</h3>
|
||
<ul class="reasons">
|
||
<li>建议：${esc(behavior.verdict || '-')}（行为分 ${Number(behavior.score || 0).toFixed(0)}）</li>
|
||
${behavior.reason_text ? `<li>${esc(behavior.reason_text)}</li>` : ''}
|
||
<li>出口：${behavior.asn_brand ? esc((behavior.asn_icon || '') + ' ' + behavior.asn_brand) : '未知'}${
|
||
behavior.asn ? ' · AS' + esc(behavior.asn) : ''}（${esc(TYPE_LABEL[behavior.asn_type] || '未知')}）</li>
|
||
<li>24h 请求 ${fmt(behavior.total)} 次 · 失败率 ${Number(behavior.pct4 || 0).toFixed(0)}% ·
|
||
不同路径 ${behavior.uri_card || 0} 个 · UA ${behavior.ua_card || 0} 种</li>
|
||
</ul>
|
||
${behavior.asn_org ? `<div class="hint">归属组织：${esc(behavior.asn_org)}</div>` : ''}
|
||
${behavior.share_note ? `<div class="hint">${esc(behavior.share_note)}</div>` : ''}
|
||
${(behavior.signals || []).length ? `<div class="hint">信号：${behavior.signals.map(esc).join('；')}</div>` : ''}
|
||
</div>` : '';
|
||
view.innerHTML = `
|
||
<a class="back" href="#/">← 返回名单</a>
|
||
<div class="detail-head">
|
||
<div>
|
||
<div class="detail-title">
|
||
<h2 class="copyable" data-copyip="${esc(item.ip)}" title="点击复制 IP">${esc(item.ip)}</h2>
|
||
<span class="badge ${src[1]}">${src[0]}</span>
|
||
</div>
|
||
<div class="geo-line">
|
||
<span class="badge badge-muted">
|
||
${flagOf(item.country_code)} ${esc(item.country_name || '未知地区')}${
|
||
item.region ? ' · ' + esc(item.region) : ''}${
|
||
item.city ? ' · ' + esc(item.city) : ''}
|
||
</span>
|
||
<span class="badge badge-muted" title="${esc(item.asn_org || '未知')} · ${esc(TYPE_LABEL[item.asn_type] || '未知')}">
|
||
${item.asn_brand ? esc(item.asn_icon + ' ' + item.asn_brand) : '❓ 未知'}${
|
||
item.asn ? ' · AS' + esc(item.asn) : ''}
|
||
</span>
|
||
</div>
|
||
<div class="muted" style="margin-top:6px">
|
||
发现时间 ${localTime(item.first_seen)} · 最近活动 ${localTime(item.last_seen)}
|
||
· 观测窗口 ${windowText}
|
||
${item.evidence_at ? ' · 证据生成 ' + esc(localTime(item.evidence_at)) : ''}
|
||
</div>
|
||
</div>
|
||
<div class="ring" style="--pct:${Math.min(100, item.score)}"><b>${Number(item.score).toFixed(0)}</b></div>
|
||
</div>
|
||
|
||
<div class="kpis">
|
||
<div class="kpi"><div class="n">${fmt(item.total)}</div><div class="k">请求总量</div></div>
|
||
<div class="kpi"><div class="n">${perMinute(item.total, window)}</div><div class="k">平均频率（次/分）</div></div>
|
||
<div class="kpi"><div class="n">${fmt(rules.reduce((a, r) => a + r.count, 0))}</div><div class="k">规则命中次数</div></div>
|
||
<div class="kpi"><div class="n">${fmt(item.hits)}</div><div class="k">被检测到轮次</div></div>
|
||
</div>
|
||
|
||
${advice(item)}
|
||
${behaviorPanel}
|
||
${ev.stale ? `<div class="advice warn"><b>注意：</b>${esc(ev.reason || 'Graylog 中该时段日志已过期，以下为判定时的快照')}</div>` : ''}
|
||
|
||
<div class="panel">
|
||
<h3>判定依据</h3>
|
||
${ruleRows}
|
||
${extraNotes.length ? `<div class="notes">${extraNotes.map(note =>
|
||
`<div>${esc(note)}</div>`).join('')}</div>` : ''}
|
||
<div class="hint">计数为该观测窗口内命中次数，不是全量历史。</div>
|
||
</div>
|
||
|
||
<div class="panel">
|
||
<h3>请求频率分布</h3>
|
||
${ev.available ? timeline(ev.timeline) : '<div class="muted">不可用</div>'}
|
||
<div class="hint">${esc(ev.interval || '')} 粒度
|
||
${ev.peak ? ' · 峰值 ' + esc(localTime(ev.peak.timestamp)) + '（' + fmt(ev.peak.count) + ' 次）' : ''}
|
||
${ev.active_buckets ? ' · ' + ev.active_buckets + ' 个活跃时段' : ''}
|
||
· 用于判断是持续扫描还是短时爆发。</div>
|
||
</div>
|
||
|
||
<div class="panel">
|
||
<h3>状态码分布</h3>
|
||
${statusRows}
|
||
</div>
|
||
|
||
<div class="panel">
|
||
<h3>请求方法分布</h3>
|
||
${methodRows}
|
||
</div>
|
||
|
||
<div class="panel">
|
||
<h3>访问域名分布</h3>
|
||
${hostRows}
|
||
</div>
|
||
|
||
<div class="panel">
|
||
<h3>证据样本</h3>
|
||
${sampleCards || '<div class="muted">暂无样本</div>'}
|
||
<div class="hint">${esc(sampleNote)} 点击虚线字段可在弹窗中查看完整内容，Esc 关闭。</div>
|
||
</div>`;
|
||
} catch (err) {
|
||
view.innerHTML = `<a class="back" href="#/">← 返回名单</a>
|
||
<div class="empty">加载失败：${esc(err.message)}</div>`;
|
||
}
|
||
}
|
||
|
||
function showList() {
|
||
state.inDetail = false;
|
||
state.inMonitor = false;
|
||
document.title = '恶意 IP 清单';
|
||
$('#monitorView').hidden = true;
|
||
$('#detailView').hidden = true;
|
||
$('#detailView').innerHTML = '';
|
||
$('#listView').hidden = false;
|
||
loadStatus();
|
||
loadStats();
|
||
loadList();
|
||
}
|
||
|
||
|
||
function monitorCard(title, value, rows, valueClass) {
|
||
const body = rows.map(row =>
|
||
`<div class="mrow"><span>${esc(row[0])}</span><span>${esc(row[1])}</span></div>`
|
||
).join('');
|
||
return `<div class="mcard"><h3>${esc(title)}</h3>
|
||
<div class="mbig ${valueClass || ''}">${esc(value)}</div>${body}</div>`;
|
||
}
|
||
|
||
function monitorHtml(m) {
|
||
const d = m.detect, b = m.behavior, p = m.push, l = m.list, bf = m.backfill;
|
||
const stateText = m.graylog === 'ok' ? '正常' : m.graylog === 'error' ? '失败'
|
||
: m.graylog === 'stale' ? '过期' : '未知';
|
||
const stateCls = m.graylog === 'ok' ? 'mok' : m.graylog === 'error' ? 'mbad'
|
||
: 'mwarn';
|
||
const ratio = p.success_ratio_24h == null ? '-' :
|
||
(p.success_ratio_24h * 100).toFixed(1) + '%';
|
||
const cards = [
|
||
monitorCard('Graylog 连接', stateText, [
|
||
['最近成功', d.last_success ? localTime(d.last_success) : '-'],
|
||
['距上次成功', d.gap_seconds == null ? '-' : Math.round(d.gap_seconds) + ' 秒'],
|
||
['最近失败', d.last_error ? d.last_error.slice(0, 24) : '无'],
|
||
], stateCls),
|
||
monitorCard('特征检测', fmt(d.rounds) + ' 轮', [
|
||
['失败次数', fmt(d.errors)],
|
||
['最近耗时', d.last_duration ? d.last_duration + ' 秒' : '-'],
|
||
['检测间隔', '60 秒'],
|
||
]),
|
||
monitorCard('行为分析', fmt(b.runs) + ' 次', [
|
||
['最近耗时', b.last_duration ? b.last_duration + ' 秒' : '-'],
|
||
['最近违规', fmt(b.last_count)],
|
||
['最近运行', b.last_success ? localTime(b.last_success) : '-'],
|
||
], b.last_error ? 'mbad' : ''),
|
||
monitorCard('推送（24h）', ratio, [
|
||
['成功 / 失败', fmt(p.sent_24h) + ' / ' + fmt(p.failed_24h)],
|
||
['连续失败', fmt(p.consecutive_failures)],
|
||
['最近错误', p.last_error ? p.last_error.slice(0, 24) : '无'],
|
||
], (p.consecutive_failures > 0) ? 'mbad' : 'mok'),
|
||
monitorCard('名单', fmt(l.fused_total) + ' 个', [
|
||
['今日新增', fmt(l.fused_new_24h)],
|
||
['双来源', fmt(l.fused_both)],
|
||
['行为新发现', fmt(l.fused_behavior_only)],
|
||
]),
|
||
monitorCard('自动补测', fmt(bf.runs) + ' 次', [
|
||
['上次补测', bf.last_at ? localTime(bf.last_at) : '无'],
|
||
['补测区间', bf.last_from ? (localTime(bf.last_from) + ' → ' + localTime(bf.last_to)) : '-'],
|
||
['上次新增', fmt(bf.last_added)],
|
||
], bf.last_error ? 'mbad' : ''),
|
||
].join('');
|
||
const rows = (m.recent_notifications || []).map(row => {
|
||
const cls = row.status === 'sent' ? 'ok' : 'bad';
|
||
return `<tr><td class="muted">${localTime(row.created_at)}</td>
|
||
<td>${esc(row.ip)}</td>
|
||
<td><span class="nbadge ${cls}">${row.status === 'sent' ? '成功' : '失败'}</span></td>
|
||
<td class="muted">${esc(row.error || '-')}</td></tr>`;
|
||
}).join('');
|
||
return `<div class="monitor-head">
|
||
<a class="back" href="#/">← 返回名单</a>
|
||
<div class="monitor-title">运行监控</div>
|
||
<span class="muted">每 30 秒刷新</span>
|
||
</div>
|
||
<div class="mgrid">${cards}</div>
|
||
<div class="mtablewrap">
|
||
<table class="mtable">
|
||
<thead><tr><th>时间</th><th>IP</th><th>结果</th><th>错误</th></tr></thead>
|
||
<tbody>${rows || '<tr><td colspan="4" class="empty">暂无推送记录</td></tr>'}</tbody>
|
||
</table>
|
||
</div>`;
|
||
}
|
||
|
||
async function showMonitor() {
|
||
state.inMonitor = true;
|
||
state.inDetail = false;
|
||
document.title = '运行监控';
|
||
$('#listView').hidden = true;
|
||
$('#detailView').hidden = true;
|
||
$('#monitorView').hidden = false;
|
||
loadStatus();
|
||
loadStats();
|
||
await loadMonitor();
|
||
}
|
||
|
||
async function loadMonitor() {
|
||
const view = $('#monitorView');
|
||
try {
|
||
const data = await jget(BASE + '/api/monitor');
|
||
view.innerHTML = monitorHtml(data);
|
||
} catch (err) {
|
||
view.innerHTML = `<a class="back" href="#/">← 返回名单</a>
|
||
<div class="empty">加载失败：${esc(err.message)}</div>`;
|
||
}
|
||
}
|
||
|
||
function route() {
|
||
if (location.hash === '#/monitor') { showMonitor(); return; }
|
||
const match = location.hash.match(/^#\/ip\/(.+)$/);
|
||
if (match) showDetail(decodeURIComponent(match[1]));
|
||
else showList();
|
||
}
|
||
|
||
let searchTimer = null;
|
||
$('#search').addEventListener('input', e => {
|
||
state.q = e.target.value;
|
||
clearTimeout(searchTimer);
|
||
searchTimer = setTimeout(() => { state.page = 1; loadList(); }, 250);
|
||
});
|
||
document.addEventListener('click', e => {
|
||
const target = e.target.closest('[data-copyip]');
|
||
if (!target) return;
|
||
e.stopPropagation();
|
||
copyText(target.dataset.copyip, '已复制 IP：' + target.dataset.copyip);
|
||
});
|
||
$('#copyBtn').addEventListener('click', e => {
|
||
e.stopPropagation();
|
||
toggleMenu('copyMenu');
|
||
});
|
||
$('#exportBtn').addEventListener('click', e => {
|
||
e.stopPropagation();
|
||
toggleMenu('exportMenu');
|
||
});
|
||
$('#copyMenu').addEventListener('click', e => {
|
||
const item = e.target.closest('[data-copy]');
|
||
if (!item) return;
|
||
closeMenus();
|
||
copyIps(item.dataset.copy);
|
||
});
|
||
$('#exportMenu').addEventListener('click', e => {
|
||
const item = e.target.closest('[data-export]');
|
||
if (!item) return;
|
||
closeMenus();
|
||
exportList(item.dataset.export);
|
||
});
|
||
$('#pager').addEventListener('click', e => {
|
||
const btn = e.target.closest('button[data-page]');
|
||
if (!btn || btn.disabled) return;
|
||
state.page = Number(btn.dataset.page);
|
||
loadList();
|
||
});
|
||
$('#pager').addEventListener('change', e => {
|
||
if (e.target.id !== 'pageSize') return;
|
||
state.pageSize = Number(e.target.value);
|
||
state.page = 1;
|
||
loadList();
|
||
});
|
||
$('#auto').addEventListener('change', e => {
|
||
if (e.target.checked) startTimer(); else clearInterval(state.timer);
|
||
});
|
||
function startTimer() {
|
||
clearInterval(state.timer);
|
||
state.timer = setInterval(() => {
|
||
loadStatus();
|
||
if (state.inMonitor) { loadMonitor(); }
|
||
else if (!state.inDetail) { loadStats(); loadList(); }
|
||
}, 30000);
|
||
}
|
||
window.addEventListener('hashchange', route);
|
||
startTimer();
|
||
route();
|
||
</script>
|
||
</body></html>"""
|
||
|
||
|
||
def make_handler(store, window, interval, detector=None, path_prefix=""):
|
||
class Handler(BaseHTTPRequestHandler):
|
||
def log_message(self, *args):
|
||
pass
|
||
|
||
def _send(self, code, payload, content_type="application/json; charset=utf-8"):
|
||
if isinstance(payload, (dict, list)):
|
||
payload = json.dumps(payload, ensure_ascii=False)
|
||
raw = payload.encode("utf-8")
|
||
self.send_response(code)
|
||
self.send_header("Content-Type", content_type)
|
||
self.send_header("Content-Length", str(len(raw)))
|
||
self.send_header("Cache-Control", "no-store")
|
||
self.end_headers()
|
||
self.wfile.write(raw)
|
||
|
||
def _send_raw(self, code, text, content_type, filename=None):
|
||
raw = text.encode("utf-8")
|
||
self.send_response(code)
|
||
self.send_header("Content-Type", content_type)
|
||
self.send_header("Content-Length", str(len(raw)))
|
||
if filename:
|
||
self.send_header("Content-Disposition",
|
||
'attachment; filename="%s"' % filename)
|
||
self.send_header("Cache-Control", "no-store")
|
||
self.end_headers()
|
||
self.wfile.write(raw)
|
||
|
||
def do_GET(self):
|
||
parsed = urlparse(self.path)
|
||
path = parsed.path
|
||
query = {key: values[0] for key, values in parse_qs(parsed.query).items()}
|
||
if path_prefix:
|
||
if path == path_prefix:
|
||
self.send_response(302)
|
||
self.send_header("Location", path_prefix + "/")
|
||
self.end_headers()
|
||
return
|
||
if path.startswith(path_prefix + "/"):
|
||
path = path[len(path_prefix):] or "/"
|
||
elif path != "/healthz":
|
||
return self._send_raw(404, "Not Found",
|
||
"text/plain; charset=utf-8")
|
||
if path == "/":
|
||
return self._send(200, DASHBOARD.replace("__BASE__", path_prefix),
|
||
"text/html; charset=utf-8")
|
||
if path == "/healthz":
|
||
return self._send(200, {"ok": True})
|
||
if path == "/api/stats":
|
||
return self._send(200, store.stats())
|
||
if path in ("/metrics", "/api/monitor"):
|
||
stats = store.stats()
|
||
meta = store.meta_dict()
|
||
notif = store.notification_stats(24)
|
||
now_epoch = time.time()
|
||
|
||
def stamp_epoch(value):
|
||
try:
|
||
return datetime.strptime(
|
||
value, "%Y-%m-%dT%H:%M:%SZ").replace(
|
||
tzinfo=timezone.utc).timestamp()
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
last_success = stamp_epoch(meta.get("detect_last_success"))
|
||
last_behavior = stamp_epoch(meta.get("behavior_last_success"))
|
||
age = round(now_epoch - last_success, 1) if last_success else None
|
||
behavior_age = (round(now_epoch - last_behavior, 1)
|
||
if last_behavior else None)
|
||
if age is None:
|
||
graylog = "unknown"
|
||
elif meta.get("detect_last_error") and age > interval * 2:
|
||
graylog = "error"
|
||
elif age <= max(120, interval * 2.5):
|
||
graylog = "ok"
|
||
else:
|
||
graylog = "stale"
|
||
|
||
payload = {
|
||
"time": now_iso(),
|
||
"graylog": graylog,
|
||
"detect": {
|
||
"rounds": int(meta.get("detect_rounds_total") or 0),
|
||
"errors": int(meta.get("detect_errors_total") or 0),
|
||
"last_success": meta.get("detect_last_success"),
|
||
"gap_seconds": age,
|
||
"last_duration": float(meta.get("detect_last_duration") or 0),
|
||
"last_error": meta.get("detect_last_error") or "",
|
||
},
|
||
"behavior": {
|
||
"runs": int(meta.get("behavior_runs_total") or 0),
|
||
"errors": int(meta.get("behavior_errors_total") or 0),
|
||
"last_success": meta.get("behavior_last_success"),
|
||
"age_seconds": behavior_age,
|
||
"last_duration": float(meta.get("behavior_last_duration") or 0),
|
||
"last_count": int(meta.get("behavior_last_count") or 0),
|
||
"last_error": meta.get("behavior_last_error") or "",
|
||
},
|
||
"push": {
|
||
"sent_total": int(meta.get("push_sent_total") or 0),
|
||
"failed_total": int(meta.get("push_failed_total") or 0),
|
||
"sent_24h": notif["sent"],
|
||
"failed_24h": notif["failed"],
|
||
"success_ratio_24h": notif["success_ratio"],
|
||
"consecutive_failures": int(
|
||
meta.get("push_consecutive_failures") or 0),
|
||
"last_error": meta.get("push_last_error") or "",
|
||
"last_error_at": meta.get("push_last_error_at") or "",
|
||
},
|
||
"list": {
|
||
"fused_total": stats.get("fused_total"),
|
||
"fused_new_24h": stats.get("fused_new_24h"),
|
||
"fused_both": stats.get("fused_both"),
|
||
"fused_behavior_only": stats.get("fused_behavior_only"),
|
||
"updated_at": stats.get("updated_at"),
|
||
},
|
||
"backfill": {
|
||
"runs": int(meta.get("backfill_runs_total") or 0),
|
||
"last_at": meta.get("backfill_last_at") or "",
|
||
"last_from": meta.get("backfill_last_from") or "",
|
||
"last_to": meta.get("backfill_last_to") or "",
|
||
"last_added": int(meta.get("backfill_last_added") or 0),
|
||
"last_error": meta.get("backfill_last_error") or "",
|
||
},
|
||
"recent_notifications": notif["recent"],
|
||
}
|
||
if path == "/api/monitor":
|
||
return self._send(200, payload)
|
||
|
||
lines = [
|
||
"# HELP ipsec_detect_rounds_total Successful detection rounds",
|
||
"# TYPE ipsec_detect_rounds_total counter",
|
||
"ipsec_detect_rounds_total %d" % payload["detect"]["rounds"],
|
||
"# TYPE ipsec_detect_errors_total counter",
|
||
"ipsec_detect_errors_total %d" % payload["detect"]["errors"],
|
||
"# TYPE ipsec_detect_last_success_timestamp gauge",
|
||
"ipsec_detect_last_success_timestamp %s"
|
||
% (last_success or 0),
|
||
"# TYPE ipsec_detect_gap_seconds gauge",
|
||
"ipsec_detect_gap_seconds %s" % (age if age is not None else -1),
|
||
"# TYPE ipsec_detect_last_duration_seconds gauge",
|
||
"ipsec_detect_last_duration_seconds %s"
|
||
% payload["detect"]["last_duration"],
|
||
"# TYPE ipsec_behavior_runs_total counter",
|
||
"ipsec_behavior_runs_total %d" % payload["behavior"]["runs"],
|
||
"# TYPE ipsec_behavior_errors_total counter",
|
||
"ipsec_behavior_errors_total %d" % payload["behavior"]["errors"],
|
||
"# TYPE ipsec_behavior_last_duration_seconds gauge",
|
||
"ipsec_behavior_last_duration_seconds %s"
|
||
% payload["behavior"]["last_duration"],
|
||
"# TYPE ipsec_behavior_last_violations gauge",
|
||
"ipsec_behavior_last_violations %d"
|
||
% payload["behavior"]["last_count"],
|
||
"# TYPE ipsec_fused_total gauge",
|
||
"ipsec_fused_total %s" % (payload["list"]["fused_total"] or 0),
|
||
"# TYPE ipsec_fused_new_24h gauge",
|
||
"ipsec_fused_new_24h %s" % (payload["list"]["fused_new_24h"] or 0),
|
||
"# TYPE ipsec_fused_both gauge",
|
||
"ipsec_fused_both %s" % (payload["list"]["fused_both"] or 0),
|
||
"# TYPE ipsec_push_sent_total counter",
|
||
"ipsec_push_sent_total %d" % payload["push"]["sent_total"],
|
||
"# TYPE ipsec_push_failed_total counter",
|
||
"ipsec_push_failed_total %d" % payload["push"]["failed_total"],
|
||
"# TYPE ipsec_push_consecutive_failures gauge",
|
||
"ipsec_push_consecutive_failures %d"
|
||
% payload["push"]["consecutive_failures"],
|
||
"# TYPE ipsec_push_success_ratio_24h gauge",
|
||
"ipsec_push_success_ratio_24h %s"
|
||
% (payload["push"]["success_ratio_24h"]
|
||
if payload["push"]["success_ratio_24h"] is not None else -1),
|
||
"# TYPE ipsec_graylog_up gauge",
|
||
"ipsec_graylog_up %d" % (1 if graylog == "ok" else 0),
|
||
"",
|
||
]
|
||
return self._send(200, "\n".join(lines),
|
||
content_type="text/plain; version=0.0.4; charset=utf-8")
|
||
|
||
if path == "/api/status":
|
||
status = {"graylog": "unknown", "last_run": None,
|
||
"last_error": None, "age_seconds": None}
|
||
if detector is not None:
|
||
age = (time.time() - detector.last_success
|
||
if getattr(detector, "last_success", None) else None)
|
||
status["last_run"] = detector.last_run
|
||
status["last_error"] = detector.last_error
|
||
status["age_seconds"] = round(age, 1) if age is not None else None
|
||
if detector.last_error and (age is None
|
||
or age > interval * 2):
|
||
status["graylog"] = "error"
|
||
elif age is not None and age <= max(120, interval * 2.5):
|
||
status["graylog"] = "ok"
|
||
elif age is not None:
|
||
status["graylog"] = "stale"
|
||
return self._send(200, status)
|
||
|
||
if path == "/api/ips":
|
||
page_size = int(query.get("page_size", query.get("limit", 50)) or 50)
|
||
page_size = max(1, min(2000, page_size))
|
||
page = max(1, int(query.get("page", 1) or 1))
|
||
offset = max(0, int(query.get("offset",
|
||
(page - 1) * page_size) or 0))
|
||
search_q = (query.get("q") or "").strip()
|
||
total = store.fused_count(search_q)
|
||
fused = store.fused_items(page_size, offset, search_q)
|
||
window = detector.window if detector is not None else 3600
|
||
items = []
|
||
for row in fused:
|
||
try:
|
||
rules = json.loads(row.get("s_rules") or "{}")
|
||
except ValueError:
|
||
rules = {}
|
||
items.append({
|
||
"ip": row["ip"],
|
||
"sources": row["sources"],
|
||
"tier": row.get("tier") or "behavior",
|
||
"score": max(float(row.get("feature_score") or 0),
|
||
float(row.get("behavior_score") or 0)),
|
||
"feature_score": row.get("feature_score"),
|
||
"behavior_score": row.get("behavior_score"),
|
||
"total": row.get("total") or 0,
|
||
"rules": rules,
|
||
"reasons": [row.get("fused_reason") or ""],
|
||
"first_seen": row.get("first_banned_at"),
|
||
"last_seen": row.get("last_activity_at"),
|
||
"last_malicious_at": row.get("last_activity_at"),
|
||
"window": window,
|
||
"country_code": row.get("country_code") or "",
|
||
"country_name": row.get("country_name") or "",
|
||
"region": row.get("region") or "",
|
||
"city": row.get("city") or "",
|
||
"asn": row.get("asn") or "",
|
||
"asn_org": row.get("asn_org") or "",
|
||
"asn_brand": row.get("asn_brand") or "",
|
||
"asn_type": row.get("asn_type") or "",
|
||
"asn_icon": row.get("asn_icon") or "",
|
||
})
|
||
pages = max(1, (total + page_size - 1) // page_size)
|
||
return self._send(200, {
|
||
"count": len(items), "total": total, "page": page,
|
||
"page_size": page_size, "pages": pages, "q": search_q,
|
||
"updated_at": store.stats().get("updated_at"),
|
||
"items": items})
|
||
if path in ("/api/export.csv", "/api/export.txt"):
|
||
scope = query.get("scope", "core")
|
||
if scope not in ("core", "all"):
|
||
scope = "core"
|
||
sort = query.get("sort", "first_seen")
|
||
if sort not in Store.SORT_COLUMNS:
|
||
sort = "first_seen"
|
||
order = query.get("order", "desc")
|
||
if order not in ("asc", "desc"):
|
||
order = "desc"
|
||
items = store.list(limit=20000,
|
||
scope=None if scope == "all" else scope,
|
||
sort=sort, order=order)
|
||
if path.endswith(".csv"):
|
||
body = _csv_text(items)
|
||
ctype = "text/csv; charset=utf-8"
|
||
filename = "malicious-ips.csv"
|
||
else:
|
||
body = "\n".join(item["ip"] for item in items) + ("\n" if items else "")
|
||
ctype = "text/plain; charset=utf-8"
|
||
filename = "malicious-ips.txt"
|
||
return self._send_raw(200, body, ctype, filename)
|
||
|
||
if path.startswith("/api/ip/"):
|
||
ip = path[len("/api/ip/"):]
|
||
item = store.get(ip)
|
||
fused = store.fused_row(ip)
|
||
behavior = store.behavior_profile(ip)
|
||
if item is None and fused is None:
|
||
return self._send(404, {"error": "not found", "ip": ip})
|
||
if item is None:
|
||
item = {
|
||
"ip": ip, "tier": "behavior",
|
||
"score": (behavior or {}).get("score") or 0,
|
||
"total": (behavior or {}).get("total") or 0,
|
||
"rules": {},
|
||
"reasons": [(fused or {}).get("fused_reason")
|
||
or (behavior or {}).get("reason_text") or ""],
|
||
"first_seen": (fused or {}).get("first_banned_at"),
|
||
"last_seen": (fused or {}).get("last_activity_at"),
|
||
"last_malicious_at": (fused or {}).get("last_activity_at"),
|
||
"window": 86400,
|
||
"country_code": (behavior or {}).get("country_code", ""),
|
||
"country_name": (behavior or {}).get("country_name", ""),
|
||
"region": (behavior or {}).get("region", ""),
|
||
"city": (behavior or {}).get("city", ""),
|
||
"asn": (behavior or {}).get("asn", ""),
|
||
"asn_org": (behavior or {}).get("asn_org", ""),
|
||
"asn_brand": (behavior or {}).get("asn_brand", ""),
|
||
"asn_type": (behavior or {}).get("asn_type", ""),
|
||
"asn_icon": (behavior or {}).get("asn_icon", ""),
|
||
}
|
||
item["sources"] = (fused or {}).get("sources") or "feature"
|
||
item["behavior"] = behavior
|
||
item["fused"] = fused
|
||
if item.get("window") is None and detector is not None:
|
||
item["window"] = detector.window
|
||
|
||
# Evidence is fetched live from Graylog on click (memory
|
||
# cached) so the database stays small; a compact snapshot is
|
||
# kept as a fallback once the index has rotated away.
|
||
stored_raw = item.pop("evidence", None)
|
||
try:
|
||
evidence = collect_evidence(
|
||
detector, ip, item.get("rules"),
|
||
item.get("window") or (detector.window if detector else None))
|
||
except Exception as exc: # never kill the connection
|
||
evidence = {"available": False, "error": str(exc)}
|
||
|
||
live_ok = bool(evidence.get("available")
|
||
and (evidence.get("uris") or evidence.get("samples")))
|
||
if live_ok:
|
||
try:
|
||
stamp = now_iso()
|
||
store.set_evidence(ip, _compact_evidence(evidence), stamp)
|
||
item["evidence_at"] = stamp
|
||
except Exception:
|
||
pass
|
||
evidence["stale"] = False
|
||
else:
|
||
snapshot = None
|
||
if stored_raw:
|
||
try:
|
||
snapshot = json.loads(stored_raw)
|
||
except ValueError:
|
||
snapshot = None
|
||
if snapshot:
|
||
snapshot["available"] = True
|
||
snapshot["stale"] = True
|
||
snapshot["samples"] = []
|
||
snapshot["reason"] = "Graylog 中该时段日志已过期，以下为判定时的快照"
|
||
snapshot.setdefault("window", item.get("window"))
|
||
evidence = snapshot
|
||
return self._send(200, {"item": item, "evidence": evidence})
|
||
return self._send(404, {"error": "not found", "path": path})
|
||
|
||
return Handler
|
||
|
||
|
||
# --------------------------------------------------------------------- entry
|
||
def run(ctx, args):
|
||
fields = dict(DEFAULT_FIELDS)
|
||
profile_path = getattr(args, "profile", None)
|
||
if profile_path:
|
||
with open(profile_path, "r", encoding="utf-8") as handle:
|
||
profile = json.load(handle)
|
||
for key, value in (profile.get("fields") or {}).items():
|
||
if value and key in fields:
|
||
fields[key] = value
|
||
for attribute, key in (("field", "client_ip"), ("status_field", "status"),
|
||
("uri_field", "uri"), ("ua_field", "ua")):
|
||
value = getattr(args, attribute, None)
|
||
if value:
|
||
fields[key] = value
|
||
|
||
allow_entries = []
|
||
for path in (getattr(args, "allow", None) or []):
|
||
with open(path, "r", encoding="utf-8") as handle:
|
||
text = handle.read()
|
||
for line in text.splitlines():
|
||
line = line.split("#", 1)[0].strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
allow_entries.append(ipaddress.ip_network(line, strict=False))
|
||
except ValueError:
|
||
raise SystemExit("白名单 %s 不是合法 IP/CIDR: %r" % (path, line))
|
||
|
||
def allowlisted(ip):
|
||
try:
|
||
address = ipaddress.ip_address(ip)
|
||
except ValueError:
|
||
return False
|
||
return any(address.version == net.version and address in net
|
||
for net in allow_entries)
|
||
|
||
notifier = None
|
||
notify_path = getattr(args, "notify_config", "") or ""
|
||
if notify_path and os.path.exists(notify_path):
|
||
try:
|
||
notifier = load_notifier(notify_path)
|
||
print("推送已启用: %s" % notifier.kind, flush=True)
|
||
except Exception as exc:
|
||
print("推送配置无效: %s" % exc, flush=True)
|
||
if notifier is None:
|
||
try:
|
||
notifier = notifier_from_env()
|
||
if notifier is not None:
|
||
print("推送已启用（环境变量）: %s" % notifier.kind, flush=True)
|
||
except Exception as exc:
|
||
print("推送环境变量无效: %s" % exc, flush=True)
|
||
|
||
if getattr(args, "notify_test", False):
|
||
if notifier is None:
|
||
print("未配置推送：请创建 %s（参考 notify.example.json）"
|
||
% (notify_path or "notify.json"), flush=True)
|
||
return 1
|
||
try:
|
||
reply = notifier.send_test()
|
||
except Exception as exc:
|
||
print("测试推送失败: %s" % exc, flush=True)
|
||
return 1
|
||
print("测试推送成功: %s" % str(reply)[:300], flush=True)
|
||
return 0
|
||
|
||
store = Store(args.db)
|
||
detector = Detector(
|
||
ctx.client, store, fields,
|
||
window=int(args.window), size=int(args.size),
|
||
min_requests=int(args.min_requests), verify_limit=int(args.verify),
|
||
forget=int(args.forget), allowlist=allowlisted,
|
||
min_observe_score=float(args.min_observe_score),
|
||
warm_limit=int(getattr(args, "warm_limit", 5) or 0),
|
||
filter_str=("streams:" + ctx.stream_id) if ctx.stream_id else None,
|
||
geo_db=getattr(args, "geo_db", "") or None,
|
||
asn_db=getattr(args, "asn_db", "") or None,
|
||
)
|
||
if getattr(detector, "geo", None) is not None:
|
||
filled = detector.backfill_geo(3000)
|
||
if filled:
|
||
print("补充地理信息 %d 个 IP" % filled, flush=True)
|
||
if getattr(detector, "asn", None) is not None:
|
||
filled = detector.backfill_asn(3000)
|
||
if filled:
|
||
print("补充 ASN 信息 %d 个 IP" % filled, flush=True)
|
||
|
||
# Fused block list: existing feature detections first (historical rows keep
|
||
# their original discovery time), then the behaviour thread keeps adding.
|
||
store.sync_fused()
|
||
filter_str = ("streams:" + ctx.stream_id) if ctx.stream_id else None
|
||
|
||
def auto_backfill_worker(from_dt, to_dt):
|
||
from .backfill import auto_backfill_gap
|
||
summary = auto_backfill_gap(
|
||
store, ctx.client, fields, filter_str, from_dt, to_dt,
|
||
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
print("[自动补测] %s" % json.dumps(summary, ensure_ascii=False),
|
||
flush=True)
|
||
|
||
auto_enabled = not getattr(args, "no_auto_backfill", False)
|
||
threshold = int(getattr(args, "backfill_threshold", 1800) or 0)
|
||
max_gap = int(getattr(args, "backfill_max_gap", 7 * 86400) or 0)
|
||
if (auto_enabled and threshold > 0
|
||
and not getattr(args, "serve_only", False)):
|
||
from .backfill import parse_iso
|
||
last_meta = store.get_meta("detect_last_success")
|
||
if not last_meta:
|
||
row = store.conn.execute(
|
||
"SELECT MAX(updated_at) AS t FROM ip_stats").fetchone()
|
||
last_meta = row["t"] if row and row["t"] else None
|
||
last_dt = parse_iso(last_meta) if last_meta else None
|
||
if last_dt is not None:
|
||
now_dt = datetime.now(timezone.utc)
|
||
gap = (now_dt - last_dt).total_seconds()
|
||
if gap > max_gap:
|
||
print("检测缺口 %.1f 小时，超过自动补测上限 %d 小时，已跳过"
|
||
% (gap / 3600.0, max_gap // 3600), flush=True)
|
||
elif gap > threshold:
|
||
print("检测缺口 %.1f 小时，后台自动补测 %s → %s"
|
||
% (gap / 3600.0, last_dt.strftime("%m-%d %H:%M"),
|
||
now_dt.astimezone().strftime("%m-%d %H:%M")),
|
||
flush=True)
|
||
threading.Thread(target=auto_backfill_worker,
|
||
args=(last_dt, now_dt), daemon=True).start()
|
||
else:
|
||
print("检测连续（上次成功 %s），无需补测"
|
||
% last_dt.strftime("%m-%d %H:%M"), flush=True)
|
||
else:
|
||
print("没有历史检测记录，跳过自动补测", flush=True)
|
||
|
||
behavior_analyzer = None
|
||
if not getattr(args, "no_behavior", False):
|
||
behavior_analyzer = BehaviorAnalyzer(
|
||
ctx.client, store, fields,
|
||
filter_str=("streams:" + ctx.stream_id) if ctx.stream_id else None,
|
||
root=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
|
||
interval=int(getattr(args, "behavior_interval", 900) or 900),
|
||
notifier=notifier,
|
||
)
|
||
threading.Thread(target=behavior_analyzer.run_forever, daemon=True).start()
|
||
print("行为检测线程已启动（每 %d 秒一次）" % behavior_analyzer.interval,
|
||
flush=True)
|
||
|
||
def report(result):
|
||
line = ("[%s] 窗口 %ds: %d 个候选（恶意 %d / 可疑 %d / 自动化 %d），耗时 %.1fs"
|
||
% (now_iso(), result["window"], result["candidates"],
|
||
result["malicious"], result["suspicious"], result["automation"],
|
||
result["request_seconds"]))
|
||
print(line, flush=True)
|
||
|
||
if getattr(args, "fix_first_seen", False):
|
||
count = detector.backfill_first_seen(int(getattr(args, "warm_limit", 500) or 500))
|
||
print("已修正 %d 个 IP 的首次发现时间" % count, flush=True)
|
||
store.close()
|
||
return 0
|
||
|
||
if getattr(args, "warm_only", False):
|
||
count = detector.warm_snapshots(int(getattr(args, "warm_limit", 50) or 50))
|
||
print("已预热 %d 个 IP 的证据快照" % count, flush=True)
|
||
store.close()
|
||
return 0
|
||
|
||
if getattr(args, "once", False):
|
||
report(detector.run_once())
|
||
if getattr(args, "export", None):
|
||
chosen = export_blocklist(store, args.export)
|
||
print("已导出 %d 个恶意 IP -> %s" % (len(chosen), args.export))
|
||
for row in store.list(limit=int(args.top)):
|
||
print("%-16s %-11s %5.1f %6d %s" % (
|
||
row["ip"], TIER_LABELS.get(row["tier"], row["tier"]),
|
||
row["score"], row["total"], "；".join(row["reasons"][:3])))
|
||
store.close()
|
||
return 0
|
||
|
||
prefix = (getattr(args, "path_prefix", "") or "").strip()
|
||
if prefix:
|
||
prefix = "/" + prefix.strip("/")
|
||
server = ThreadingHTTPServer((args.host, int(args.port)),
|
||
make_handler(store, int(args.window),
|
||
int(args.interval), detector, prefix))
|
||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||
thread.start()
|
||
print("面板: http://%s:%s%s/ (Ctrl+C 退出)"
|
||
% (args.host, args.port, prefix), flush=True)
|
||
|
||
if getattr(args, "serve_only", False):
|
||
try:
|
||
thread.join()
|
||
except KeyboardInterrupt:
|
||
print("", flush=True)
|
||
finally:
|
||
server.shutdown()
|
||
store.close()
|
||
return 0
|
||
|
||
try:
|
||
while True:
|
||
try:
|
||
report(detector.run_once())
|
||
store.sync_fused()
|
||
if notifier is not None:
|
||
notify_fused(store, notifier)
|
||
if getattr(args, "export", None):
|
||
chosen = export_blocklist(store, args.export)
|
||
print(" 封禁名单: %d 个 -> %s" % (len(chosen), args.export), flush=True)
|
||
except GraylogError as exc:
|
||
detector.last_error = str(exc)
|
||
detector.error_count += 1
|
||
detector.store.set_meta("detect_last_error", str(exc))
|
||
detector.store.incr_meta("detect_errors_total")
|
||
print("[%s] 检测失败: %s" % (now_iso(), exc), flush=True)
|
||
except Exception as exc:
|
||
detector.last_error = str(exc)
|
||
detector.error_count += 1
|
||
detector.store.set_meta("detect_last_error", str(exc))
|
||
detector.store.incr_meta("detect_errors_total")
|
||
print("[%s] 检测异常: %s" % (now_iso(), exc), flush=True)
|
||
time.sleep(max(5, int(args.interval)))
|
||
except KeyboardInterrupt:
|
||
print("\n停止。数据库: %s" % os.path.abspath(args.db), flush=True)
|
||
finally:
|
||
server.shutdown()
|
||
store.close()
|
||
return 0
|