初始提交：恶意 IP 特征+行为融合检测系统

- 特征检测（规则/签名）与行为检测（24h 统计）单进程运行
- 融合名单按首次发现时间倒序，展示位置/归属（GeoIP + ASN）
- 钉钉融合推送（加签、24h 节流、去重）
- 页面状态灯、CSV 导出、Docker 镜像构建文件

graylog_ops/__init__.py

@@ -0,0 +1,8 @@
+"""Read-only Graylog inspection toolkit.
+
+Target API: Graylog 4.0.x (legacy ``/api/search/universal`` surface).
+No third-party dependencies: stdlib only, so it also runs on old CentOS 7
+python3 installations.
+"""
+
+__version__ = "0.2.0"

graylog_ops/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

graylog_ops/asn.py

@@ -0,0 +1,107 @@
+"""OFFLINE ASN lookup + provider branding (DB-IP ASN Lite)."""
+import bisect
+import csv
+import ipaddress
+import json
+import os
+import sqlite3
+
+DEFAULT_MAP = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+                           "asn_brand_map.json")
+
+
+def load_brand_map(path=None):
+    try:
+        with open(path or DEFAULT_MAP, encoding="utf-8") as handle:
+            data = json.load(handle)
+    except (OSError, ValueError):
+        data = {"items": [], "defaults": {"brand": "未知", "type": "unknown",
+                                          "icon": "❓"}}
+    return data
+
+
+def classify_org(org, brand_map=None):
+    brand_map = brand_map or load_brand_map()
+    text = (org or "").lower()
+    for item in brand_map.get("items", []):
+        if any(keyword in text for keyword in item.get("keywords", [])):
+            return item["brand"], item["type"], item["icon"]
+    fallback = brand_map.get("defaults") or {}
+    return (fallback.get("brand", "未知"), fallback.get("type", "unknown"),
+            fallback.get("icon", "❓"))
+
+
+def build_asn_db(csv_path, out_path):
+    os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
+    conn = sqlite3.connect(out_path)
+    conn.execute("DROP TABLE IF EXISTS asn")
+    conn.execute("CREATE TABLE asn (start INTEGER PRIMARY KEY, end INTEGER, "
+                 "asn TEXT, org TEXT)")
+    batch, total = [], 0
+
+    def flush():
+        nonlocal batch
+        if batch:
+            conn.executemany("INSERT OR REPLACE INTO asn VALUES (?,?,?,?)", batch)
+            batch = []
+
+    with open(csv_path, encoding="utf-8", errors="replace") as handle:
+        for row in csv.reader(handle):
+            if len(row) < 4 or ":" in row[0]:
+                continue
+            try:
+                first = int(ipaddress.IPv4Address(row[0]))
+                last = int(ipaddress.IPv4Address(row[1]))
+            except ValueError:
+                continue
+            batch.append((first, last, row[2], row[3]))
+            total += 1
+            if len(batch) >= 100000:
+                flush()
+                conn.commit()
+    flush()
+    conn.commit()
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.close()
+    return total
+
+
+class AsnDB:
+    def __init__(self, path, brand_map_path=None):
+        self.conn = sqlite3.connect(path, check_same_thread=False)
+        self.brand_map = load_brand_map(brand_map_path)
+        self._prefix = None
+        with self.conn:
+            try:
+                self.conn.execute("PRAGMA query_only=ON")
+            except sqlite3.Error:
+                pass
+
+    def lookup(self, ip):
+        try:
+            value = int(ipaddress.IPv4Address(ip))
+        except ValueError:
+            return None
+        row = self.conn.execute(
+            "SELECT end, asn, org FROM asn WHERE start <= ? "
+            "ORDER BY start DESC LIMIT 1", (value,)).fetchone()
+        if not row or row[0] < value:
+            return None
+        brand, kind, icon = classify_org(row[2], self.brand_map)
+        return {"asn": row[1], "org": row[2], "brand": brand,
+                "type": kind, "icon": icon}
+
+    def close(self):
+        try:
+            self.conn.close()
+        except sqlite3.Error:
+            pass
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--csv", default="data/geo/dbip-asn-lite-2026-10.csv")
+    parser.add_argument("--out", default="data/geo/asn.db")
+    args = parser.parse_args()
+    print("imported %d ranges -> %s" % (build_asn_db(args.csv, args.out), args.out))

graylog_ops/behavior.py

@@ -0,0 +1,695 @@
+"""Behaviour detection running inside the main service.
+
+It scans 24h of F5 traffic with Graylog pivot aggregations, scores each IP by
+behaviour only (no rule hits, no known list), and stores the result in SQLite.
+A background thread keeps it refreshed; the web UI reads the stored result.
+"""
+import csv
+import ipaddress
+import json
+import os
+import statistics
+import threading
+import time
+
+from .http import GraylogError
+from .asn import AsnDB
+
+WINDOW = 86400
+INTERNAL = ("10.", "192.168.", "172.16.", "172.17.", "172.18.", "172.19.",
+            "172.2", "172.3")
+
+UA_SDK = ("curl", "wget", "python-requests", "python-urllib", "go-http-client",
+          "okhttp", "java/", "apache-httpclient", "httpcomponents",
+          "libredtail", "axios", "node-fetch", "aiohttp", "scrapy",
+          "postmanruntime", "insomnia", "unityplayer", "dart/", "httpie",
+          "guzzle", "faraday", "restsharp", "libwww", "eeo/", "academy_widget")
+UA_BROWSER = ("chrome", "firefox", "safari", "edg/", "opr/", "gecko/20100101")
+
+CRAWLER_TOKENS = ("sogou", "baiduspider", "googlebot", "bingbot", "yandex",
+                  "bytespider", "ahrefs", "semrush", "mj12", "dotbot",
+                  "petalbot")
+
+RULE_TITLES = {
+    "probe_java": "Spring/中间件探测", "probe_config": "配置/凭证探测",
+    "probe_cms": "CMS 探测", "probe_iot": "IoT/路由器探测",
+    "probe_devops": "DevOps/监控探测", "probe_webshell": "WebShell/上传探测",
+    "exploit_payload": "漏洞利用 payload", "login_attack": "登录失败尝试",
+    "login_burst": "登录接口高频", "bad_ua": "扫描器 UA",
+    "bot_ua": "僵尸网络 UA", "not_found": "404 探测",
+}
+
+ASN_CACHE = {}
+
+
+def _asn_csv(root):
+    geo_dir = os.path.join(root, "data", "geo")
+    try:
+        names = sorted(name for name in os.listdir(geo_dir)
+                       if name.startswith("dbip-asn-lite-") and name.endswith(".csv"))
+    except FileNotFoundError:
+        return None
+    return os.path.join(geo_dir, names[-1]) if names else None
+
+
+def _load_asn(path):
+    if ASN_CACHE:
+        return ASN_CACHE
+    starts, ranges = [], []
+    try:
+        with open(path, encoding="utf-8", errors="replace") as fh:
+            for row in csv.reader(fh):
+                if len(row) < 4 or ":" in row[0]:
+                    continue
+                try:
+                    first = int(ipaddress.IPv4Address(row[0]))
+                    last = int(ipaddress.IPv4Address(row[1]))
+                except ValueError:
+                    continue
+                starts.append(first)
+                ranges.append((last, row[2], row[3]))
+    except (FileNotFoundError, TypeError):
+        pass
+    order = sorted(range(len(starts)), key=lambda i: starts[i])
+    ASN_CACHE.update({"starts": [starts[i] for i in order],
+                      "ranges": [ranges[i] for i in order]})
+    return ASN_CACHE
+
+
+def shared_type(org):
+    text = (org or "").lower()
+    cloud = ("alibaba", "aliyun", "tencent", "huawei cloud", "aws", "amazon",
+             "google", "microsoft", "digitalocean", "vultr", "linode", "hetzner",
+             "ovh", "wowrack", "idc", "data center", "datacenter", "cloud",
+             "hosting", "colo")
+    if any(token in text for token in cloud):
+        return "hosting"
+    if "china mobile" in text or "中国移动" in text or "cmnet" in text:
+        return "mobile"
+    if ("china unicom" in text or "china telecom" in text or "chinanet" in text
+            or "联通" in text or "电信" in text or "broadband" in text):
+        return "broadband"
+    if "education" in text or "cernet" in text or "教育" in text:
+        return "education"
+    return "unknown"
+
+
+ORG_CN = (("china mobile", "中国移动"), ("china unicom", "中国联通"),
+          ("china telecom", "中国电信"), ("chinanet", "中国电信"),
+          ("cernet", "中国教育网"), ("education", "教育网"),
+          ("alibaba", "阿里云"), ("aliyun", "阿里云"), ("huawei cloud", "华为云"),
+          ("tencent", "腾讯云"), ("wowrack", "Wowrack 机房"),
+          ("vmiss", "VMISS 机房"), ("idc", "IDC 机房"))
+
+TYPE_TO_SHARED = {"mobile": "mobile", "broadband": "broadband",
+                  "education": "education", "cloud": "hosting", "idc": "hosting"}
+
+SHARE_LABEL = {"mobile": "移动网络", "broadband": "宽带 ISP",
+               "education": "教育网", "hosting": "云主机/机房", "unknown": "未知网络"}
+
+
+def org_cn(org):
+    text = (org or "").lower()
+    for token, name in ORG_CN:
+        if token in text:
+            return name
+    return org or "未知"
+
+
+def asn_info(ip, root):
+    cache = _load_asn(_asn_csv(root))
+    starts = cache.get("starts") or []
+    if not starts:
+        return {"asn": "", "org": "未知", "shared": "unknown"}
+    try:
+        value = int(ipaddress.IPv4Address(ip))
+    except ValueError:
+        return {"asn": "", "org": "未知", "shared": "unknown"}
+    import bisect
+    index = bisect.bisect_right(starts, value) - 1
+    if index >= 0:
+        last, asn, org = cache["ranges"][index]
+        if last >= value:
+            return {"asn": asn, "org": org, "shared": shared_type(org)}
+    return {"asn": "", "org": "未知", "shared": "unknown"}
+
+
+def pivot(client, series, row_groups, column_groups=None, query="*",
+          limit=300, timeout=180000):
+    body = {"queries": [{
+        "id": "q1",
+        "timerange": {"type": "relative", "range": WINDOW},
+        "query": {"type": "elasticsearch", "query_string": query},
+        "filter": {"type": "or", "filters": [{"type": "stream",
+                                               "id": "6ac83059a6878e26256fcacc"}]},
+        "search_types": [{
+            "type": "pivot", "id": "st1", "series": series,
+            "row_groups": [{"type": "values", "field": f, "limit": limit}
+                           for f in row_groups],
+            "column_groups": column_groups or [], "sort": [], "rollup": True,
+        }],
+    }], "parameters": []}
+    data = client.http.request("POST", "/api/views/search/sync",
+                               params={"timeout": timeout}, body=body)
+    return data["results"]["q1"]["search_types"]["st1"]
+
+
+def rows_to_dict(search_type, series_id):
+    out = {}
+    for row in search_type.get("rows") or []:
+        key = row.get("key") or []
+        if not key:
+            continue
+        ip = str(key[0])
+        for value in row.get("values") or []:
+            if (value.get("key") or [None])[0] == series_id:
+                try:
+                    out[ip] = float(value.get("value") or 0)
+                except (TypeError, ValueError):
+                    out[ip] = 0.0
+    return out
+
+
+def hourly_matrix(search_type):
+    out = {}
+    for row in search_type.get("rows") or []:
+        key = row.get("key") or []
+        if not key:
+            continue
+        counts = []
+        for value in row.get("values") or []:
+            stamp = (value.get("key") or [None])[0]
+            if not stamp or "cnt" not in (value.get("key") or []):
+                continue
+            try:
+                counts.append(float(value.get("value") or 0))
+            except (TypeError, ValueError):
+                pass
+        out[str(key[0])] = counts
+    return out
+
+
+def bucket_stats(counts):
+    active = [c for c in counts if c > 0]
+    if not counts:
+        return {"coverage": 0.0, "cv": None, "peak_share": 0.0}
+    cv = (statistics.pstdev(active) / statistics.fmean(active)
+          if len(active) > 2 and statistics.fmean(active) else None)
+    total = sum(counts)
+    return {"coverage": len(active) / len(counts) * 100,
+            "cv": cv,
+            "peak_share": (max(counts) / total * 100) if total else 0.0}
+
+
+def uri_template(uri):
+    import re
+    path = str(uri or "").split("?")[0]
+    path = re.sub(
+        r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-"
+        r"[0-9a-fA-F]{12}", "{uuid}", path)
+    path = re.sub(r"[0-9a-fA-F]{16,}", "{hex}", path)
+    path = re.sub(r"\d+", "{n}", path)
+    return path
+
+
+def ua_family(ua):
+    text = str(ua or "").strip()
+    if not text:
+        return "empty"
+    lower = text.lower()
+    if any(token in lower for token in UA_SDK):
+        return "sdk"
+    if lower.startswith("mozilla/") and any(t in lower for t in UA_BROWSER):
+        return "browser"
+    if lower.startswith("mozilla/5.0"):
+        return "fake-browser"
+    return "other"
+
+
+def ua_garbage(ua_card, top_ua):
+    if ua_card > 50:
+        return True
+    text = str(top_ua or "")
+    if not text:
+        return False
+    printable = sum(1 for char in text if 32 <= ord(char) < 127 or ord(char) > 127)
+    return printable / max(1, len(text)) < 0.85
+
+
+def behavior_score(feat):
+    total = feat["total"]
+    if total < 10:
+        return 0, ["请求过少"]
+    score = 0
+    signals = []
+    pct4 = feat["pct4"]
+    if pct4 >= 50:
+        score += 30
+        signals.append("高失败率 %.0f%%" % pct4)
+    elif pct4 >= 30:
+        score += 15
+        signals.append("失败率 %.0f%%" % pct4)
+    if feat["uri_card"] >= 20:
+        score += 20
+        signals.append("路径分散（%d 个 URI）" % feat["uri_card"])
+    elif feat["uri_card"] >= 8:
+        score += 8
+    if feat["coverage"] <= 30 and feat["peak_share"] >= 25:
+        score += 20
+        signals.append("短时爆发（覆盖 %.0f%%、峰值 %.0f%%）"
+                       % (feat["coverage"], feat["peak_share"]))
+    if feat["login_ratio"] >= 20:
+        score += 15
+        signals.append("登录接口集中 %.0f%%" % feat["login_ratio"])
+    if feat["host_card"] >= 2:
+        score += 5
+        signals.append("多站点（%d 个 Host）" % feat["host_card"])
+    if feat["ua_card"] == 1:
+        score += 5
+    if feat["ua_family"] in ("sdk", "fake-browser", "other"):
+        score += 10
+        signals.append("自动化 UA（%s）" % feat["ua_family"])
+    elif feat["ua_family"] == "browser":
+        score -= 5
+    if feat.get("four_xx", 0) >= 100 and pct4 >= 20:
+        score += 10
+        signals.append("失败请求 %d 次" % int(feat["four_xx"]))
+    if feat["get_ratio"] < 30:
+        score -= 25
+        signals.append("非 GET 为主（业务接口）")
+    if pct4 < 15 and feat["pct2"] >= 80:
+        score -= 20
+        signals.append("低失败率（采集/浏览）")
+    if (feat["coverage"] >= 80 and feat["cv"] is not None
+            and feat["cv"] <= 1.2 and feat["uri_templates"] <= 8):
+        score -= 20
+        signals.append("长期稳定单接口")
+    if feat["uri_templates"] >= 15:
+        score += 10
+        signals.append("路径模板分散（%d 个）" % feat["uri_templates"])
+    if feat["status_known"] < 10:
+        score -= 10
+        signals.append("状态字段缺失")
+    return max(0, min(100, score)), signals
+
+
+def humanize(item):
+    parts = []
+    if item["uri_card"] >= 20:
+        parts.append("访问了 %s 个不同路径" % format(item["uri_card"], ","))
+    if item["four_xx"] >= 20:
+        parts.append("产生 %s 次失败请求（失败率 %.0f%%）"
+                     % (format(int(item["four_xx"]), ","), item["pct4"]))
+    elif item["pct4"] >= 20:
+        parts.append("失败率 %.0f%%" % item["pct4"])
+    if item["coverage"] <= 30 and item["peak_share"] >= 25:
+        parts.append("流量集中在很短的时间内")
+    if item["login_ratio"] >= 20:
+        parts.append("大量请求集中在登录接口")
+    if item["host_card"] >= 2:
+        parts.append("同时访问了 %d 个站点" % item["host_card"])
+    if item["ua_family"] in ("sdk", "fake-browser", "other"):
+        parts.append("使用脚本/非浏览器客户端")
+    behavior = item.get("behavior_type") or "异常自动化"
+    tail = {"目录扫描": "符合目录扫描特征", "短时扫描": "符合短时扫描特征",
+            "登录爆破": "符合登录爆破特征",
+            "伪造爬虫 UA": "伪装成搜索引擎爬虫，且请求大量失败",
+            "采集爬虫": "是采集爬虫", "异常自动化": "行为高度自动化，不像正常用户"}.get(
+                behavior, "行为异常")
+    if parts:
+        return "24 小时内" + "，".join(parts) + "——" + tail + "。"
+    return "24 小时内请求 %d 次，%s。" % (item["total"], tail)
+
+
+def behavior_type(feat, top_ua):
+    tokens = CRAWLER_TOKENS
+    if any(token in (top_ua or "").lower() for token in tokens):
+        return "伪造爬虫 UA" if feat["pct4"] >= 50 else "采集爬虫"
+    if feat["login_ratio"] >= 20:
+        return "登录爆破"
+    if feat["pct4"] >= 50 and feat["uri_card"] >= 20:
+        return "目录扫描"
+    if feat["coverage"] <= 30 and feat["peak_share"] >= 25:
+        return "短时扫描"
+    if feat["uri_templates"] >= 100 and feat["pct2"] >= 70:
+        return "内容抓取"
+    return "异常自动化"
+
+
+def empty_feat(ip):
+    return {"ip": ip, "total": 0, "uri_card": 0, "ua_card": 0, "host_card": 0,
+            "method_card": 0, "get_ratio": 0.0, "login_ratio": 0.0,
+            "pct2": 0.0, "pct3": 0.0, "pct4": 0.0, "pct5": 0.0,
+            "status_known": 0.0, "coverage": 0.0, "cv": None, "peak_share": 0.0,
+            "uri_templates": 0, "ua_family": "", "ua_garbage": False,
+            "top_uri": "", "top_ua": "", "four_xx": 0}
+
+
+class BehaviorAnalyzer:
+    def __init__(self, client, store, fields, filter_str=None, root=None,
+                 interval=900, window=WINDOW, notifier=None):
+        self.client = client
+        self.store = store
+        self.fields = fields
+        self.filter_str = filter_str
+        self.root = root or os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        self.interval = max(120, int(interval or 900))
+        self.window = int(window or WINDOW)
+        self.notifier = notifier
+        self.last_run = None
+        self.last_error = None
+        self.runs = 0
+        self.asn = None
+        try:
+            asn_path = os.path.join(self.root, "data", "geo", "asn.db")
+            if os.path.exists(asn_path):
+                self.asn = AsnDB(asn_path)
+        except Exception:
+            self.asn = None
+        self.geo = None
+        try:
+            from .geo import GeoIP
+            geo_path = os.path.join(self.root, "data", "geo", "geo.db")
+            if os.path.exists(geo_path):
+                self.geo = GeoIP(geo_path)
+        except Exception:
+            self.geo = None
+
+    # ---------- data collection ----------
+    def _details(self, ips):
+        def run(ip):
+            q = 'client_ip:"%s"' % ip
+            detail = {"uri": [], "ua": [], "buckets5m": []}
+            try:
+                st = pivot(self.client, [{"type": "count", "id": "cnt"}],
+                           ["uri"], query=q, limit=40)
+                for row in st.get("rows") or []:
+                    key = row.get("key") or []
+                    value = (row.get("values") or [{}])[0].get("value")
+                    if key:
+                        detail["uri"].append((str(key[0]), int(float(value or 0))))
+            except GraylogError:
+                pass
+            try:
+                st = pivot(self.client, [{"type": "count", "id": "cnt"}],
+                           ["ua"], query=q, limit=3)
+                for row in st.get("rows") or []:
+                    key = row.get("key") or []
+                    value = (row.get("values") or [{}])[0].get("value")
+                    if key:
+                        detail["ua"].append((str(key[0]), int(float(value or 0))))
+            except GraylogError:
+                pass
+            try:
+                hist = self.client.histogram(query=q, range_seconds=self.window,
+                                             interval="5m",
+                                             filter_str=self.filter_str)
+                from .client import normalize_histogram
+                detail["buckets5m"] = [b["count"]
+                                       for b in normalize_histogram(hist)]
+            except GraylogError:
+                pass
+            return ip, detail
+
+        from concurrent.futures import ThreadPoolExecutor
+        out = {}
+        with ThreadPoolExecutor(max_workers=8) as pool:
+            for ip, detail in pool.map(run, ips):
+                out[ip] = detail
+        return out
+
+    def analyze(self):
+        from concurrent.futures import ThreadPoolExecutor
+        fields = self.fields
+        client = self.client
+
+        st = pivot(client,
+                   [{"type": "count", "id": "cnt"},
+                    {"type": "card", "field": "uri", "id": "uri_card"},
+                    {"type": "card", "field": "ua", "id": "ua_card"},
+                    {"type": "card", "field": "host", "id": "host_card"},
+                    {"type": "card", "field": "method", "id": "method_card"}],
+                   ["client_ip"], limit=600)
+        total = rows_to_dict(st, "cnt")
+        uri_card = rows_to_dict(st, "uri_card")
+        ua_card = rows_to_dict(st, "ua_card")
+        host_card = rows_to_dict(st, "host_card")
+        method_card = rows_to_dict(st, "method_card")
+
+        status = {}
+        for cls in ("2*", "3*", "4*", "5*"):
+            status[cls] = rows_to_dict(
+                pivot(client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+                      query="status:%s" % cls, limit=600), "cnt")
+        get_count = rows_to_dict(
+            pivot(client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+                  query="method:GET", limit=600), "cnt")
+        LOGIN_QUERY = ("uri:(\\\\/login* OR \\\\/passport* OR \\\\/captcha* "
+                       "OR \\\\/jsxy\\\\/api\\\\/login*)")
+        login_count = rows_to_dict(
+            pivot(client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+                  query=LOGIN_QUERY, limit=600), "cnt")
+        hourly = hourly_matrix(pivot(
+            client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+            column_groups=[{"type": "time", "field": "timestamp", "interval": "1h"}],
+            limit=600))
+        four_xx = rows_to_dict(
+            pivot(client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+                  query="status:4*", limit=800), "cnt")
+
+        def build(ip, count, uris, uas, hosts, methods, status_map, get_map,
+                  login_map, hourly_map):
+            hour = bucket_stats(hourly_map.get(ip, []))
+            sum_status = sum(status_map[cls].get(ip, 0) for cls in status_map)
+            feat = empty_feat(ip)
+            feat.update({
+                "total": int(count),
+                "uri_card": int(uris.get(ip, 0)),
+                "ua_card": int(uas.get(ip, 0)),
+                "host_card": int(hosts.get(ip, 0)),
+                "method_card": int(methods.get(ip, 0)),
+                "get_ratio": get_map.get(ip, 0) / count * 100 if count else 0.0,
+                "login_ratio": login_map.get(ip, 0) / count * 100 if count else 0.0,
+                "pct2": status_map["2*"].get(ip, 0) / max(1, sum_status) * 100,
+                "pct3": status_map["3*"].get(ip, 0) / max(1, sum_status) * 100,
+                "pct4": status_map["4*"].get(ip, 0) / max(1, sum_status) * 100,
+                "pct5": status_map["5*"].get(ip, 0) / max(1, sum_status) * 100,
+                "status_known": (sum_status / count * 100) if count else 0.0,
+                "coverage": hour["coverage"], "cv": hour["cv"],
+                "peak_share": hour["peak_share"],
+                "four_xx": status_map["4*"].get(ip, 0),
+            })
+            return feat
+
+        base = {}
+        for ip, count in total.items():
+            if count <= 0:
+                continue
+            base[ip] = build(ip, count, uri_card, ua_card, host_card,
+                             method_card, status, get_count, login_count, hourly)
+
+        extra = sorted(set(four_xx) - set(total), key=lambda ip: -four_xx[ip])
+        extra = [ip for ip in extra if not ip.startswith(INTERNAL)][:600]
+        for offset in range(0, len(extra), 80):
+            chunk = extra[offset:offset + 80]
+            chunk_query = " OR ".join('client_ip:"%s"' % ip for ip in chunk)
+            st = pivot(client,
+                       [{"type": "count", "id": "cnt"},
+                        {"type": "card", "field": "uri", "id": "uri_card"},
+                        {"type": "card", "field": "ua", "id": "ua_card"},
+                        {"type": "card", "field": "host", "id": "host_card"},
+                        {"type": "card", "field": "method", "id": "method_card"}],
+                       ["client_ip"], query=chunk_query, limit=200)
+            totals = rows_to_dict(st, "cnt")
+            uris = rows_to_dict(st, "uri_card")
+            uas = rows_to_dict(st, "ua_card")
+            hosts = rows_to_dict(st, "host_card")
+            methods = rows_to_dict(st, "method_card")
+            chunk_status = {}
+            for cls in ("2*", "3*", "4*", "5*"):
+                chunk_status[cls] = rows_to_dict(
+                    pivot(client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+                          query='status:%s AND (%s)' % (cls, chunk_query),
+                          limit=200), "cnt")
+            chunk_get = rows_to_dict(
+                pivot(client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+                      query='method:GET AND (%s)' % chunk_query, limit=200), "cnt")
+            chunk_login = rows_to_dict(
+                pivot(client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+                      query='%s AND (%s)' % (LOGIN_QUERY, chunk_query),
+                      limit=200), "cnt")
+            chunk_hour = hourly_matrix(pivot(
+                client, [{"type": "count", "id": "cnt"}], ["client_ip"],
+                column_groups=[{"type": "time", "field": "timestamp",
+                                "interval": "1h"}], query=chunk_query, limit=200))
+            for cls in chunk_status:
+                status[cls].update(chunk_status[cls])
+            for ip in chunk:
+                if ip not in totals or totals[ip] <= 0:
+                    continue
+                get_count[ip] = chunk_get.get(ip, 0)
+                login_count[ip] = chunk_login.get(ip, 0)
+                base[ip] = build(ip, totals[ip], uris, uas, hosts, methods,
+                                 status, get_count, login_count, chunk_hour)
+
+        shortlist = [ip for ip, feat in base.items()
+                     if not ip.startswith(INTERNAL) and feat["total"] >= 10
+                     and (feat["pct4"] >= 20 or feat["four_xx"] >= 20
+                          or feat["uri_card"] >= 12)]
+        shortlist.sort(key=lambda ip: -base[ip]["four_xx"])
+        shortlist = shortlist[:260]
+        details = self._details(shortlist)
+
+        items = []
+        for ip, feat in base.items():
+            if ip.startswith(INTERNAL):
+                continue
+            detail = details.get(ip) or {}
+            five = bucket_stats(detail.get("buckets5m") or [])
+            uris = detail.get("uri") or []
+            uas = detail.get("ua") or []
+            feat["uri_templates"] = len({uri_template(uri) for uri, _ in uris})
+            feat["top_uri"] = uris[0][0] if uris else ""
+            feat["top_ua"] = uas[0][0] if uas else ""
+            feat["ua_family"] = ua_family(feat["top_ua"])
+            feat["ua_garbage"] = ua_garbage(feat["ua_card"], feat["top_ua"])
+            feat["coverage5m"] = five["coverage"]
+            score, signals = behavior_score(feat)
+            if score < 60 or len(signals) < 2:
+                continue
+            feat["score"] = score
+            feat["signals"] = signals
+            feat["behavior_type"] = behavior_type(feat, feat["top_ua"])
+            info = (self.asn.lookup(ip) if self.asn is not None else None) or {
+                "asn": "", "org": "未知", "brand": "未知",
+                "type": "unknown", "icon": "❓"}
+            shared = TYPE_TO_SHARED.get(info["type"], "unknown")
+            multi = feat["ua_card"] >= 5 or feat["host_card"] >= 10
+            if shared in ("mobile", "education"):
+                verdict = "仅告警（共享出口）"
+                note = ("出口属于%s（%s），大概率有大量用户共享。"
+                        % (info["brand"], SHARE_LABEL[shared]))
+            elif shared == "broadband" and multi:
+                verdict = "仅告警（疑似多用户）"
+                note = ("出口属于%s（宽带 ISP），但该 IP 出现 %d 种 UA / %d 个站点，"
+                        "疑似多用户共享。" % (info["brand"], feat["ua_card"],
+                                              feat["host_card"]))
+            elif shared == "broadband":
+                verdict = "建议封禁" if score >= 75 else "建议人工确认"
+                note = ("出口属于%s（宽带 ISP），但只有 %d 种 UA，更像单台设备。"
+                        % (info["brand"], feat["ua_card"]))
+            elif shared == "hosting":
+                verdict = "建议封禁"
+                note = "出口属于%s（云主机/机房）。" % info["brand"]
+            else:
+                verdict = "建议人工确认"
+                note = "出口属于%s，归属不明确。" % info["brand"]
+            geo = None
+            if self.geo is not None:
+                geo = self.geo.lookup(ip)
+            feat.update({
+                "country_code": (geo or {}).get("country_code", ""),
+                "country_name": (geo or {}).get("country_name", ""),
+                "region": (geo or {}).get("region", ""),
+                "city": (geo or {}).get("city", ""),
+                "verdict": verdict, "share_note": note,
+                "reason_text": humanize(feat),
+                "asn": info["asn"], "asn_org": info["org"],
+                "asn_brand": info["brand"], "asn_type": info["type"],
+                "asn_icon": info["icon"], "shared_type": shared,
+            })
+            items.append(feat)
+        items.sort(key=lambda item: (-item["score"], -item["total"]))
+        return items
+
+    def run_once(self):
+        started = time.time()
+        try:
+            items = self.analyze()
+            self.store.replace_behavior_profiles(items)
+            self.store.sync_fused()
+            self.last_run = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+            self.last_error = None
+            self.runs += 1
+            print("[%s] 行为分析: %d 个违规 IP，%d 个建议封禁，耗时 %.1fs"
+                  % (self.last_run, len(items),
+                     sum(1 for item in items if item["verdict"] == "建议封禁"),
+                     time.time() - started), flush=True)
+            if self.notifier is not None:
+                notify_fused(self.store, self.notifier, self.root)
+            return items
+        except Exception as exc:  # keep the service alive
+            self.last_error = str(exc)
+            print("[行为分析失败] %s" % exc, flush=True)
+            return []
+
+    def run_forever(self):
+        time.sleep(8)
+        while True:
+            self.run_once()
+            time.sleep(self.interval)
+
+
+TYPE_LABEL = {"cloud": "云服务商", "idc": "IDC/机房", "mobile": "移动网络",
+              "broadband": "宽带", "education": "教育网",
+              "enterprise": "企业/政府", "unknown": "未知"}
+
+
+def _local_time(value):
+    import datetime as _dt
+    try:
+        stamp = _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+        return stamp.astimezone().strftime("%m-%d %H:%M")
+    except (TypeError, ValueError):
+        return str(value or "-")
+
+
+_PUSH_ERROR_AT = 0
+
+
+def notify_fused(store, notifier, root=None):
+    """Push a single merged message for IPs due for a reminder (24h)."""
+    due = store.due_pushes(active_hours=24, cooldown_hours=24)
+    if not due:
+        return 0
+    now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+    lines = ["**建议封禁 · 融合检测**", ""]
+    for row in due[:20]:
+        geo = " · ".join(part for part in (row.get("country_name"),
+                                           row.get("region")) if part)
+        lines.append("**%s**%s" % (row["ip"], ("（%s）" % geo) if geo else ""))
+        source = ("双来源" if row.get("sources") == "both"
+                  else ("行为检测" if row.get("sources") == "behavior"
+                        else "特征检测"))
+        brand = row.get("asn_brand") or ""
+        if brand or row.get("asn"):
+            lines.append("归属：%s %s%s"
+                         % (row.get("asn_icon") or "", brand or "未知",
+                            (" · AS" + str(row["asn"])) if row.get("asn") else ""))
+        score = max(float(row.get("feature_score") or 0),
+                    float(row.get("behavior_score") or 0))
+        lines.append("来源：%s · 发现：%s"
+                     % (source, _local_time(row.get("first_banned_at"))))
+        lines.append("评分 %.0f · 请求 %s 次"
+                     % (score, format(int(row.get("total") or 0), ",")))
+        if int(row.get("push_count") or 0) > 0:
+            lines.append("⏰ 该 IP 仍在活动（第 %d 次提醒）"
+                         % (int(row["push_count"]) + 1))
+        if row.get("fused_reason"):
+            lines.append("依据：" + row["fused_reason"])
+        link = notifier.link_for(row["ip"])
+        if link:
+            lines.append("[查看详情](%s)" % link)
+        lines.append("")
+    try:
+        notifier.post("\n".join(lines).strip())
+    except Exception as exc:
+        # 失败时不刷屏：每 5 分钟最多提示一次；下一轮会自动重试
+        global _PUSH_ERROR_AT
+        now = time.time()
+        if now - _PUSH_ERROR_AT > 300:
+            print("推送失败（每 5 分钟最多提示一次）: %s" % exc, flush=True)
+            _PUSH_ERROR_AT = now
+        return 0
+    store.mark_pushed([row["ip"] for row in due[:20]], now)
+    return len(due[:20])

graylog_ops/cli.py

@@ -0,0 +1,289 @@
+"""CLI entry point."""
+
+import argparse
+import sys
+
+from . import __version__, commands
+from .client import GraylogClient
+from .config import ConfigError, load_config
+from .http import GraylogError, HttpClient
+
+EPILOG = """\
+示例:
+  # 1) 先确认连通性、权限和这个版本支持哪些接口
+  graylog-ops --url http://edu-master:9000 --token $GRAYLOG_TOKEN probe
+
+  # 2) 摸清"客户端 IP 到底在哪个字段"（最关键的一步）
+  graylog-ops discover --range 3600 --stream 业务访问日志
+
+  # 3) 按条件看 top IP，判断阈值是否合理
+  graylog-ops top-ips --field client_ip --query 'status:401' --range 300 --size 30
+
+  # 3.5) 生成 profile，然后做封禁判断（只读，dry-run 语义）
+  graylog-ops discover --range 3600 --save-profile profiles/edu.json
+  graylog-ops judge --profile profiles/edu.json --range 300 --allow allowlist.txt --explain 5
+
+  # 4) 把环境信息整体落盘，便于离线分析/复盘
+  graylog-ops snapshot --dir snapshots/$(date +%F) --sample 500
+
+  # 5) 探测未知接口（版本差异时用）
+  graylog-ops raw /api/system/inputs
+  graylog-ops raw /api/search/universal/terms --param field=source --param range=60
+
+  # 6) 一次性产出整套信息（probe/discover/judge/snapshot），便于离线分析
+  graylog-ops collect --dir collect-2026-10-18 --range 3600 --allow allowlist.example.txt
+
+环境变量: GRAYLOG_URL GRAYLOG_TOKEN GRAYLOG_USER GRAYLOG_PASSWORD
+          GRAYLOG_INSECURE GRAYLOG_TIMEOUT GRAYLOG_RETRIES
+配置文件: ~/.config/graylog-ops/config.json 或 ./graylog-ops.json
+"""
+
+
+def _add_common(parser, suppress):
+    """Add connection/output flags.
+
+    With ``suppress=True`` the defaults are SUPPRESS so a sub-parser never
+    clobbers a value that was already parsed by the root parser.
+    """
+
+    def default(value):
+        return argparse.SUPPRESS if suppress else value
+
+    parser.add_argument("--url", metavar="URL", default=default(None),
+                        help="Graylog 地址，如 http://edu-master:9000")
+    parser.add_argument("--token", metavar="TOKEN", default=default(None),
+                        help="API token（推荐；设置后忽略 --user/--password）")
+    parser.add_argument("--user", metavar="USER", default=default(None),
+                        help="用户名（配合 --password）")
+    parser.add_argument("--password", metavar="PASSWORD", default=default(None),
+                        help="密码")
+    parser.add_argument("--insecure", action="store_true",
+                        default=argparse.SUPPRESS if suppress else False,
+                        help="跳过 HTTPS 证书校验")
+    parser.add_argument("--timeout", type=float, metavar="SECONDS", default=default(None),
+                        help="单请求超时，默认 30")
+    parser.add_argument("--retries", type=int, metavar="N", default=default(None),
+                        help="失败重试次数，默认 2")
+    parser.add_argument("--config", metavar="PATH", default=default(None),
+                        help="JSON 配置文件路径")
+    parser.add_argument("--json", action="store_true",
+                        default=argparse.SUPPRESS if suppress else False,
+                        help="输出 JSON（便于管道/脚本消费）")
+    parser.add_argument("--out", metavar="PATH", default=default(None),
+                        help="把结果 JSON 写入文件")
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        default=argparse.SUPPRESS if suppress else False,
+                        help="打印每个 HTTP 请求")
+
+
+def build_parser():
+    common = argparse.ArgumentParser(add_help=False)
+    _add_common(common, suppress=True)
+
+    parser = argparse.ArgumentParser(
+        prog="graylog-ops",
+        description="Graylog 4.0 只读信息采集工具（为 IP 封禁规则摸底）",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=EPILOG,
+    )
+    _add_common(parser, suppress=False)
+    parser.add_argument("--version", action="version", version="graylog-ops %s" % __version__)
+
+    sub = parser.add_subparsers(dest="command", metavar="<command>")
+
+    def add(name, handler, help_text):
+        cmd = sub.add_parser(name, parents=[common], help=help_text, description=help_text)
+        cmd.set_defaults(func=handler)
+        return cmd
+
+    add("probe", commands.cmd_probe, "检查连通性、认证、接口可用性矩阵")
+    add("system", commands.cmd_system, "节点/索引器/版本/license 概要")
+    add("streams", commands.cmd_streams, "列出 stream（分流与权限边界）").add_argument(
+        "--rules", action="store_true", help="同时拉取每个 stream 的规则（N+1 请求）")
+    add("inputs", commands.cmd_inputs, "列出 input 及监听地址")
+    add("fields", commands.cmd_fields, "列出 Graylog 已注册字段与类型")
+    add("indices", commands.cmd_indices, "index set 与索引大小/保留策略")
+    add("alerts", commands.cmd_alerts, "现有事件定义与通知方式（改造前先看）").add_argument(
+        "--limit", type=int, default=None, help="最多返回条数")
+
+    discover = add("discover", commands.cmd_discover, "采样日志，推断真实客户端 IP 字段")
+    discover.add_argument("--stream", help="stream id 或标题子串")
+    discover.add_argument("--query", default="*", help="查询条件，默认 *")
+    discover.add_argument("--range", type=int, default=3600, help="时间窗（秒），默认 3600")
+    discover.add_argument("--sample", type=int, default=500, help="采样消息数，默认 500")
+    discover.add_argument("--save-profile", metavar="PATH",
+                          help="把推断出的字段映射写成 profile，供 judge 使用")
+
+    judge = add("judge", commands.cmd_judge,
+                "把聚合信号判成封禁候选（只读，不执行封禁）")
+    judge.add_argument("--profile", metavar="PATH", help="discover --save-profile 生成的文件")
+    judge.add_argument("--range", type=int, default=300, help="时间窗（秒），默认 300")
+    judge.add_argument("--query", default="*", help="限定查询范围，默认 *")
+    judge.add_argument("--stream", help="stream id 或标题子串")
+    judge.add_argument("--field", help="客户端 IP 字段（覆盖 profile）")
+    judge.add_argument("--status-field", help="状态码字段")
+    judge.add_argument("--uri-field", help="URI 字段")
+    judge.add_argument("--ua-field", help="User-Agent 字段")
+    judge.add_argument("--method-field", help="HTTP 方法字段")
+    judge.add_argument("--allow", action="append", metavar="FILE",
+                       help="白名单文件（IP/CIDR，可重复）")
+    judge.add_argument("--include-internal", action="store_true",
+                       help="允许内网地址进入候选（默认排除）")
+    judge.add_argument("--min-requests", type=int, help="低于该请求数的 IP 不参与判分")
+    judge.add_argument("--min-score", type=float, help="低于该分数的候选不输出")
+    judge.add_argument("--top", type=int, default=30, help="最多输出条数，默认 30")
+    judge.add_argument("--size", type=int, default=200, help="每类聚合返回的 IP 数上限，默认 200")
+    judge.add_argument("--verify", type=int, default=0, metavar="N",
+                       help="对前 N 个候选反查 URI 集中度（识别自动化客户端，每候选多 1 次查询）")
+    judge.add_argument("--explain", type=int, default=0, metavar="N",
+                       help="为前 N 个候选附带样例消息")
+    judge.add_argument("--all", action="store_true", help="输出全部候选（含 ignore）")
+
+    watch = add("watch", commands.cmd_watch,
+                "持续检测恶意公网 IP + 本地 Web 面板（SQLite 持久化）")
+    watch.add_argument("--profile", metavar="PATH", help="字段映射 profile")
+    watch.add_argument("--stream", help="只检测某个 stream")
+    watch.add_argument("--interval", type=int, default=60, help="检测间隔（秒），默认 60")
+    watch.add_argument("--window", type=int, default=900, help="检测时间窗（秒），默认 900")
+    watch.add_argument("--host", default="127.0.0.1", help="面板监听地址")
+    watch.add_argument("--port", type=int, default=8088, help="面板端口，默认 8088")
+    watch.add_argument("--db", default="data/ips.db", help="SQLite 路径")
+    watch.add_argument("--behavior-interval", type=int, default=900,
+                       help="行为检测刷新间隔（秒，默认 900）")
+    watch.add_argument("--no-behavior", action="store_true",
+                       help="关闭行为检测线程，只跑特征检测")
+    watch.add_argument("--notify-config", default="notify.json",
+                       help="新增恶意 IP 的 webhook 配置（不存在则不推送）")
+    watch.add_argument("--notify-test", action="store_true",
+                       help="发送一条测试推送后退出")
+    watch.add_argument("--asn-db", default="data/geo/asn.db",
+                       help="ASN 库路径（空字符串禁用）")
+    watch.add_argument("--geo-db", default="data/geo/geo.db",
+                       help="DB-IP Lite 生成的离线地理库路径（空字符串禁用）")
+    watch.add_argument("--path-prefix", metavar="PATH", default="",
+                       help="只在该路径前缀下提供服务，根路径返回 404（用于对外分享）")
+    watch.add_argument("--forget", type=int, default=7200, help="多久没再出现就从列表移除（秒）")
+    watch.add_argument("--allow", action="append", metavar="FILE", help="白名单文件，可重复")
+    watch.add_argument("--field", help="客户端 IP 字段")
+    watch.add_argument("--status-field", help="状态码字段")
+    watch.add_argument("--uri-field", help="URI 字段")
+    watch.add_argument("--ua-field", help="User-Agent 字段")
+    watch.add_argument("--min-requests", type=int, default=3, help="低于该请求数不参与判定")
+    watch.add_argument("--size", type=int, default=200, help="每类聚合返回的 IP 数上限")
+    watch.add_argument("--verify", type=int, default=8, help="对前 N 个候选做 URI 核验")
+    watch.add_argument("--min-observe-score", type=float, default=10.0,
+                       help="观察级候选低于该分数不入库，默认 10")
+    watch.add_argument("--top", type=int, default=50, help="--once 模式下打印的条数")
+    watch.add_argument("--export", metavar="PATH",
+                       help="把高置信恶意 IP 导出为纯文本名单（排除疑似 NAT）")
+    watch.add_argument("--once", action="store_true", help="只跑一轮后退出（用于离线分析）")
+    watch.add_argument("--warm-limit", type=int, default=5,
+                       help="每轮为多少个恶意 IP 预生成证据快照，默认 5")
+    watch.add_argument("--fix-times", "--fix-first-seen", dest="fix_first_seen",
+                       action="store_true",
+                       help="回查日志，修正首次攻击 / 最近攻击 / 最近出现时间")
+    watch.add_argument("--warm-only", action="store_true",
+                       help="只预热证据快照后退出（配合 --warm-limit）")
+    watch.add_argument("--serve-only", action="store_true",
+                       help="只提供面板，不做检测（用于浏览已有数据库）")
+
+    collect = add("collect", commands.cmd_collect,
+                  "一条命令产出 probe/discover/judge/snapshot 整套信息")
+    collect.add_argument("--dir", help="输出目录，默认 graylog-collect-<时间戳>")
+    collect.add_argument("--range", type=int, default=3600, help="discover 的时间窗（秒）")
+    collect.add_argument("--judge-range", type=int, default=300, help="judge 的时间窗（秒）")
+    collect.add_argument("--stream", help="只针对某个 stream（传给 discover/judge）")
+    collect.add_argument("--allow", action="append", metavar="FILE", help="白名单文件，可重复")
+
+    for name, default_field in (("terms", None), ("top-ips", "client_ip")):
+        cmd = add(name, commands.cmd_terms,
+                  "按字段做 terms 聚合（top-ips 是 field=client_ip 的快捷方式）")
+        cmd.add_argument("--field", required=(default_field is None), default=default_field,
+                         help="聚合字段")
+        cmd.add_argument("--query", default="*", help="查询条件，默认 *")
+        cmd.add_argument("--range", type=int, default=300, help="时间窗（秒），默认 300")
+        cmd.add_argument("--size", type=int, default=50, help="返回条数，默认 50")
+        cmd.add_argument("--min-count", type=int, default=None, help="过滤低于该计数的项")
+        cmd.add_argument("--stream", help="stream id 或标题子串")
+        cmd.add_argument("--from", dest="from_ts", metavar="ISO8601", help="绝对时间起（UTC）")
+        cmd.add_argument("--to", dest="to_ts", metavar="ISO8601", help="绝对时间止（UTC）")
+
+    messages = add("messages", commands.cmd_messages, "拉取原始消息（JSON/表格/JSONL）")
+    messages.add_argument("--query", default="*", help="查询条件，默认 *")
+    messages.add_argument("--range", type=int, default=300, help="时间窗（秒），默认 300")
+    messages.add_argument("--limit", type=int, default=50, help="条数，默认 50")
+    messages.add_argument("--offset", type=int, default=0, help="偏移")
+    messages.add_argument("--fields", help="只取这些字段，逗号分隔")
+    messages.add_argument("--stream", help="stream id 或标题子串")
+    messages.add_argument("--jsonl", action="store_true", help="每行一个 JSON 消息")
+
+    histogram = add("histogram", commands.cmd_histogram, "消息量时间分布（估算阈值/容量）")
+    histogram.add_argument("--query", default="*", help="查询条件，默认 *")
+    histogram.add_argument("--range", type=int, default=3600, help="时间窗（秒），默认 3600")
+    histogram.add_argument("--interval", default="1m", help="时间桶，如 1m/5m/1h")
+    histogram.add_argument("--stream", help="stream id 或标题子串")
+
+    snapshot = add("snapshot", commands.cmd_snapshot, "整体环境快照落到目录（每个资源一个 json）")
+    snapshot.add_argument("--dir", help="输出目录，默认 graylog-snapshot-<时间戳>")
+    snapshot.add_argument("--sample", type=int, default=0, help="附带采样消息条数，默认 0")
+
+    raw = add("raw", commands.cmd_raw, "直接调用任意 API 路径（探索未封装的接口）")
+    raw.add_argument("path", help="如 /api/system/inputs")
+    raw.add_argument("--method", default="GET", help="HTTP 方法，默认 GET")
+    raw.add_argument("--body", metavar="JSON", help="JSON 请求体，或 @文件路径")
+    raw.add_argument("--param", action="append", metavar="K=V", help="查询参数，可重复")
+    raw.add_argument("--text", action="store_true", help="按文本返回（非 JSON）")
+
+    return parser
+
+
+def main(argv=None):
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if not getattr(args, "command", None):
+        parser.print_help()
+        return 2
+
+    try:
+        config = load_config(args)
+    except ConfigError as exc:
+        print("配置错误: %s" % exc, file=sys.stderr)
+        return 2
+
+    http_client = HttpClient(
+        config.url,
+        username=config.credentials[0],
+        password=config.credentials[1],
+        timeout=config.timeout,
+        retries=config.retries,
+        insecure=config.insecure,
+        verbose=bool(getattr(args, "verbose", False)),
+    )
+    ctx = commands.Context(args, config, GraylogClient(http_client))
+
+    try:
+        return args.func(ctx, args) or 0
+    except GraylogError as exc:
+        print("Graylog API 错误: %s" % exc, file=sys.stderr)
+        if exc.status == 401:
+            print(
+                "提示: token 认证是把 token 当用户名、字面量 token 当密码；"
+                "确认 token 未吊销且账号有对应权限",
+                file=sys.stderr,
+            )
+        elif exc.status == 403:
+            print("提示: 账号缺少该资源权限（如 messages:read / searches:*）", file=sys.stderr)
+        elif exc.status == 404:
+            print(
+                "提示: 该版本可能没有这个接口。先跑 `probe` 看可用性，"
+                "或用 `raw /api/...` 直接验证",
+                file=sys.stderr,
+            )
+        elif exc.status in (400, 422):
+            print("提示: 参数不被该版本接受，加 -v 查看实际请求的 URL", file=sys.stderr)
+        return 1
+    except BrokenPipeError:
+        return 0
+    except KeyboardInterrupt:
+        print("", file=sys.stderr)
+        return 130

graylog_ops/client.py

@@ -0,0 +1,463 @@
+"""Thin, defensive wrappers around the Graylog 4.0 REST API.
+
+Every method returns plain Python data (dict/list) that has been normalised
+just enough for callers not to care about cosmetic response differences.
+Anything that is version specific and ambiguous is passed through untouched
+and documented as such.
+
+Endpoint reference for 4.0 lives in the server itself:
+``http://<graylog>:9002/api/api-browser/``
+"""
+
+import json
+
+from .http import GraylogError
+
+
+def as_list(data, *keys):
+    """Best effort extraction of a list from a Graylog response."""
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        for key in keys:
+            value = data.get(key)
+            if isinstance(value, list):
+                return value
+        for value in data.values():
+            if isinstance(value, list):
+                return value
+    return []
+
+
+def dig(data, *keys, **kwargs):
+    default = kwargs.get("default")
+    current = data
+    for key in keys:
+        if not isinstance(current, dict):
+            return default
+        current = current.get(key)
+        if current is None:
+            return default
+    return current
+
+
+def normalize_terms(data):
+    """Turn a terms aggregation response into ``[{"term", "count"}, ...]``.
+
+    Graylog has shipped a couple of shapes for this over the years, so accept
+    the common ones instead of assuming a single layout.
+    """
+    items = as_list(data, "terms", "values", "results")
+    rows = []
+    for item in items:
+        if isinstance(item, dict):
+            term = item.get("term", item.get("value", item.get("name")))
+            count = item.get("count", item.get("doc_count", item.get("hits", 0)))
+            rows.append({"term": term, "count": count})
+        elif isinstance(item, (list, tuple)) and len(item) >= 2:
+            rows.append({"term": item[0], "count": item[1]})
+    return rows
+
+
+def normalize_histogram(data):
+    """Turn a histogram response into ``[{"timestamp", "count"}, ...]``."""
+    if not isinstance(data, dict):
+        return []
+    results = data.get("results")
+    rows = []
+    if isinstance(results, dict):
+        for timestamp, value in results.items():
+            if isinstance(value, dict):
+                count = value.get("count", value.get("doc_count", 0))
+            else:
+                count = value
+            rows.append({"timestamp": timestamp, "count": count or 0})
+    elif isinstance(results, list):
+        for item in results:
+            if isinstance(item, dict):
+                rows.append({
+                    "timestamp": item.get("timestamp", item.get("time")),
+                    "count": item.get("count", 0),
+                })
+    rows.sort(key=lambda row: str(row["timestamp"]))
+    return rows
+
+
+class GraylogClient:
+    def __init__(self, http):
+        self.http = http
+        # Which aggregation backend to use. Graylog 4.0 removed the legacy
+        # /api/search/universal/{terms,histogram,stats} endpoints, so 4.x must
+        # use the views API (POST /api/views/search/sync). 3.x has no views API.
+        self.agg_backend = None
+        # Legacy fallback only: some builds only accept "size" on terms,
+        # others tolerate/require "limit".
+        self.terms_size_param = "size"
+
+    # ---------------------------------------------------------------- system
+    def system_info(self):
+        return self.http.get("/api/system") or {}
+
+    def cluster_nodes(self):
+        return as_list(self.http.get("/api/system/cluster/nodes"), "nodes")
+
+    def indexer_health(self):
+        return self.http.get("/api/system/indexer/cluster/health") or {}
+
+    def indexer_cluster_name(self):
+        return self.http.get("/api/system/indexer/cluster/name") or ""
+
+    def deflector(self):
+        return self.http.get("/api/system/deflector") or {}
+
+    def field_types(self):
+        data = self.http.get("/api/system/fields")
+        if isinstance(data, dict):
+            fields = data.get("fields")
+            if isinstance(fields, list):
+                return fields
+            if isinstance(fields, dict):
+                return [{"name": name, "type": kind} for name, kind in fields.items()]
+        return as_list(data, "fields")
+
+    def license_info(self):
+        return self.http.get("/api/system/license") or {}
+
+    def index_sets(self):
+        data = self.http.get("/api/system/indices/index_sets", {"skip": 0, "limit": 100})
+        return as_list(data, "index_sets")
+
+    def indices(self):
+        data = self.http.get("/api/system/indexer/indices")
+        if isinstance(data, dict):
+            # Strict per-key access: as_list()'s "first list in the dict"
+            # fallback would merge the same list three times here.
+            merged = []
+            for key in ("indices", "closed_indices", "reopened_indices"):
+                value = data.get(key)
+                if isinstance(value, list):
+                    merged.extend(value)
+            return merged
+        return as_list(data)
+
+    # --------------------------------------------------------------- streams
+    def streams(self):
+        return as_list(self.http.get("/api/streams"), "streams")
+
+    def stream(self, stream_id):
+        return self.http.get("/api/streams/%s" % stream_id) or {}
+
+    # ---------------------------------------------------------------- inputs
+    def inputs(self):
+        return as_list(self.http.get("/api/system/inputs"), "inputs")
+
+    def extractors(self, input_id):
+        data = self.http.get("/api/system/inputs/%s/extractors" % input_id)
+        return as_list(data, "extractors")
+
+    # ------------------------------------------------------------- pipelines
+    def pipelines(self):
+        return as_list(self.http.get("/api/system/pipelines/pipeline"), "pipelines")
+
+    def pipeline_rules(self):
+        return as_list(self.http.get("/api/system/pipelines/rule"), "rules")
+
+    # ---------------------------------------------------------------- events
+    def _paged(self, path, key, limit=None, per_page=50, params=None):
+        items = []
+        page = 1
+        while True:
+            query = dict(params or {})
+            query.update({"page": page, "per_page": per_page})
+            data = self.http.get(path, query)
+            batch = as_list(data, key)
+            if not batch:
+                return items
+            items.extend(batch)
+            if limit is not None and len(items) >= limit:
+                return items[:limit]
+            # Trust the server's total when it tells us there is nothing left;
+            # this also stops runaway pagination on servers that ignore `page`.
+            total = dig(data, "total")
+            if isinstance(total, int) and len(items) >= total:
+                return items
+            if len(batch) < per_page:
+                return items
+            page += 1
+            if page > 500:  # defensive stop
+                return items
+
+    def event_definitions(self, limit=None, per_page=50):
+        return self._paged(
+            "/api/events/definitions",
+            "event_definitions",
+            limit=limit,
+            per_page=per_page,
+            params={"sort": "title", "order": "asc"},
+        )
+
+    def notifications(self, limit=None, per_page=50):
+        return self._paged(
+            "/api/events/notifications", "notifications", limit=limit, per_page=per_page
+        )
+
+    def events(self, limit=None, per_page=50):
+        data = self.http.get("/api/events", {"page": 1, "per_page": per_page})
+        items = as_list(data, "events")
+        return items[:limit] if limit else items
+
+    # --------------------------------------------------------------- sidecar
+    def sidecars(self):
+        return as_list(self.http.get("/api/sidecars"), "sidecars")
+
+    # ---------------------------------------------------------------- search
+    @staticmethod
+    def _time_params(range_seconds=None, from_ts=None, to_ts=None):
+        if from_ts and to_ts:
+            return {"from": from_ts, "to": to_ts}
+        return {"range": int(range_seconds if range_seconds is not None else 300)}
+
+    def search_relative(self, query="*", range_seconds=300, limit=100, offset=0,
+                        sort="timestamp:desc", fields=None, filter_str=None):
+        params = {"query": query or "*", "limit": int(limit), "offset": int(offset), "sort": sort}
+        params.update(self._time_params(range_seconds=range_seconds))
+        if fields:
+            params["fields"] = ",".join(fields)
+        if filter_str:
+            params["filter"] = filter_str
+        data = self.http.get("/api/search/universal/relative", params)
+        return self._messages(data)
+
+    def search_absolute(self, query="*", from_ts=None, to_ts=None, limit=100, offset=0,
+                        sort="timestamp:desc", fields=None, filter_str=None):
+        params = {"query": query or "*", "limit": int(limit), "offset": int(offset), "sort": sort}
+        params.update(self._time_params(from_ts=from_ts, to_ts=to_ts))
+        if fields:
+            params["fields"] = ",".join(fields)
+        if filter_str:
+            params["filter"] = filter_str
+        data = self.http.get("/api/search/universal/absolute", params)
+        return self._messages(data)
+
+    @staticmethod
+    def _messages(data):
+        if not isinstance(data, dict):
+            return []
+        messages = data.get("messages")
+        if not isinstance(messages, list):
+            return []
+        result = []
+        for item in messages:
+            if isinstance(item, dict) and isinstance(item.get("message"), dict):
+                message = dict(item["message"])
+                if item.get("index") and "_index" not in message:
+                    message["_index"] = item["index"]
+                result.append(message)
+            elif isinstance(item, dict):
+                result.append(item)
+        return result
+
+    # ------------------------------------------------------- views aggregation
+    BASE_PATH = "/api/views/search"
+
+    @staticmethod
+    def _timerange(range_seconds=None, from_ts=None, to_ts=None):
+        if from_ts and to_ts:
+            return {"type": "absolute", "from": from_ts, "to": to_ts}
+        return {"type": "relative", "range": int(range_seconds or 300)}
+
+    @staticmethod
+    def _stream_filter(filter_str):
+        """Translate the legacy ``streams:<id>[,<id>]`` filter to a views filter."""
+        if not filter_str or not filter_str.startswith("streams:"):
+            return None
+        ids = [part.strip() for part in filter_str[len("streams:"):].split(",") if part.strip()]
+        if not ids:
+            return None
+        return {"type": "or", "filters": [{"type": "stream", "id": sid} for sid in ids]}
+
+    def _views_pivot(self, query, timerange, row_groups, column_groups, series,
+                     sort, filter_str=None, timeout_ms=60000):
+        search_type = {
+            "type": "pivot",
+            "id": "st1",
+            "series": series,
+            "row_groups": row_groups,
+            "column_groups": column_groups,
+            "sort": sort,
+            "rollup": True,
+        }
+        query_object = {
+            "id": "q1",
+            "timerange": timerange,
+            "query": {"type": "elasticsearch", "query_string": query or "*"},
+            "search_types": [search_type],
+        }
+        stream_filter = self._stream_filter(filter_str)
+        if stream_filter:
+            query_object["filter"] = stream_filter
+        body = {"queries": [query_object], "parameters": []}
+        data = self.http.request(
+            "POST", self.BASE_PATH + "/sync",
+            params={"timeout": int(timeout_ms)}, body=body,
+        )
+        results = ((data or {}).get("results") or {}).get("q1") or {}
+        errors = results.get("errors") or []
+        if errors:
+            raise GraylogError(
+                "views search error: %s" % json.dumps(errors, ensure_ascii=False)[:400],
+                path=self.BASE_PATH + "/sync", method="POST",
+            )
+        search_types = results.get("search_types") or {}
+        if isinstance(search_types, list):
+            search_types = {item.get("id"): item for item in search_types
+                            if isinstance(item, dict)}
+        return search_types.get("st1") or {}
+
+    def _terms_views(self, field, query, size, range_seconds, from_ts, to_ts, filter_str):
+        search_type = self._views_pivot(
+            query=query,
+            timerange=self._timerange(range_seconds, from_ts, to_ts),
+            row_groups=[{"type": "values", "field": field, "limit": int(size)}],
+            column_groups=[],
+            series=[{"type": "count", "id": "c1"}],
+            sort=[{"type": "pivot", "field": "c1", "direction": "Descending"}],
+            filter_str=filter_str,
+        )
+        terms = []
+        for row in search_type.get("rows") or []:
+            key = row.get("key") or []
+            if not key:
+                continue  # rollup row (total)
+            values = row.get("values") or []
+            count = values[0].get("value") if values else 0
+            terms.append({"term": key[0], "count": count})
+        return {
+            "field": field,
+            "terms": terms,
+            "total": search_type.get("total", sum(row["count"] or 0 for row in terms)),
+        }
+
+    def _histogram_views(self, query, range_seconds, interval, filter_str, from_ts, to_ts):
+        search_type = self._views_pivot(
+            query=query,
+            timerange=self._timerange(range_seconds, from_ts, to_ts),
+            row_groups=[],
+            column_groups=[{"type": "time", "field": "timestamp", "interval": interval}],
+            series=[{"type": "count", "id": "c1"}],
+            sort=[],
+            filter_str=filter_str,
+        )
+        results = {}
+        for row in search_type.get("rows") or []:
+            for item in row.get("values") or []:
+                key = item.get("key") or []
+                # Column keys are [timestamp, series_id]; the pivot rollup row
+                # only carries [series_id] and must not become a time bucket.
+                if len(key) < 2:
+                    continue
+                results[key[0]] = {"count": item.get("value", 0)}
+        return {"results": results, "interval": interval}
+
+    # ------------------------------------------------------------- aggregation
+    def search_views_messages(self, query="*", range_seconds=300, limit=5,
+                              offset=0, filter_str=None, from_ts=None, to_ts=None,
+                              timeout_ms=60000, sort_order="DESC"):
+        """Message search through the views API (the legacy endpoint is flaky
+        on long windows with complex queries and drops the connection)."""
+        search_type = {
+            "type": "messages",
+            "id": "st1",
+            "limit": int(limit),
+            "offset": int(offset),
+            "sort": [{"field": "timestamp",
+                      "order": "ASC" if str(sort_order).upper() == "ASC" else "DESC"}],
+        }
+        query_object = {
+            "id": "q1",
+            "timerange": self._timerange(range_seconds, from_ts, to_ts),
+            "query": {"type": "elasticsearch", "query_string": query or "*"},
+            "search_types": [search_type],
+        }
+        stream_filter = self._stream_filter(filter_str)
+        if stream_filter:
+            query_object["filter"] = stream_filter
+        body = {"queries": [query_object], "parameters": []}
+        data = self.http.request(
+            "POST", self.BASE_PATH + "/sync",
+            params={"timeout": int(timeout_ms)}, body=body,
+        )
+        results = ((data or {}).get("results") or {}).get("q1") or {}
+        errors = results.get("errors") or []
+        if errors:
+            raise GraylogError(
+                "views message search error: %s" % json.dumps(errors, ensure_ascii=False)[:300],
+                path=self.BASE_PATH + "/sync", method="POST",
+            )
+        search_types = results.get("search_types") or {}
+        if isinstance(search_types, list):
+            search_types = {item.get("id"): item for item in search_types
+                            if isinstance(item, dict)}
+        search_type_result = search_types.get("st1") or {}
+        return self._messages({"messages": search_type_result.get("messages") or []})
+
+    def terms(self, field, query="*", size=50, range_seconds=300,
+              from_ts=None, to_ts=None, filter_str=None):
+        if self.agg_backend in (None, "views"):
+            try:
+                data = self._terms_views(field, query, size, range_seconds,
+                                         from_ts, to_ts, filter_str)
+                self.agg_backend = "views"
+                return data
+            except GraylogError as exc:
+                if exc.status != 404 or self.agg_backend == "views":
+                    raise
+                self.agg_backend = "legacy"
+        return self._terms_legacy(field, query, size, range_seconds,
+                                  from_ts, to_ts, filter_str)
+
+    def _terms_legacy(self, field, query, size, range_seconds, from_ts, to_ts, filter_str):
+        params = {"query": query or "*", "field": field}
+        params.update(self._time_params(range_seconds, from_ts, to_ts))
+        if filter_str:
+            params["filter"] = filter_str
+        params[self.terms_size_param] = int(size)
+        try:
+            return self.http.get("/api/search/universal/terms", params)
+        except GraylogError as exc:
+            # Older/newer builds disagree on the parameter name; retry once.
+            if exc.status in (400, 422) and self.terms_size_param == "size":
+                params.pop("size", None)
+                params["limit"] = int(size)
+                data = self.http.get("/api/search/universal/terms", params)
+                self.terms_size_param = "limit"
+                return data
+            raise
+
+    def histogram(self, query="*", range_seconds=3600, interval="1m",
+                  filter_str=None, from_ts=None, to_ts=None):
+        if self.agg_backend in (None, "views"):
+            try:
+                data = self._histogram_views(query, range_seconds, interval,
+                                             filter_str, from_ts, to_ts)
+                self.agg_backend = "views"
+                return data
+            except GraylogError as exc:
+                if exc.status != 404 or self.agg_backend == "views":
+                    raise
+                self.agg_backend = "legacy"
+        params = {"query": query or "*", "interval": interval}
+        params.update(self._time_params(range_seconds, from_ts, to_ts))
+        if filter_str:
+            params["filter"] = filter_str
+        return self.http.get("/api/search/universal/histogram", params)
+
+    def stats(self, field, query="*", range_seconds=300, stats=None,
+              filter_str=None, from_ts=None, to_ts=None):
+        params = {"query": query or "*", "field": field}
+        params.update(self._time_params(range_seconds, from_ts, to_ts))
+        if stats:
+            params["stats"] = ",".join(stats)
+        if filter_str:
+            params["filter"] = filter_str
+        return self.http.get("/api/search/universal/stats", params)

graylog_ops/config.py

@@ -0,0 +1,161 @@
+"""Configuration resolution.
+
+Precedence (highest wins):
+  1. command line flags
+  2. environment variables (GRAYLOG_URL, GRAYLOG_TOKEN, ...)
+  3. JSON config file (--config, ./graylog-ops.json,
+     ~/.config/graylog-ops/config.json)
+
+File format::
+
+    {
+      "url": "http://edu-master:9000",
+      "token": "xxxxxxxx",
+      "insecure": false,
+      "timeout": 30
+    }
+"""
+
+import json
+import os
+
+ENV_KEYS = {
+    "url": "GRAYLOG_URL",
+    "token": "GRAYLOG_TOKEN",
+    "user": "GRAYLOG_USER",
+    "password": "GRAYLOG_PASSWORD",
+    "insecure": "GRAYLOG_INSECURE",
+    "timeout": "GRAYLOG_TIMEOUT",
+    "retries": "GRAYLOG_RETRIES",
+}
+
+DEFAULT_CONFIG_PATHS = (
+    os.path.expanduser("~/.config/graylog-ops/config.json"),
+    "graylog-ops.json",
+)
+
+
+class ConfigError(Exception):
+    """User facing configuration problem."""
+
+
+def _truthy(value):
+    return str(value).strip().lower() in ("1", "true", "yes", "on", "y")
+
+
+def _read_file(path):
+    try:
+        with open(path, "r", encoding="utf-8") as handle:
+            data = json.load(handle)
+    except OSError as exc:
+        raise ConfigError("cannot read config %s: %s" % (path, exc))
+    except ValueError as exc:
+        raise ConfigError("config %s is not valid JSON: %s" % (path, exc))
+    if not isinstance(data, dict):
+        raise ConfigError("config %s must contain a JSON object" % path)
+    return data
+
+
+class Config:
+    def __init__(self, url, token=None, user=None, password=None,
+                 insecure=False, timeout=30.0, retries=2, source=None):
+        self.url = url
+        self.token = token
+        self.user = user
+        self.password = password
+        self.insecure = bool(insecure)
+        self.timeout = float(timeout)
+        self.retries = int(retries)
+        self.source = source
+
+    @property
+    def credentials(self):
+        """(username, password) pair for HTTP Basic auth."""
+        if self.token:
+            # Graylog token auth: username is the token, password is "token".
+            return self.token, "token"
+        return self.user, self.password
+
+    @property
+    def auth_kind(self):
+        return "token" if self.token else "user/password"
+
+
+def _pick(values, key, default=None):
+    value = values.get(key, default)
+    return value
+
+
+def load_config(args):
+    """Build a Config from CLI args, env vars and the config file."""
+    values = {}
+    chosen = None
+
+    explicit = getattr(args, "config", None)
+    if explicit:
+        if not os.path.exists(explicit):
+            raise ConfigError("config file not found: %s" % explicit)
+        values.update(_read_file(explicit))
+        chosen = explicit
+    else:
+        for candidate in DEFAULT_CONFIG_PATHS:
+            if os.path.exists(candidate):
+                values.update(_read_file(candidate))
+                chosen = candidate
+                break
+
+    for key, env_name in ENV_KEYS.items():
+        env_value = os.environ.get(env_name)
+        if env_value is not None and env_value.strip() != "":
+            values[key] = env_value
+
+    for key in ("url", "token", "user", "password", "timeout", "retries"):
+        flag_value = getattr(args, key, None)
+        if flag_value is not None and flag_value != "":
+            values[key] = flag_value
+    if getattr(args, "insecure", False):
+        values["insecure"] = True
+
+    url = values.get("url")
+    if not url:
+        raise ConfigError(
+            "no Graylog URL: pass --url, set GRAYLOG_URL, or add \"url\" to the config file"
+        )
+    url = str(url).strip()
+    if "://" not in url:
+        url = "http://" + url
+
+    token = (values.get("token") or "").strip() or None
+    user = (values.get("user") or "").strip() or None
+    password = values.get("password")
+    if not token and not (user and password):
+        raise ConfigError(
+            "no credentials: pass --token or --user/--password "
+            "(env GRAYLOG_TOKEN / GRAYLOG_USER + GRAYLOG_PASSWORD)"
+        )
+
+    timeout = values.get("timeout", 30)
+    retries = values.get("retries", 2)
+    try:
+        timeout = float(timeout)
+    except (TypeError, ValueError):
+        raise ConfigError("timeout must be a number, got %r" % (timeout,))
+    try:
+        retries = int(retries)
+    except (TypeError, ValueError):
+        raise ConfigError("retries must be an integer, got %r" % (retries,))
+
+    insecure = values.get("insecure", False)
+    if isinstance(insecure, str):
+        insecure = _truthy(insecure)
+
+    return Config(
+        url=url,
+        token=token,
+        user=user,
+        password=password,
+        insecure=bool(insecure),
+        timeout=timeout,
+        retries=retries,
+        source=chosen,
+    )

graylog_ops/detect.py

@@ -0,0 +1,300 @@
+"""Field heuristics: find the field that actually holds the client IP.
+
+The ban pipeline is only as good as the field it keys on. In a typical
+deployment behind a reverse proxy / CDN, the field you *think* is the client
+IP is the proxy's address:
+
+* ``source``         -- the host that shipped the log line to Graylog
+* ``gl2_remote_ip``  -- the host that opened the GELF/Beats connection
+* ``remote_addr``    -- could be the proxy if XFF is not parsed
+
+So instead of assuming, sample real messages and score every field by
+"how often does this field hold an IP address" plus "does the field name look
+like a client address".
+"""
+
+import ipaddress
+import re
+
+# Field names that strongly suggest a client address.
+STRONG_HINT = re.compile(
+    r"(?i)client[_. -]?ip|remote[_. -]?addr|real[_. -]?ip|"
+    r"(^|[_. -])ip([_. -]|$)|x[_. -]?forwarded[_. -]?for|"
+    r"(^|[_. -])xff([_. -]|$)|src[_. -]?ip|c[_. -]?ip"
+)
+
+WEAK_HINT = re.compile(
+    r"(?i)addr|host|client|proxy|peer|origin|caller|visitor|remote|forward|endpoint"
+)
+
+# Graylog internals that are never the client IP (gl2_remote_ip is the shipper).
+INTERNAL_PREFIXES = ("gl2_", "_")
+INTERNAL_NAMES = ("timestamp", "streams", "message")
+
+HTTP_HINTS = {
+    "status": re.compile(r"(?i)^(http[_. ]?)?status([_. ]?code)?$|^response([_. ]?(code|status))?$|^code$"),
+    "uri": re.compile(r"(?i)(request[_. ]?)?uri$|^url$|^path$|^request$|^endpoint$|^target$"),
+    "method": re.compile(r"(?i)^(http[_. ]?)?method$|^verb$"),
+    "user_agent": re.compile(r"(?i)user[_. -]?agent|^ua$|^agent$"),
+    "host": re.compile(r"(?i)^(http[_. ]?)?host$|^vhost$|^domain$|^server_name$"),
+    "bytes": re.compile(r"(?i)bytes|^size$|^body_bytes_sent$"),
+    "referer": re.compile(r"(?i)refer(r)?er$"),
+    "duration": re.compile(r"(?i)request[_. ]?time|response[_. ]?time|duration|latency|elapsed"),
+}
+
+# Deliberately not using ``ipaddress.is_private``: that property also covers
+# documentation ranges (192.0.2.0/24, 198.51.100.0/24, 203.0.113.0/24) and
+# other special-use blocks, which would make every public test address look
+# like an internal proxy. For ban decisions "internal" means RFC1918 plus
+# loopback, link-local and CGNAT.
+_INTERNAL_V4 = tuple(ipaddress.ip_network(cidr) for cidr in (
+    "10.0.0.0/8",
+    "172.16.0.0/12",
+    "192.168.0.0/16",
+    "100.64.0.0/10",   # CGNAT
+    "127.0.0.0/8",
+    "169.254.0.0/16",
+))
+_INTERNAL_V6 = tuple(ipaddress.ip_network(cidr) for cidr in (
+    "::1/128",
+    "fc00::/7",        # unique local
+    "fe80::/10",       # link local
+))
+
+
+def is_internal(ip):
+    networks = _INTERNAL_V4 if ip.version == 4 else _INTERNAL_V6
+    return any(ip in network for network in networks)
+
+
+def as_ip(value):
+    """Return an ``ipaddress`` object if *value* looks like a bare IP.
+
+    Accepts ``1.2.3.4``, ``2001:db8::1``, ``[2001:db8::1]:443`` and
+    ``1.2.3.4:5678``. Rejects CIDRs, hostnames and arbitrary integers.
+    """
+    if not isinstance(value, str):
+        return None
+    text = value.strip().strip('"')
+    if not text or len(text) > 64:
+        return None
+    try:
+        return ipaddress.ip_address(text)
+    except ValueError:
+        pass
+    if text.startswith("[") and "]" in text:
+        try:
+            return ipaddress.ip_address(text[1:text.index("]")])
+        except ValueError:
+            return None
+    if text.count(":") == 1 and "." in text:
+        host = text.split(":", 1)[0]
+        try:
+            return ipaddress.ip_address(host)
+        except ValueError:
+            return None
+    return None
+
+
+def _empty(name):
+    return {
+        "name": name,
+        "present": 0,
+        "ip_messages": 0,
+        "ip_values": 0,
+        "distinct": set(),
+        "distinct_ip": set(),
+        "private": 0,
+        "versions": set(),
+        "examples": [],
+    }
+
+
+def _analyse(messages):
+    total = len(messages)
+    stats = {}
+    embedded = 0
+    for message in messages:
+        embedded_here = False
+        for name, raw in message.items():
+            values = raw if isinstance(raw, list) else [raw]
+            entry = stats.get(name)
+            if entry is None:
+                entry = stats[name] = _empty(name)
+            emitted = False
+            has_ip = False
+            for value in values:
+                if value is None or value == "":
+                    continue
+                text = str(value)[:160]
+                entry["distinct"].add(text)
+                emitted = True
+                ip = as_ip(value)
+                if ip is not None:
+                    has_ip = True
+                    entry["ip_values"] += 1
+                    entry["distinct_ip"].add(str(ip))
+                    entry["versions"].add(ip.version)
+                    if is_internal(ip):
+                        entry["private"] += 1
+                    text_ip = str(ip)
+                    if len(entry["examples"]) < 3 and text_ip not in entry["examples"]:
+                        entry["examples"].append(text_ip)
+                    if name == "message":
+                        embedded_here = True
+            if emitted:
+                entry["present"] += 1
+            if has_ip:
+                entry["ip_messages"] += 1
+        if embedded_here:
+            embedded += 1
+    return stats, total, embedded
+
+
+def _score(name, entry, total):
+    present = max(entry["present"], 1)
+    ip_rate = entry["ip_messages"] / present
+    presence = entry["present"] / max(total, 1)
+    score = 60.0 * ip_rate + 5.0 * presence
+    notes = []
+
+    if STRONG_HINT.search(name):
+        score += 30
+    elif WEAK_HINT.search(name):
+        score += 10
+
+    if name == "source":
+        score -= 20
+        notes.append("`source` 是日志发送方（shipper/反代），通常不是真实客户端")
+    elif name == "gl2_remote_ip":
+        score -= 25
+        notes.append("`gl2_remote_ip` 是向 Graylog 发送消息的对端地址，不是访问者")
+    return score, notes
+
+
+def _confidence(best):
+    if not best:
+        return "none"
+    if best["score"] >= 90 and best["ip_rate"] >= 0.95:
+        return "high"
+    if best["score"] >= 60:
+        return "medium"
+    return "low"
+
+
+def analyse_messages(messages):
+    """Build the field report used by ``graylog-ops discover``."""
+    stats, total, embedded = _analyse(messages)
+    if total == 0:
+        return {
+            "sampled": 0,
+            "fields": [],
+            "candidates": [],
+            "http_fields": {},
+            "embedded_ip_in_message_rate": 0.0,
+            "recommendation": {
+                "field": None,
+                "confidence": "none",
+                "notes": ["没有采到消息：检查时间范围、查询条件、stream 过滤或权限"],
+                "warnings": [],
+            },
+            "sample_message": None,
+        }
+
+    fields = []
+    candidates = []
+    for name, entry in stats.items():
+        if entry["present"] == 0:
+            continue
+        ip_rate = entry["ip_messages"] / entry["present"]
+        row = {
+            "name": name,
+            "present": entry["present"],
+            "presence": round(entry["present"] / total, 4),
+            "distinct": len(entry["distinct"]),
+            "distinct_ips": len(entry["distinct_ip"]),
+            "ip_rate": round(ip_rate, 4),
+            "examples": entry["examples"],
+        }
+        fields.append(row)
+
+        if name in INTERNAL_NAMES or name.startswith(INTERNAL_PREFIXES):
+            continue
+        if ip_rate <= 0:
+            continue
+        score, notes = _score(name, entry, total)
+        private_ratio = (entry["private"] / entry["ip_values"]) if entry["ip_values"] else 0.0
+        candidates.append({
+            "field": name,
+            "score": round(score, 1),
+            "ip_rate": round(ip_rate, 4),
+            "presence": round(entry["present"] / total, 4),
+            "distinct_ips": len(entry["distinct_ip"]),
+            "private_ratio": round(private_ratio, 4),
+            "versions": sorted(entry["versions"]),
+            "examples": entry["examples"],
+            "notes": notes,
+        })
+
+    fields.sort(key=lambda row: (-row["present"], row["name"]))
+    candidates.sort(key=lambda row: (-row["score"], -row["distinct_ips"]))
+
+    http_fields = {kind: [] for kind in HTTP_HINTS}
+    for name in stats:
+        for kind, pattern in HTTP_HINTS.items():
+            if pattern.search(name):
+                http_fields[kind].append(name)
+    http_fields = {kind: names for kind, names in http_fields.items() if names}
+
+    best = candidates[0] if candidates else None
+    source_entry = stats.get("source")
+    warnings = []
+    notes = []
+
+    if best:
+        notes.append(
+            "字段 `%s` 的 IP 占比 %.0f%%，样例 %s"
+            % (best["field"], best["ip_rate"] * 100, ", ".join(best["examples"][:3]) or "-")
+        )
+        if source_entry and len(source_entry["distinct"]) <= 2 and best["distinct_ips"] > 2:
+            warnings.append(
+                "`source` 只有 %d 个不同取值，而 `%s` 有 %d 个：`source` 是采集器/反代，"
+                "封禁必须用 `%s`" % (len(source_entry["distinct"]), best["field"],
+                                     best["distinct_ips"], best["field"])
+            )
+        if best["private_ratio"] >= 0.5:
+            warnings.append(
+                "`%s` 中 %.0f%% 是内网地址（私有/回环），可能仍是 LB 或代理地址，"
+                "需要确认 X-Forwarded-For 是否已在采集侧解析"
+                % (best["field"], best["private_ratio"] * 100)
+            )
+    else:
+        notes.append("没有发现任何字段存放纯 IP 值")
+
+    # Only worth flagging when there is no usable field: then the IP lives in
+    # the free-text `message` and has to be extracted server side.
+    if embedded / max(total, 1) >= 0.5 and not best:
+        warnings.append(
+            "%.0f%% 的消息在 `message` 文本里内嵌了 IP。要么在 input 上加 extractor / "
+            "pipeline 解析成独立字段，要么在采集端（nginx log_format）直接输出 client_ip 字段"
+            % (100.0 * embedded / total)
+        )
+
+    # HTTP-ish fields help with the next phase (per-URI thresholds), surface them.
+    if not best and http_fields:
+        warnings.append("只找到 HTTP 相关字段，没有 IP 字段，见 http_fields")
+
+    return {
+        "sampled": total,
+        "fields": fields,
+        "candidates": candidates,
+        "http_fields": http_fields,
+        "embedded_ip_in_message_rate": round(embedded / total, 4),
+        "recommendation": {
+            "field": best["field"] if best else None,
+            "confidence": _confidence(best),
+            "notes": notes,
+            "warnings": warnings,
+        },
+        "sample_message": messages[0] if messages else None,
+    }

graylog_ops/geo.py

@@ -0,0 +1,138 @@
+"""Offline GeoIP lookup backed by the free DB-IP Lite CSV dumps.
+
+The city dump has no country name, so a small code -> name table is shipped
+here; anything unknown falls back to the ISO code.
+"""
+import csv
+import ipaddress
+import os
+import sqlite3
+
+COUNTRY_CN = {
+    "CN": "中国", "HK": "中国香港", "TW": "中国台湾", "MO": "中国澳门",
+    "US": "美国", "CA": "加拿大", "MX": "墨西哥", "BR": "巴西", "AR": "阿根廷",
+    "CL": "智利", "CO": "哥伦比亚", "PE": "秘鲁", "VE": "委内瑞拉",
+    "GB": "英国", "IE": "爱尔兰", "FR": "法国", "DE": "德国", "NL": "荷兰",
+    "BE": "比利时", "LU": "卢森堡", "CH": "瑞士", "AT": "奥地利", "IT": "意大利",
+    "ES": "西班牙", "PT": "葡萄牙", "GR": "希腊", "PL": "波兰", "CZ": "捷克",
+    "SK": "斯洛伐克", "HU": "匈牙利", "RO": "罗马尼亚", "BG": "保加利亚",
+    "HR": "克罗地亚", "RS": "塞尔维亚", "SI": "斯洛文尼亚", "LT": "立陶宛",
+    "LV": "拉脱维亚", "EE": "爱沙尼亚", "FI": "芬兰", "SE": "瑞典",
+    "NO": "挪威", "DK": "丹麦", "IS": "冰岛", "UA": "乌克兰", "BY": "白俄罗斯",
+    "RU": "俄罗斯", "TR": "土耳其", "IL": "以色列", "SA": "沙特阿拉伯",
+    "AE": "阿联酋", "QA": "卡塔尔", "KW": "科威特", "IR": "伊朗", "IQ": "伊拉克",
+    "EG": "埃及", "ZA": "南非", "NG": "尼日利亚", "KE": "肯尼亚", "MA": "摩洛哥",
+    "ET": "埃塞俄比亚", "IN": "印度", "PK": "巴基斯坦", "BD": "孟加拉国",
+    "LK": "斯里兰卡", "NP": "尼泊尔", "MM": "缅甸", "KH": "柬埔寨", "LA": "老挝",
+    "TH": "泰国", "VN": "越南", "MY": "马来西亚", "SG": "新加坡", "ID": "印度尼西亚",
+    "PH": "菲律宾", "JP": "日本", "KR": "韩国", "MN": "蒙古", "KZ": "哈萨克斯坦",
+    "UZ": "乌兹别克斯坦", "GE": "格鲁吉亚", "AM": "亚美尼亚", "AZ": "阿塞拜疆",
+    "AU": "澳大利亚", "NZ": "新西兰", "ZZ": "未知",
+}
+
+CN_REGION = {
+    "Beijing": "北京", "Tianjin": "天津", "Hebei": "河北", "Shanxi": "山西",
+    "Inner Mongolia": "内蒙古", "Liaoning": "辽宁", "Jilin": "吉林",
+    "Heilongjiang": "黑龙江", "Shanghai": "上海", "Jiangsu": "江苏",
+    "Zhejiang": "浙江", "Anhui": "安徽", "Fujian": "福建", "Jiangxi": "江西",
+    "Shandong": "山东", "Henan": "河南", "Hubei": "湖北", "Hunan": "湖南",
+    "Guangdong": "广东", "Guangxi": "广西", "Hainan": "海南", "Chongqing": "重庆",
+    "Sichuan": "四川", "Guizhou": "贵州", "Yunnan": "云南", "Tibet": "西藏",
+    "Shaanxi": "陕西", "Gansu": "甘肃", "Qinghai": "青海", "Ningxia": "宁夏",
+    "Xinjiang": "新疆", "Taiwan": "台湾", "Hong Kong": "香港", "Macau": "澳门",
+}
+
+
+def flag(cc):
+    if not cc or len(cc) != 2 or not cc.isalpha() or cc == "ZZ":
+        return ""
+    return "".join(chr(0x1F1E6 + ord(char) - 65) for char in cc.upper())
+
+
+def country_name(cc):
+    return COUNTRY_CN.get(cc, cc or "")
+
+
+def region_name(cc, region):
+    if cc == "CN" and region:
+        return CN_REGION.get(region, region)
+    return region or ""
+
+
+def build_geo_db(city_csv, out_db, country_csv=None):
+    """Import the DB-IP city CSV (IPv4 only) into a small SQLite database."""
+    os.makedirs(os.path.dirname(out_db) or ".", exist_ok=True)
+    conn = sqlite3.connect(out_db)
+    conn.execute("DROP TABLE IF EXISTS geo")
+    conn.execute("CREATE TABLE geo (start INTEGER PRIMARY KEY, end INTEGER, "
+                 "cc TEXT, region TEXT, city TEXT)")
+    batch = []
+    total = 0
+
+    def flush():
+        nonlocal batch
+        if batch:
+            conn.executemany("INSERT OR REPLACE INTO geo VALUES (?,?,?,?,?)", batch)
+            batch = []
+
+    with open(city_csv, newline="", encoding="utf-8", errors="replace") as fh:
+        for row in csv.reader(fh):
+            if len(row) < 6:
+                continue
+            start, end, cc = row[0], row[1], row[3]
+            if ":" in start or ":" in end or not cc:
+                continue
+            try:
+                first = int(ipaddress.IPv4Address(start))
+                last = int(ipaddress.IPv4Address(end))
+            except ValueError:
+                continue
+            batch.append((first, last, cc, row[4], row[5]))
+            total += 1
+            if len(batch) >= 100000:
+                flush()
+                conn.commit()
+    flush()
+    conn.commit()
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.close()
+    return total
+
+
+class GeoIP:
+    """Read-only lookup over the generated geo.db."""
+
+    def __init__(self, path):
+        if not path or not os.path.exists(path):
+            raise FileNotFoundError(path)
+        self.conn = sqlite3.connect("file:%s?mode=ro" % path, uri=True,
+                                    check_same_thread=False)
+
+    def lookup(self, ip):
+        try:
+            value = int(ipaddress.ip_address(ip))
+        except ValueError:
+            return None
+        row = self.conn.execute(
+            "SELECT end, cc, region, city FROM geo WHERE start <= ? "
+            "ORDER BY start DESC LIMIT 1", (value,)).fetchone()
+        if not row or row[0] < value:
+            return None
+        return {"country_code": row[1], "country_name": country_name(row[1]),
+                "region": region_name(row[1], row[2]), "city": row[3] or ""}
+
+    def close(self):
+        try:
+            self.conn.close()
+        except sqlite3.Error:
+            pass
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--city-csv", default="data/geo/dbip-city-lite-2026-10.csv")
+    parser.add_argument("--out", default="data/geo/geo.db")
+    args = parser.parse_args()
+    count = build_geo_db(args.city_csv, args.out)
+    print("imported %d IPv4 ranges -> %s" % (count, args.out))

graylog_ops/http.py

@@ -0,0 +1,214 @@
+"""Minimal HTTP layer for the Graylog REST API.
+
+Deliberately built on urllib so the tool has no third-party dependencies and
+can be copied onto old jump hosts. Handles the three things Graylog's API
+demands that plain urllib does not do for you:
+
+* HTTP Basic auth (API token used as username with the literal password
+  ``token``, or a real username/password pair),
+* the ``X-Requested-By`` header required by Graylog's CSRF protection on all
+  non-GET requests,
+* bounded retries with backoff for 429/5xx and transient connection errors.
+"""
+
+import base64
+import json
+import random
+import socket
+import ssl
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+USER_AGENT = "graylog-ops/0.1 (read-only recon)"
+RETRY_STATUS = frozenset((429, 500, 502, 503, 504))
+
+
+class GraylogError(Exception):
+    """Any failure talking to the Graylog API."""
+
+    def __init__(self, message, status=None, body=None, path=None, method=None):
+        super().__init__(message)
+        self.message = message
+        self.status = status
+        self.body = body
+        self.path = path
+        self.method = method
+
+    def __str__(self):
+        parts = []
+        if self.status is not None:
+            parts.append("HTTP %s" % self.status)
+        if self.method and self.path:
+            parts.append("%s %s" % (self.method, self.path))
+        parts.append(self.message)
+        return " | ".join(parts)
+
+
+def _basic_auth(username, password):
+    raw = ("%s:%s" % (username, password)).encode("utf-8")
+    return "Basic " + base64.b64encode(raw).decode("ascii")
+
+
+def _error_detail(body):
+    """Pull a human readable message out of a Graylog/ES error body."""
+    if not body:
+        return ""
+    try:
+        data = json.loads(body)
+    except (TypeError, ValueError):
+        return body.strip()[:300]
+    if isinstance(data, dict):
+        for key in ("message", "error", "description"):
+            value = data.get(key)
+            if isinstance(value, str) and value.strip():
+                return value.strip()
+        if isinstance(data.get("error"), dict):
+            reason = data["error"].get("reason")
+            if isinstance(reason, str):
+                return reason
+    return body.strip()[:300]
+
+
+class HttpClient:
+    def __init__(
+        self,
+        base_url,
+        username=None,
+        password=None,
+        timeout=30.0,
+        retries=2,
+        insecure=False,
+        verbose=False,
+    ):
+        self.base_url = base_url.rstrip("/")
+        self.timeout = float(timeout)
+        self.retries = max(0, int(retries))
+        self.verbose = verbose
+        self.auth = None
+        if username is not None:
+            self.auth = _basic_auth(username, "" if password is None else password)
+
+        handlers = []
+        if insecure:
+            context = ssl.create_default_context()
+            context.check_hostname = False
+            context.verify_mode = ssl.CERT_NONE
+            handlers.append(urllib.request.HTTPSHandler(context=context))
+        # build_opener keeps the default ProxyHandler, so HTTP(S)_PROXY env vars
+        # keep working for environments that need a proxy to reach Graylog.
+        self.opener = urllib.request.build_opener(*handlers)
+
+    # -- url helpers -----------------------------------------------------
+    def url(self, path, params=None):
+        if not path.startswith("/"):
+            path = "/" + path
+        target = self.base_url + path
+        if params:
+            clean = {}
+            for key, value in params.items():
+                if value is None:
+                    continue
+                if isinstance(value, bool):
+                    value = "true" if value else "false"
+                clean[key] = value
+            if clean:
+                sep = "&" if "?" in target else "?"
+                target += sep + urllib.parse.urlencode(clean, doseq=True)
+        return target
+
+    # -- request core ----------------------------------------------------
+    def request(self, method, path, params=None, body=None, expect_json=True):
+        method = method.upper()
+        url = self.url(path, params)
+        payload = None
+        headers = {
+            "Accept": "application/json",
+            "Accept-Encoding": "identity",
+            "User-Agent": USER_AGENT,
+            # Graylog rejects state changing requests without this header.
+            "X-Requested-By": "graylog-ops",
+        }
+        if body is not None:
+            payload = json.dumps(body).encode("utf-8")
+            headers["Content-Type"] = "application/json"
+        if self.auth:
+            headers["Authorization"] = self.auth
+
+        started = time.time()
+        last_error = None
+        for attempt in range(self.retries + 1):
+            request = urllib.request.Request(
+                url, data=payload, headers=headers, method=method
+            )
+            try:
+                with self.opener.open(request, timeout=self.timeout) as response:
+                    status = response.getcode()
+                    raw = response.read()
+                    self._debug(method, url, status, started)
+                    if not raw.strip():
+                        return None
+                    text = raw.decode("utf-8", "replace")
+                    if not expect_json:
+                        return text
+                    try:
+                        return json.loads(text)
+                    except ValueError:
+                        raise GraylogError(
+                            "response was not JSON",
+                            status=status,
+                            body=text[:500],
+                            path=path,
+                            method=method,
+                        )
+            except urllib.error.HTTPError as exc:
+                text = exc.read().decode("utf-8", "replace")
+                detail = _error_detail(text)
+                self._debug(method, url, exc.code, started)
+                last_error = GraylogError(
+                    detail or ("request rejected" if exc.code != 401 else "authentication failed"),
+                    status=exc.code,
+                    body=text[:2000],
+                    path=path,
+                    method=method,
+                )
+                if exc.code in RETRY_STATUS and attempt < self.retries:
+                    self._sleep(attempt, exc.headers.get("Retry-After"))
+                    continue
+                raise last_error
+            except (urllib.error.URLError, socket.timeout, ConnectionError, OSError) as exc:
+                reason = getattr(exc, "reason", exc)
+                self._debug(method, url, "ERR", started)
+                last_error = GraylogError(
+                    "connection failed: %s" % reason,
+                    path=path,
+                    method=method,
+                )
+                if attempt < self.retries:
+                    self._sleep(attempt, None)
+                    continue
+                raise last_error
+        raise last_error  # pragma: no cover - loop always raises or returns
+
+    # -- internals -------------------------------------------------------
+    def _sleep(self, attempt, retry_after):
+        delay = (1.5 ** attempt) + random.uniform(0, 0.3)
+        try:
+            if retry_after is not None:
+                delay = max(delay, float(retry_after))
+        except (TypeError, ValueError):
+            pass
+        time.sleep(min(delay, 30.0))
+
+    def _debug(self, method, url, status, started):
+        if not self.verbose:
+            return
+        elapsed = int((time.time() - started) * 1000)
+        sys.stderr.write("[http] %s %s -> %s (%dms)\n" % (method, url, status, elapsed))
+        sys.stderr.flush()
+
+    # -- convenience -----------------------------------------------------
+    def get(self, path, params=None):
+        return self.request("GET", path, params=params)

graylog_ops/judge.py

@@ -0,0 +1,397 @@
+"""Judgement layer: aggregated Graylog signals -> ranked ban candidates.
+
+This module is **read only**. It answers one question:
+
+    "在这个时间窗里，哪些客户端 IP 看起来像在攻击，为什么，建议怎么处理？"
+
+It does not touch the enforcement side (Sangfor AD / WAF / iptables). The
+output is a list of ``{ip, score, action, reasons, signals}`` which the next
+phase can feed into whatever blacklist API we end up using.
+
+Design notes
+------------
+* A **fixed number of ``terms`` queries** (one per signal) keeps the load on
+  Elasticsearch predictable no matter how many IPs show up in the window.
+* Signal queries are templates over field names. The report exposes each
+  signal's query and hit count, so they can be tuned against real data. A
+  signal that matched 0 IPs usually means the query does not fit the field
+  mapping (e.g. ``request_uri`` is a keyword-ish analyzed field).
+* **Volume alone never bans** (``require_evidence``). A busy but legitimate
+  client must not be blackholed by a rate threshold alone; at least one
+  behavioural signal (auth failures / scanner UA / sensitive path / 404
+  probing) has to be present for a ban recommendation.
+"""
+
+import ipaddress
+import json
+import math
+import statistics
+
+from . import detect
+from .client import normalize_terms
+from .http import GraylogError
+
+DEFAULT_FIELDS = {
+    "client_ip": "client_ip",
+    "status": "status",
+    "uri": "request_uri",
+    "ua": "http_user_agent",
+    "method": "request_method",
+}
+
+# name, title, required field keys, query template, weight, saturation count
+SIGNAL_DEFS = (
+    {
+        "name": "auth_fail",
+        "title": "认证失败",
+        "requires": ("status",),
+        "query": lambda fields: "%s:(401 OR 403)" % fields["status"],
+        "weight": 45.0,
+        "saturate": 10.0,
+    },
+    {
+        "name": "scanner_ua",
+        "title": "扫描器 UA",
+        "requires": ("ua",),
+        "query": lambda fields: (
+            "%s:(sqlmap* OR nikto* OR nuclei* OR nmap* OR masscan* OR zgrab* OR "
+            "wpscan* OR gobuster* OR dirbuster* OR dirsearch* OR acunetix*)"
+            % fields["ua"]
+        ),
+        "weight": 50.0,
+        "saturate": 2.0,
+    },
+    {
+        "name": "sensitive_path",
+        "title": "敏感路径",
+        "requires": ("uri",),
+        # Slashes must be escaped, otherwise Lucene treats /.../ as a regex.
+        "query": lambda fields: (
+            "%s:(\\/wp-login* OR \\/phpmyadmin* OR \\/.env* OR \\/actuator* "
+            "OR \\/cgi-bin* OR \\/manager\\/html*)"
+            % fields["uri"]
+        ),
+        "weight": 25.0,
+        "saturate": 5.0,
+    },
+    {
+        "name": "not_found",
+        "title": "404 探测",
+        "requires": ("status",),
+        "query": lambda fields: "%s:404" % fields["status"],
+        "weight": 20.0,
+        "saturate": 20.0,
+    },
+    {
+        "name": "bad_method",
+        "title": "异常方法",
+        "requires": ("method",),
+        "query": lambda fields: "%s:(PUT OR DELETE OR TRACE OR CONNECT)" % fields["method"],
+        "weight": 10.0,
+        "saturate": 5.0,
+    },
+)
+
+# URIs containing these hints mean auth failures are likely credential
+# attacks rather than an integration with a broken/expired token.
+AUTH_PATH_HINTS = ("login", "signin", "auth", "token", "captcha", "oauth", "password")
+
+DEFAULT_THRESHOLDS = {
+    "min_requests": 20,        # ignore IPs with fewer requests in the window
+    "min_score": 40.0,         # below this: ignore
+    "short_ban_score": 55.0,   # >= this: short ban recommendation
+    "ban_score": 75.0,         # >= this: ban recommendation
+    "rate_weight": 25.0,
+    "rate_saturate": 200.0,    # requests in window for full rate points
+    "spike_weight": 15.0,
+    "auth_ratio_weight": 25.0,
+    "require_evidence": True,
+    "evidence_signals": ("auth_fail", "scanner_ua", "sensitive_path", "not_found", "bad_method"),
+}
+
+
+# --------------------------------------------------------------------- profile
+def load_profile(path):
+    if not path:
+        return {}
+    try:
+        with open(path, "r", encoding="utf-8") as handle:
+            data = json.load(handle)
+    except OSError as exc:
+        raise SystemExit("无法读取 profile %s: %s" % (path, exc))
+    except ValueError as exc:
+        raise SystemExit("profile %s 不是合法 JSON: %s" % (path, exc))
+    if not isinstance(data, dict):
+        raise SystemExit("profile %s 必须是 JSON 对象" % path)
+    return data
+
+
+def build_fields(profile, args=None):
+    fields = dict(DEFAULT_FIELDS)
+    for key, value in (profile.get("fields") or {}).items():
+        if value:
+            fields[key] = value
+
+    flag_map = {
+        "client_ip": "field",
+        "status": "status_field",
+        "uri": "uri_field",
+        "ua": "ua_field",
+        "method": "method_field",
+    }
+    for key, attribute in flag_map.items():
+        value = getattr(args, attribute, None) if args is not None else None
+        if value:
+            fields[key] = value
+    return fields
+
+
+def build_signals(fields, overrides=None):
+    overrides = overrides or {}
+    signals = []
+    for definition in SIGNAL_DEFS:
+        if any(not fields.get(key) for key in definition["requires"]):
+            continue
+        signal = {
+            "name": definition["name"],
+            "title": definition["title"],
+            "query": definition["query"](fields),
+            "weight": float(definition["weight"]),
+            "saturate": float(definition["saturate"]),
+        }
+        override = overrides.get(signal["name"]) or {}
+        if override.get("enabled") is False:
+            continue
+        if override.get("query"):
+            signal["query"] = override["query"]
+        if override.get("weight") is not None:
+            signal["weight"] = float(override["weight"])
+        if override.get("saturate") is not None:
+            signal["saturate"] = float(override["saturate"])
+        signals.append(signal)
+    return signals
+
+
+# ------------------------------------------------------------------- allowlist
+class Allowlist:
+    """Never ban infrastructure, internal addresses or explicitly listed nets."""
+
+    def __init__(self, entries=None, exclude_internal=True):
+        self.entries = list(entries or [])
+        self.exclude_internal = exclude_internal
+
+    def reason(self, ip_text):
+        try:
+            ip = ipaddress.ip_address(ip_text)
+        except ValueError:
+            return None
+        for network, source in self.entries:
+            if ip.version == network.version and ip in network:
+                return "白名单 %s" % source
+        if self.exclude_internal and detect.is_internal(ip):
+            return "内部地址（RFC1918/回环/CGNAT）"
+        return None
+
+
+def load_allowlist(paths):
+    entries = []
+    for path in paths or []:
+        try:
+            with open(path, "r", encoding="utf-8") as handle:
+                lines = handle.readlines()
+        except OSError as exc:
+            raise SystemExit("无法读取白名单 %s: %s" % (path, exc))
+        for number, line in enumerate(lines, 1):
+            text = line.split("#", 1)[0].strip()
+            if not text:
+                continue
+            try:
+                network = ipaddress.ip_network(text, strict=False)
+            except ValueError:
+                raise SystemExit("白名单 %s:%d 不是合法 IP/CIDR: %r" % (path, number, text))
+            entries.append((network, "%s:%d" % (path, number)))
+    return entries
+
+
+# ----------------------------------------------------------------- collection
+def collect_signals(client, field, signals, window_seconds, query="*",
+                    filter_str=None, size=200):
+    """Run one terms query per signal plus one for the total request counts."""
+    term_frame = lambda data: {
+        row["term"]: int(row["count"] or 0)
+        for row in normalize_terms(data)
+        if row["term"] not in (None, "")
+    }
+
+    totals = term_frame(client.terms(
+        field=field, query=query, size=size,
+        range_seconds=window_seconds, filter_str=filter_str,
+    ))
+
+    per_signal = {}
+    errors = []
+    for signal in signals:
+        try:
+            data = client.terms(
+                field=field, query=signal["query"], size=size,
+                range_seconds=window_seconds, filter_str=filter_str,
+            )
+            per_signal[signal["name"]] = term_frame(data)
+        except GraylogError as exc:
+            per_signal[signal["name"]] = {}
+            errors.append({
+                "signal": signal["name"],
+                "status": exc.status,
+                "error": exc.message,
+            })
+    return totals, per_signal, errors
+
+
+# ------------------------------------------------------------------- verify
+def verify_candidates(client, candidates, fields, window_seconds,
+                      filter_str=None, limit=5, size=10):
+    """Drill into top candidates: is the traffic one automated endpoint poller?
+
+    A client that hammers a *single* business API at a steady rate is usually an
+    integration with a broken/expired token, not an attacker. Banning it breaks
+    a business system, so such candidates are downgraded to ``observe`` and the
+    decision is left to a human.
+    """
+    uri_field = fields.get("uri")
+    client_field = fields.get("client_ip")
+    if not uri_field or not client_field:
+        return
+
+    for candidate in candidates[:limit]:
+        if not candidate["action"].startswith("ban"):
+            continue
+        query = '%s:"%s"' % (client_field, candidate["ip"])
+        try:
+            data = client.terms(
+                field=uri_field, query=query, size=size,
+                range_seconds=window_seconds, filter_str=filter_str,
+            )
+        except GraylogError as exc:
+            candidate["verify_error"] = exc.message
+            continue
+
+        rows = [(row["term"], row["count"]) for row in normalize_terms(data)
+                if row["term"] not in (None, "")]
+        total = sum(count for _, count in rows)
+        candidate["verify"] = {
+            "distinct_uris": len(rows),
+            "total_requests": total,
+            "top_uri": rows[0][0] if rows else None,
+            "top_uri_ratio": round(rows[0][1] / total, 4) if rows and total else 0.0,
+        }
+        if not rows or not total:
+            continue
+
+        top_uri, top_count = rows[0]
+        ratio = top_count / float(total)
+        login_like = any(hint in str(top_uri).lower() for hint in AUTH_PATH_HINTS)
+        candidate["verify"]["login_like"] = login_like
+
+        if ratio >= 0.9 and len(rows) <= 2 and not login_like:
+            candidate["verify"]["automation_suspected"] = True
+            candidate["action"] = "observe"
+            candidate["score"] = round(min(candidate["score"], 54.0), 1)
+            candidate["reasons"].append(
+                "流量集中在单一接口 %s（%.0f%%），疑似业务客户端/自动化任务，已降级为 observe"
+                % (top_uri, ratio * 100)
+            )
+
+
+# -------------------------------------------------------------------- scoring
+def score_candidates(totals, per_signal, signals, thresholds, allowlist):
+    pool = {ip: count for ip, count in totals.items()
+            if count >= thresholds["min_requests"]}
+    median = statistics.median(pool.values()) if pool else 0
+
+    candidates = []
+    excluded = []
+
+    for ip, requests in pool.items():
+        reason = allowlist.reason(ip)
+        if reason:
+            excluded.append({"ip": ip, "requests": requests, "reason": reason})
+            continue
+
+        points = {}
+        counts = {}
+        reasons = []
+
+        for signal in signals:
+            count = per_signal.get(signal["name"], {}).get(ip, 0)
+            if not count:
+                continue
+            counts[signal["name"]] = count
+            contributed = signal["weight"] * min(1.0, count / signal["saturate"])
+            points[signal["name"]] = round(contributed, 1)
+            reasons.append("%s %d 次" % (signal["title"], count))
+
+        rate_points = thresholds["rate_weight"] * min(
+            1.0, requests / float(thresholds["rate_saturate"]))
+        points["rate"] = round(rate_points, 1)
+
+        if median and requests >= max(50, median * 4):
+            ratio = requests / float(median)
+            spike = thresholds["spike_weight"] * min(1.0, math.log(ratio, 2) / 6.0)
+            points["spike"] = round(spike, 1)
+            reasons.append("请求量是候选中位数的 %.0f 倍" % ratio)
+
+        auth = counts.get("auth_fail", 0)
+        auth_ratio = (auth / float(requests)) if requests else 0.0
+        if auth >= 10 and auth_ratio >= 0.5:
+            points["auth_ratio"] = thresholds["auth_ratio_weight"]
+            reasons.append("失败请求占 %.0f%%" % (auth_ratio * 100))
+
+        score = min(100.0, sum(points.values()))
+        evidence = any(counts.get(name) for name in thresholds["evidence_signals"])
+
+        if evidence and score >= thresholds["ban_score"]:
+            action = "ban_1h"
+        elif evidence and score >= thresholds["short_ban_score"]:
+            action = "ban_30m"
+        elif score >= thresholds["min_score"]:
+            action = "observe"
+        else:
+            action = "ignore"
+
+        if not evidence and thresholds["require_evidence"] and action in ("ban_1h", "ban_30m"):
+            action = "observe"
+            reasons.append("仅请求量异常、无攻击特征，不自动封禁")
+
+        candidates.append({
+            "ip": ip,
+            "score": round(score, 1),
+            "requests": requests,
+            "action": action,
+            "evidence": evidence,
+            "auth_ratio": round(auth_ratio, 4),
+            "signals": counts,
+            "points": points,
+            "reasons": reasons,
+        })
+
+    candidates.sort(key=lambda row: (-row["score"], -row["requests"], row["ip"]))
+    excluded.sort(key=lambda row: (-row["requests"], row["ip"]))
+
+    signal_stats = []
+    for signal in signals:
+        matched = per_signal.get(signal["name"], {})
+        signal_stats.append({
+            "name": signal["name"],
+            "title": signal["title"],
+            "query": signal["query"],
+            "ips_matched": len(matched),
+            "requests_matched": sum(matched.values()),
+        })
+
+    return {
+        "median_requests": median,
+        "candidate_pool": len(pool),
+        "candidates": candidates,
+        "excluded": excluded,
+        "signal_stats": signal_stats,
+    }

graylog_ops/notify.py

@@ -0,0 +1,167 @@
+"""Webhook notifications for newly convicted malicious IPs.
+
+Supported payload flavours: WeCom (企业微信), DingTalk, Feishu, Slack and a
+generic JSON body. Delivery failures are reported but never break detection.
+"""
+import base64
+import hashlib
+import re
+import hmac
+import json
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from datetime import datetime
+
+from .geo import flag
+
+DEFAULT_TITLE = "新增恶意 IP"
+
+
+class Notifier:
+    def __init__(self, config):
+        self.kind = (config.get("type") or "generic").lower()
+        self.url = config.get("url") or ""
+        if not self.url:
+            raise ValueError("notify config is missing 'url'")
+        self.secret = config.get("secret") or ""
+        self.link_prefix = config.get("link_prefix") or ""
+        self.timeout = int(config.get("timeout") or 10)
+        self.title = config.get("title") or DEFAULT_TITLE
+
+    def link_for(self, ip):
+        if not self.link_prefix:
+            return ""
+        return self.link_prefix.rstrip("/") + "/#/ip/" + ip
+
+    @staticmethod
+    def local_time(value):
+        try:
+            text = str(value).replace("Z", "+00:00")
+            return datetime.fromisoformat(text).astimezone().strftime("%m-%d %H:%M")
+        except (ValueError, TypeError):
+            return str(value)
+
+    @staticmethod
+    def risk_label(score):
+        if score >= 90:
+            return "🔴 高危"
+        if score >= 70:
+            return "🟠 中危"
+        return "🟡 关注"
+
+    def render(self, entries):
+        count = len(entries)
+        heading = self.title if count == 1 else "新增 %d 个恶意 IP" % count
+        lines = ["### 🚨 %s" % heading, ""]
+        for index, entry in enumerate(entries, 1):
+            ip = entry.get("ip", "?")
+            geo = " · ".join(part for part in
+                             (entry.get("country_name"), entry.get("region")) if part)
+            flag_text = flag(entry.get("country_code"))
+            place = ("%s %s" % (flag_text, geo)).strip() if geo else ""
+            score = float(entry.get("score") or 0)
+            lead = ("**%d. %s**" % (index, ip)) if count > 1 else ("**%s**" % ip)
+            lines.append("%s%s" % (lead, ("  " + place) if place else ""))
+            lines.append("评分 **%.1f**（%s）｜请求 **%s** 次"
+                         % (score, self.risk_label(score), entry.get("total") or 0))
+            if entry.get("first_seen"):
+                lines.append("首次攻击 **%s**" % self.local_time(entry["first_seen"]))
+            rules = entry.get("rules_text")
+            if rules:
+                lines.append("")
+                lines.append("**命中类型**")
+                for part in rules.split("；"):
+                    part = part.strip()
+                    if not part:
+                        continue
+                    part = re.sub(r"\s+(\d+)\s*次$", r" × \1", part)
+                    lines.append("- " + part)
+            link = self.link_for(ip)
+            if link:
+                lines.append("")
+                lines.append("[🔗 查看详情与证据](%s)" % link)
+            if index != count:
+                lines.append("")
+                lines.append("---")
+                lines.append("")
+        return "\n".join(lines)
+
+    def signed_url(self, timestamp=None):
+        """DingTalk custom robots with 加签 need timestamp + HMAC-SHA256."""
+        if self.kind != "dingtalk" or not self.secret:
+            return self.url
+        stamp = str(timestamp if timestamp is not None
+                    else int(time.time() * 1000))
+        string_to_sign = stamp + "\n" + self.secret
+        digest = hmac.new(self.secret.encode("utf-8"),
+                          string_to_sign.encode("utf-8"),
+                          hashlib.sha256).digest()
+        sign = urllib.parse.quote_plus(base64.b64encode(digest).decode("ascii"))
+        separator = "&" if "?" in self.url else "?"
+        return "%s%stimestamp=%s&sign=%s" % (self.url, separator, stamp, sign)
+
+    def payload(self, text, title=None):
+        title = title or self.title
+        if self.kind == "wecom":
+            return {"msgtype": "markdown", "markdown": {"content": text}}
+        if self.kind == "dingtalk":
+            return {"msgtype": "markdown", "markdown": {"title": title, "text": text}}
+        if self.kind == "feishu":
+            return {"msg_type": "text", "content": {"text": text}}
+        if self.kind == "slack":
+            return {"text": text}
+        return {"title": title, "text": text}
+
+    def post(self, text, title=None):
+        body = json.dumps(self.payload(text, title), ensure_ascii=False).encode("utf-8")
+        request = urllib.request.Request(
+            self.signed_url(), data=body, method="POST",
+            headers={"Content-Type": "application/json; charset=utf-8"})
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                reply = response.read().decode("utf-8", "replace")
+        except urllib.error.HTTPError as exc:
+            raise RuntimeError("webhook HTTP %s: %s"
+                               % (exc.code, exc.read()[:200].decode("utf-8", "replace")))
+        except urllib.error.URLError as exc:
+            raise RuntimeError("webhook unreachable: %s" % exc.reason)
+        if reply and self.kind in ("wecom", "dingtalk", "feishu"):
+            try:
+                parsed = json.loads(reply)
+            except ValueError:
+                return reply
+            code = parsed.get("errcode", parsed.get("code", 0))
+            if code:
+                raise RuntimeError("webhook rejected: %s" % reply[:200])
+        return reply
+
+    def send(self, entries):
+        if not entries:
+            return ""
+        return self.post(self.render(entries))
+
+    def send_test(self):
+        return self.post("**%s**\n\n这是一条测试推送，说明通知通道工作正常。"
+                         % self.title, title=self.title + " · 测试")
+
+
+def load_notifier(path):
+    with open(path, "r", encoding="utf-8") as handle:
+        return Notifier(json.load(handle))
+
+
+def notifier_from_env():
+    """Build a notifier from NOTIFY_* environment variables (container mode)."""
+    import os
+    url = (os.environ.get("NOTIFY_URL") or "").strip()
+    if not url:
+        return None
+    return Notifier({
+        "type": os.environ.get("NOTIFY_TYPE", "generic"),
+        "url": url,
+        "secret": os.environ.get("NOTIFY_SECRET", ""),
+        "link_prefix": os.environ.get("NOTIFY_LINK_PREFIX", ""),
+        "title": os.environ.get("NOTIFY_TITLE", "新增恶意 IP"),
+    })

graylog_ops/render.py

@@ -0,0 +1,77 @@
+"""Terminal rendering helpers (CJK-aware column widths)."""
+
+import unicodedata
+
+
+def display_width(text):
+    width = 0
+    for char in str(text):
+        if unicodedata.east_asian_width(char) in ("W", "F"):
+            width += 2
+        else:
+            width += 1
+    return width
+
+
+def clip(text, limit=60):
+    text = "" if text is None else str(text)
+    text = text.replace("\n", " ").replace("\r", " ").replace("\t", " ")
+    if len(text) <= limit:
+        return text
+    return text[: max(0, limit - 1)] + "…"
+
+
+def table(headers, rows, align=None):
+    """Render a plain-text table. *align* is a list of ``l``/``r`` per column."""
+    rows = [["" if cell is None else str(cell) for cell in row] for row in rows]
+    if align is None:
+        align = ["l"] * len(headers)
+    widths = [display_width(header) for header in headers]
+    for row in rows:
+        for index, cell in enumerate(row[: len(widths)]):
+            widths[index] = max(widths[index], display_width(cell))
+
+    def render(cells):
+        parts = []
+        for index, cell in enumerate(cells):
+            padding = widths[index] - display_width(cell)
+            if align[index] == "r":
+                parts.append(" " * padding + cell)
+            else:
+                parts.append(cell + " " * padding)
+        return "  ".join(parts).rstrip()
+
+    lines = [render(headers), "  ".join("-" * width for width in widths)]
+    lines.extend(render(row) for row in rows)
+    return "\n".join(lines)
+
+
+def human_bytes(size):
+    if not isinstance(size, (int, float)):
+        return str(size)
+    value = float(size)
+    for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+        if value < 1024 or unit == "TiB":
+            if unit == "B":
+                return "%d B" % value
+            return "%.1f %s" % (value, unit)
+        value /= 1024.0
+    return str(size)
+
+
+def percent(part, total):
+    if not total:
+        return "-"
+    return "%.1f%%" % (100.0 * part / total)
+
+
+def kv_block(pairs):
+    """Render aligned ``key: value`` lines."""
+    if not pairs:
+        return ""
+    width = max(display_width(key) for key, _ in pairs)
+    lines = []
+    for key, value in pairs:
+        padding = " " * (width - display_width(key))
+        lines.append("%s%s : %s" % (key, padding, "" if value is None else value))
+    return "\n".join(lines)