初始提交：恶意 IP 特征+行为融合检测系统
- 特征检测（规则/签名）与行为检测（24h 统计）单进程运行 - 融合名单按首次发现时间倒序，展示位置/归属（GeoIP + ASN） - 钉钉融合推送（加签、24h 节流、去重） - 页面状态灯、CSV 导出、Docker 镜像构建文件
This commit is contained in:
DeepFal committed
commit
2d76ed6672
42 files changed
+10645
No files matched your search
@@ -0,0 +1,50 @@
|
||||
"""Preview the CLI output without a real Graylog instance.
|
||||
|
||||
Runs a handful of commands against the bundled fake Graylog 4.0 server so you
|
||||
can see the exact shape of the tables/JSON before pointing the tool at
|
||||
edu-master.
|
||||
|
||||
python3 tests/demo.py
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
from fake_graylog import JUDGE_MESSAGES, FakeGraylog # noqa: E402
|
||||
from graylog_ops import cli # noqa: E402
|
||||
|
||||
COMMANDS = (
|
||||
["probe"],
|
||||
["streams"],
|
||||
["discover", "--range", "3600"],
|
||||
["top-ips", "--field", "client_ip", "--query", "status:401", "--size", "5"],
|
||||
["judge", "--range", "300", "--explain", "3"],
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
server = FakeGraylog(messages=JUDGE_MESSAGES).start()
|
||||
base = ["--url", server.url, "--token", "testtoken"]
|
||||
try:
|
||||
for command in COMMANDS:
|
||||
print("=" * 78)
|
||||
print("$ graylog-ops " + " ".join(command))
|
||||
print("-" * 78)
|
||||
captured = io.StringIO()
|
||||
with redirect_stdout(captured):
|
||||
cli.main(base + command)
|
||||
print(captured.getvalue().rstrip())
|
||||
print()
|
||||
finally:
|
||||
server.stop()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,468 @@
|
||||
"""A tiny fake Graylog 4.0 server used by the end-to-end tests.
|
||||
|
||||
It intentionally mimics the response envelopes of the real 4.0 API (including
|
||||
the ``{"messages": [{"message": {...}}]}`` shape of the legacy search API) so
|
||||
the client code is exercised the same way it will be against edu-master.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
TOKEN = "testtoken"
|
||||
EXPECTED_AUTH = "Basic " + base64.b64encode(
|
||||
("%s:token" % TOKEN).encode("utf-8")
|
||||
).decode("ascii")
|
||||
|
||||
VERSION = "4.0.8+6b8c55d"
|
||||
|
||||
MESSAGES = [
|
||||
{
|
||||
"timestamp": "2026-10-17T15:00:00.000Z",
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "203.0.113.5",
|
||||
"status": 401,
|
||||
"request_method": "POST",
|
||||
"request_uri": "/api/login",
|
||||
"http_user_agent": "python-requests/2.31",
|
||||
"response_time": 0.011,
|
||||
"message": '203.0.113.5 - - "POST /api/login HTTP/1.1" 401 0.011',
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-10-17T15:00:05.000Z",
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "203.0.113.5",
|
||||
"status": 401,
|
||||
"request_method": "POST",
|
||||
"request_uri": "/api/login",
|
||||
"http_user_agent": "python-requests/2.31",
|
||||
"response_time": 0.009,
|
||||
"message": '203.0.113.5 - - "POST /api/login HTTP/1.1" 401 0.009',
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-10-17T15:00:10.000Z",
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "203.0.113.5",
|
||||
"status": 401,
|
||||
"request_method": "POST",
|
||||
"request_uri": "/api/login",
|
||||
"http_user_agent": "python-requests/2.31",
|
||||
"response_time": 0.010,
|
||||
"message": '203.0.113.5 - - "POST /api/login HTTP/1.1" 401 0.010',
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-10-17T15:00:15.000Z",
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "198.51.100.7",
|
||||
"status": 401,
|
||||
"request_method": "GET",
|
||||
"request_uri": "/admin",
|
||||
"http_user_agent": "Mozilla/5.0",
|
||||
"response_time": 0.021,
|
||||
"message": '198.51.100.7 - - "GET /admin HTTP/1.1" 401 0.021',
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-10-17T15:00:20.000Z",
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "198.51.100.7",
|
||||
"status": 403,
|
||||
"request_method": "GET",
|
||||
"request_uri": "/admin",
|
||||
"http_user_agent": "Mozilla/5.0",
|
||||
"response_time": 0.018,
|
||||
"message": '198.51.100.7 - - "GET /admin HTTP/1.1" 403 0.018',
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-10-17T15:00:25.000Z",
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "192.0.2.44",
|
||||
"status": 404,
|
||||
"request_method": "GET",
|
||||
"request_uri": "/wp-login.php",
|
||||
"http_user_agent": "sqlmap/1.7",
|
||||
"response_time": 0.006,
|
||||
"message": '192.0.2.44 - - "GET /wp-login.php HTTP/1.1" 404 0.006',
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-10-17T15:00:30.000Z",
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "192.0.2.44",
|
||||
"status": 404,
|
||||
"request_method": "GET",
|
||||
"request_uri": "/.env",
|
||||
"http_user_agent": "sqlmap/1.7",
|
||||
"response_time": 0.005,
|
||||
"message": '192.0.2.44 - - "GET /.env HTTP/1.1" 404 0.005',
|
||||
},
|
||||
{
|
||||
"timestamp": "2026-10-17T15:00:35.000Z",
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "10.180.2.9",
|
||||
"status": 200,
|
||||
"request_method": "GET",
|
||||
"request_uri": "/health",
|
||||
"http_user_agent": "kube-probe/1.25",
|
||||
"response_time": 0.002,
|
||||
"message": '10.180.2.9 - - "GET /health HTTP/1.1" 200 0.002',
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _build_judge_messages():
|
||||
"""Synthetic traffic for the judgement tests (brute force / scanner / legit)."""
|
||||
rows = []
|
||||
|
||||
def add(ip, count, status, uri, agent, method="GET"):
|
||||
for index in range(count):
|
||||
rows.append({
|
||||
"timestamp": "2026-10-17T15:%02d:%02d.000Z" % (index // 60, index % 60),
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": ip,
|
||||
"status": status,
|
||||
"request_method": method,
|
||||
"request_uri": uri,
|
||||
"http_user_agent": agent,
|
||||
"message": '%s - "%s %s HTTP/1.1" %d' % (ip, method, uri, status),
|
||||
})
|
||||
|
||||
add("203.0.113.66", 40, 401, "/api/login", "python-requests/2.31", "POST")
|
||||
add("192.0.2.99", 25, 404, "/wp-login.php", "sqlmap/1.7")
|
||||
add("198.51.100.200", 50, 200, "/index", "Mozilla/5.0")
|
||||
add("10.180.5.5", 30, 401, "/api/login", "Mozilla/5.0", "POST")
|
||||
return rows
|
||||
|
||||
|
||||
JUDGE_MESSAGES = _build_judge_messages()
|
||||
|
||||
CLAUSE = re.compile(r'^([A-Za-z_][A-Za-z0-9_.]*):(\([^)]*\)|"[^"]*"|[^\s]+)$')
|
||||
|
||||
|
||||
def _clause_values(raw):
|
||||
raw = raw.strip()
|
||||
if raw.startswith("(") and raw.endswith(")"):
|
||||
return [part.strip().strip('"') for part in raw[1:-1].split(" OR ") if part.strip()]
|
||||
return [raw.strip('"')]
|
||||
|
||||
|
||||
def matches(message, query):
|
||||
"""Very small subset of Lucene, enough to exercise the judge signal queries."""
|
||||
query = (query or "*").strip()
|
||||
if query in ("", "*"):
|
||||
return True
|
||||
clause = CLAUSE.match(query)
|
||||
if clause is None:
|
||||
return True # unsupported syntax: do not filter (test double only)
|
||||
field, raw = clause.groups()
|
||||
value = message.get(field)
|
||||
if value is None:
|
||||
return False
|
||||
text = str(value).lower()
|
||||
for candidate in _clause_values(raw):
|
||||
# Mirror Lucene: `foo*` is a prefix match, `\/` is an escaped slash.
|
||||
needle = candidate.lower().replace("\\/", "/")
|
||||
if needle.endswith("*"):
|
||||
if needle[:-1] and text.startswith(needle[:-1]):
|
||||
return True
|
||||
continue
|
||||
if needle and needle in text:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def aggregate(messages, body):
|
||||
"""Minimal views-pivot implementation (one query, one search type)."""
|
||||
query = (body.get("queries") or [{}])[0]
|
||||
query_string = (query.get("query") or {}).get("query_string") or "*"
|
||||
filtered = [message for message in messages if matches(message, query_string)]
|
||||
|
||||
stream_filter = query.get("filter")
|
||||
stream_ids = set()
|
||||
if isinstance(stream_filter, dict):
|
||||
for item in stream_filter.get("filters") or []:
|
||||
if isinstance(item, dict) and item.get("id"):
|
||||
stream_ids.add(item["id"])
|
||||
if stream_ids:
|
||||
filtered = [message for message in filtered
|
||||
if set(message.get("streams") or []) & stream_ids]
|
||||
|
||||
search_type = (query.get("search_types") or [{}])[0]
|
||||
series_id = ((search_type.get("series") or [{}])[0]).get("id", "c1")
|
||||
row_groups = search_type.get("row_groups") or []
|
||||
column_groups = search_type.get("column_groups") or []
|
||||
|
||||
rows = []
|
||||
total = 0
|
||||
if row_groups:
|
||||
field = row_groups[0].get("field")
|
||||
limit = int(row_groups[0].get("limit") or 10)
|
||||
counts = {}
|
||||
for message in filtered:
|
||||
value = message.get(field)
|
||||
if value is None:
|
||||
continue
|
||||
counts[str(value)] = counts.get(str(value), 0) + 1
|
||||
total = sum(counts.values())
|
||||
ordered = sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:limit]
|
||||
for key, count in ordered:
|
||||
rows.append({"key": [key], "source": "leaf",
|
||||
"values": [{"key": [series_id], "value": count,
|
||||
"rollup": True, "source": "row-leaf"}]})
|
||||
rows.append({"key": [], "source": "rollup",
|
||||
"values": [{"key": [series_id], "value": total,
|
||||
"rollup": True, "source": "row-leaf"}]})
|
||||
elif column_groups:
|
||||
buckets = {}
|
||||
for message in filtered:
|
||||
stamp = str(message.get("timestamp", ""))
|
||||
key = stamp[:16] + ":00.000Z"
|
||||
buckets[key] = buckets.get(key, 0) + 1
|
||||
total = sum(buckets.values())
|
||||
for key in sorted(buckets):
|
||||
rows.append({"key": [], "source": "leaf",
|
||||
"values": [{"key": [key, series_id], "value": buckets[key],
|
||||
"rollup": False, "source": "col-leaf"}]})
|
||||
return {"id": search_type.get("id", "st1"), "type": "pivot",
|
||||
"total": total, "rows": rows}
|
||||
|
||||
|
||||
class FakeGraylog:
|
||||
def __init__(self, require_auth=True, messages=None, views_enabled=True):
|
||||
self.require_auth = require_auth
|
||||
self.messages = list(MESSAGES if messages is None else messages)
|
||||
self.views_enabled = views_enabled
|
||||
self.terms_size_param = "size"
|
||||
self.requests = []
|
||||
owner = self
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args): # silence test output
|
||||
pass
|
||||
|
||||
def _send(self, code, payload):
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_POST(self):
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path
|
||||
query = {key: values[0] for key, values in parse_qs(parsed.query).items()}
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
raw_body = self.rfile.read(length) if length else b""
|
||||
owner.requests.append((path, query))
|
||||
|
||||
if owner.require_auth and self.headers.get("Authorization") != EXPECTED_AUTH:
|
||||
return self._send(401, {"message": "Unauthorized"})
|
||||
if not owner.views_enabled:
|
||||
return self._send(404, {"message": "Sorry, Page Not Found"})
|
||||
if path == "/api/views/search/sync":
|
||||
try:
|
||||
body = json.loads(raw_body or b"{}")
|
||||
except ValueError:
|
||||
return self._send(400, {"message": "invalid json"})
|
||||
result = aggregate(owner.messages, body)
|
||||
return self._send(200, {
|
||||
"execution": {"done": True, "cancelled": False,
|
||||
"completed_exceptionally": False},
|
||||
"results": {"q1": {"errors": [], "search_types": {"st1": result}}},
|
||||
"search_id": "mock-search",
|
||||
})
|
||||
return self._send(404, {"message": "not found: %s" % path})
|
||||
|
||||
def do_GET(self):
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path
|
||||
query = {key: values[0] for key, values in parse_qs(parsed.query).items()}
|
||||
owner.requests.append((path, query))
|
||||
|
||||
if owner.require_auth and self.headers.get("Authorization") != EXPECTED_AUTH:
|
||||
return self._send(401, {"message": "Unauthorized"})
|
||||
|
||||
if path == "/api/system":
|
||||
return self._send(200, {
|
||||
"version": VERSION,
|
||||
"cluster_id": "6b8c55d0-dead-beef-0000-000000000001",
|
||||
"hostname": "edu-master",
|
||||
"is_processing": True,
|
||||
"timezone": "Asia/Shanghai",
|
||||
"locale": "zh_CN",
|
||||
})
|
||||
if path == "/api/system/cluster/nodes":
|
||||
return self._send(200, {"total": 1, "nodes": [{
|
||||
"node_id": "node-1",
|
||||
"hostname": "edu-master",
|
||||
"version": VERSION,
|
||||
"lifecycle": "RUNNING",
|
||||
"is_leader": True,
|
||||
"lb_status": "ALIVE",
|
||||
}]})
|
||||
if path == "/api/system/indexer/cluster/health":
|
||||
return self._send(200, {
|
||||
"status": "green",
|
||||
"cluster_name": "graylog",
|
||||
"number_of_nodes": 1,
|
||||
})
|
||||
if path == "/api/system/deflector":
|
||||
return self._send(200, {"index": "graylog_0", "is_up": True})
|
||||
if path == "/api/system/fields":
|
||||
return self._send(200, {"fields": [
|
||||
{"name": "client_ip", "type": "ip"},
|
||||
{"name": "status", "type": "long"},
|
||||
{"name": "request_uri", "type": "string"},
|
||||
]})
|
||||
if path == "/api/system/license":
|
||||
return self._send(200, {"license": {"type": "opensource"}})
|
||||
if path == "/api/system/inputs":
|
||||
return self._send(200, {"total": 1, "inputs": [{
|
||||
"id": "5f0000000000000000000001",
|
||||
"title": "GELF UDP",
|
||||
"type": "org.graylog2.inputs.gelf.udp.GELFUDPInput",
|
||||
"global": True,
|
||||
"configuration": {"bind_address": "0.0.0.0", "port": 12201},
|
||||
}]})
|
||||
if path == "/api/streams":
|
||||
return self._send(200, {"total": 2, "streams": [
|
||||
{
|
||||
"id": "5f00000000000000000000aa",
|
||||
"title": "业务访问日志",
|
||||
"description": "nginx access log",
|
||||
"disabled": False,
|
||||
"rules": [],
|
||||
},
|
||||
{
|
||||
"id": "000000000000000000000001",
|
||||
"title": "Default Stream",
|
||||
"description": "default",
|
||||
"disabled": False,
|
||||
"rules": [],
|
||||
},
|
||||
]})
|
||||
if path == "/api/system/indices/index_sets":
|
||||
return self._send(200, {"total": 1, "index_sets": [{
|
||||
"id": "5f00000000000000000000bb",
|
||||
"title": "Default index set",
|
||||
"index_prefix": "graylog",
|
||||
"shards": 4,
|
||||
"replicas": 0,
|
||||
"rotation_strategy": "time",
|
||||
"rotation_strategy_configuration": {"type": "time", "period": "P1D"},
|
||||
"retention_strategy": "delete",
|
||||
"retention_strategy_configuration": {"max_number_of_indices": 10},
|
||||
}]})
|
||||
if path == "/api/system/indexer/indices":
|
||||
return self._send(200, {"indices": [{
|
||||
"index_name": "graylog_0",
|
||||
"index_set_id": "5f00000000000000000000bb",
|
||||
"size": 1048576,
|
||||
"begin": "2026-10-17T00:00:00.000Z",
|
||||
"end": "2026-10-18T00:00:00.000Z",
|
||||
}]})
|
||||
if path == "/api/events/definitions":
|
||||
return self._send(200, {
|
||||
"total": 1, "count": 1, "page": 1, "per_page": 50,
|
||||
"event_definitions": [{
|
||||
"id": "5f00000000000000000000cc",
|
||||
"title": "401 风暴",
|
||||
"priority": 3,
|
||||
"disabled": False,
|
||||
"config": {"type": "aggregation-v1"},
|
||||
"notifications": [],
|
||||
}],
|
||||
})
|
||||
if path == "/api/events/notifications":
|
||||
return self._send(200, {"total": 1, "notifications": [{
|
||||
"id": "5f00000000000000000000dd",
|
||||
"title": "ban webhook",
|
||||
"config": {"type": "http-notification-v1",
|
||||
"url": "http://127.0.0.1:8080/ban"},
|
||||
}]})
|
||||
if path == "/api/events":
|
||||
return self._send(200, {"total_events": 0, "events": []})
|
||||
if path == "/api/sidecars":
|
||||
return self._send(200, {"total": 0, "sidecars": []})
|
||||
if path in ("/api/system/pipelines/pipeline", "/api/system/pipelines/rule"):
|
||||
return self._send(200, [])
|
||||
if path == "/api/views":
|
||||
return self._send(200, {"total": 0, "views": [], "page": 1, "per_page": 50})
|
||||
if path == "/api/search/universal/relative":
|
||||
try:
|
||||
limit = int(query.get("limit", 100))
|
||||
except ValueError:
|
||||
limit = 100
|
||||
try:
|
||||
offset = int(query.get("offset", 0))
|
||||
except ValueError:
|
||||
offset = 0
|
||||
matched = [item for item in owner.messages
|
||||
if matches(item, query.get("query", "*"))]
|
||||
page = matched[offset:offset + limit]
|
||||
return self._send(200, {
|
||||
"messages": [{"message": dict(item), "index": "graylog_0"} for item in page],
|
||||
"total_results": len(matched),
|
||||
"time": 3,
|
||||
"built_query": "mock",
|
||||
})
|
||||
if path == "/api/search/universal/terms":
|
||||
if owner.terms_size_param == "limit" and "size" in query:
|
||||
return self._send(400, {"message": "Parameter 'size' is not allowed"})
|
||||
field = query.get("field", "source")
|
||||
param = "size" if "size" in query else ("limit" if "limit" in query else None)
|
||||
try:
|
||||
size = int(query.get(param, 50)) if param else 50
|
||||
except ValueError:
|
||||
size = 50
|
||||
counts = {}
|
||||
query_string = query.get("query", "*")
|
||||
for message in owner.messages:
|
||||
if not matches(message, query_string):
|
||||
continue
|
||||
value = message.get(field)
|
||||
if value is None:
|
||||
continue
|
||||
counts[str(value)] = counts.get(str(value), 0) + 1
|
||||
terms = sorted(
|
||||
({"term": key, "count": value} for key, value in counts.items()),
|
||||
key=lambda row: (-row["count"], row["term"]),
|
||||
)[:size]
|
||||
return self._send(200, {
|
||||
"field": field,
|
||||
"terms": terms,
|
||||
"missing": 0,
|
||||
"other": 0,
|
||||
"total": len(terms),
|
||||
})
|
||||
if path == "/api/search/universal/histogram":
|
||||
return self._send(200, {
|
||||
"results": {
|
||||
"2026-10-17T15:00:00.000Z": {"count": 3},
|
||||
"2026-10-17T15:01:00.000Z": {"count": 5},
|
||||
},
|
||||
"interval": "1m",
|
||||
"time": 2,
|
||||
})
|
||||
return self._send(404, {"message": "not found: %s" % path})
|
||||
|
||||
self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = None
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return "http://127.0.0.1:%d" % self.port
|
||||
|
||||
def start(self):
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
if self.thread is not None:
|
||||
self.thread.join(timeout=5)
|
||||
@@ -0,0 +1,49 @@
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
|
||||
from graylog_ops.asn import AsnDB, build_asn_db, classify_org # noqa: E402
|
||||
|
||||
|
||||
class AsnTests(unittest.TestCase):
|
||||
def test_build_and_lookup(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
csv_path = os.path.join(directory, "asn.csv")
|
||||
with open(csv_path, "w", encoding="utf-8") as handle:
|
||||
handle.write('1.0.1.0,1.0.3.255,4134,"Chinanet"\n')
|
||||
handle.write('8.8.8.0,8.8.8.255,15169,"Google LLC"\n')
|
||||
handle.write('47.0.0.0,47.0.0.255,37963,'
|
||||
'"Hangzhou Alibaba Advertising Co.,Ltd."\n')
|
||||
handle.write('2001:db8::,2001:db8::ffff,15169,"Google LLC"\n')
|
||||
db_path = os.path.join(directory, "asn.db")
|
||||
count = build_asn_db(csv_path, db_path)
|
||||
self.assertEqual(count, 3) # IPv6 skipped
|
||||
|
||||
asn = AsnDB(db_path)
|
||||
try:
|
||||
china = asn.lookup("1.0.2.1")
|
||||
self.assertEqual(china["brand"], "中国电信")
|
||||
self.assertEqual(china["type"], "broadband")
|
||||
aliyun = asn.lookup("47.0.0.1")
|
||||
self.assertEqual(aliyun["brand"], "阿里云")
|
||||
self.assertEqual(aliyun["type"], "cloud")
|
||||
self.assertEqual(aliyun["icon"], "☁️")
|
||||
self.assertIsNone(asn.lookup("9.9.9.9"))
|
||||
finally:
|
||||
asn.close()
|
||||
|
||||
def test_classify_org(self):
|
||||
self.assertEqual(classify_org("China Mobile Communications Corporation")[:2],
|
||||
("中国移动", "mobile"))
|
||||
self.assertEqual(classify_org("China Education and Research Network Center")[:2],
|
||||
("教育网", "education"))
|
||||
self.assertEqual(classify_org("Wowrack.com")[:2], ("IDC 机房", "idc"))
|
||||
self.assertEqual(classify_org("Some Unknown ISP")[:2], ("未知", "unknown"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,103 @@
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
|
||||
from graylog_ops import watch # noqa: E402
|
||||
|
||||
NOW = "2026-10-17T16:00:00Z"
|
||||
|
||||
|
||||
def _behavior(ip, verdict="建议封禁", score=75):
|
||||
return {
|
||||
"ip": ip, "score": score, "verdict": verdict,
|
||||
"signals": ["高失败率"], "reason_text": "24 小时内扫描了大量路径。",
|
||||
"share_note": "", "behavior_type": "目录扫描",
|
||||
"asn": "4538", "asn_org": "教育网", "shared_type": "education",
|
||||
"total": 50, "pct4": 60.0, "uri_card": 20, "uri_templates": 15,
|
||||
"ua_card": 1, "host_card": 2, "coverage": 100.0, "peak_share": 50.0,
|
||||
"top_uri": "/.env", "top_ua": "curl/8.0",
|
||||
"country_code": "CN", "country_name": "中国", "region": "北京",
|
||||
"city": "Beijing",
|
||||
}
|
||||
|
||||
|
||||
class FusedTests(unittest.TestCase):
|
||||
def test_feature_and_behavior_merge(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
try:
|
||||
store.upsert({"ip": "203.0.113.9", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 91.0, "total": 120, "window": 3600,
|
||||
"rules": {"probe_java": 10},
|
||||
"reasons": ["探测 10 次"]}, NOW)
|
||||
store.sync_fused(NOW)
|
||||
fused = store.fused_items()
|
||||
self.assertEqual(len(fused), 1)
|
||||
self.assertEqual(fused[0]["sources"], "feature")
|
||||
self.assertEqual(fused[0]["first_banned_at"], NOW)
|
||||
self.assertIn("特征命中", fused[0]["fused_reason"])
|
||||
|
||||
# behaviour-only ban is merged
|
||||
store.replace_behavior_profiles([_behavior("198.51.100.9")])
|
||||
store.sync_fused("2026-10-17T16:10:00Z")
|
||||
merged = {item["ip"]: item for item in store.fused_items()}
|
||||
self.assertIn("198.51.100.9", merged)
|
||||
self.assertEqual(merged["198.51.100.9"]["sources"], "behavior")
|
||||
self.assertEqual(merged["203.0.113.9"]["sources"], "feature")
|
||||
|
||||
# same IP on both sides becomes "both"
|
||||
store.upsert({"ip": "198.51.100.9", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 80.0, "total": 50, "window": 3600,
|
||||
"rules": {"probe_java": 6},
|
||||
"reasons": ["探测 6 次"]}, "2026-10-17T16:11:00Z")
|
||||
store.sync_fused("2026-10-17T16:12:00Z")
|
||||
merged = {item["ip"]: item for item in store.fused_items()}
|
||||
self.assertEqual(merged["198.51.100.9"]["sources"], "both")
|
||||
|
||||
# watch-only behaviour must not enter the block list
|
||||
store.replace_behavior_profiles([
|
||||
_behavior("192.0.2.44", verdict="仅告警（共享出口）")])
|
||||
store.sync_fused("2026-10-17T16:20:00Z")
|
||||
ips = {item["ip"] for item in store.fused_items()}
|
||||
self.assertNotIn("192.0.2.44", ips)
|
||||
self.assertIn("203.0.113.9", ips) # feature rows stay forever
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
def test_push_throttle_24h(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
try:
|
||||
now = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
store.replace_behavior_profiles([_behavior("198.51.100.7")])
|
||||
store.sync_fused(now)
|
||||
# first sync suppresses historical rows; a fresh detection is due
|
||||
with store.lock, store.conn:
|
||||
store.conn.execute(
|
||||
"UPDATE ip_fused SET last_pushed_at=NULL WHERE ip=?",
|
||||
("198.51.100.7",))
|
||||
due = store.due_pushes(active_hours=24, cooldown_hours=24)
|
||||
self.assertEqual([row["ip"] for row in due], ["198.51.100.7"])
|
||||
|
||||
store.mark_pushed(["198.51.100.7"], now)
|
||||
self.assertEqual(store.due_pushes(24, 24), []) # throttled
|
||||
|
||||
old = time.strftime("%Y-%m-%dT%H:%M:%SZ",
|
||||
time.gmtime(time.time() - 25 * 3600))
|
||||
with store.lock, store.conn:
|
||||
store.conn.execute(
|
||||
"UPDATE ip_fused SET last_pushed_at=? WHERE ip=?",
|
||||
(old, "198.51.100.7"))
|
||||
due = store.due_pushes(active_hours=24, cooldown_hours=24)
|
||||
self.assertEqual([row["ip"] for row in due], ["198.51.100.7"])
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Unit tests for the client-IP field heuristics."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from graylog_ops import detect # noqa: E402
|
||||
|
||||
|
||||
def message(client_ip, source="10.180.254.17", **extra):
|
||||
payload = {
|
||||
"timestamp": "2026-10-17T15:00:00.000Z",
|
||||
"source": source,
|
||||
"client_ip": client_ip,
|
||||
"status": 401,
|
||||
"request_uri": "/api/login",
|
||||
"message": '%s - - "POST /api/login HTTP/1.1" 401 0.01' % client_ip,
|
||||
}
|
||||
payload.update(extra)
|
||||
return payload
|
||||
|
||||
|
||||
class AsIpTests(unittest.TestCase):
|
||||
def test_ipv4(self):
|
||||
self.assertEqual(str(detect.as_ip("203.0.113.5")), "203.0.113.5")
|
||||
|
||||
def test_ipv6(self):
|
||||
self.assertEqual(str(detect.as_ip("2001:db8::1")), "2001:db8::1")
|
||||
|
||||
def test_bracketed_ipv6_with_port(self):
|
||||
self.assertEqual(str(detect.as_ip("[2001:db8::1]:443")), "2001:db8::1")
|
||||
|
||||
def test_ipv4_with_port(self):
|
||||
self.assertEqual(str(detect.as_ip("192.0.2.44:8080")), "192.0.2.44")
|
||||
|
||||
def test_rejects_cidr(self):
|
||||
self.assertIsNone(detect.as_ip("10.0.0.0/8"))
|
||||
|
||||
def test_rejects_hostname(self):
|
||||
self.assertIsNone(detect.as_ip("edu-master"))
|
||||
|
||||
def test_rejects_bad_octet(self):
|
||||
self.assertIsNone(detect.as_ip("999.1.1.1"))
|
||||
|
||||
def test_rejects_empty_and_number(self):
|
||||
self.assertIsNone(detect.as_ip(""))
|
||||
self.assertIsNone(detect.as_ip(12345))
|
||||
|
||||
|
||||
class AnalyseTests(unittest.TestCase):
|
||||
def test_empty_sample(self):
|
||||
report = detect.analyse_messages([])
|
||||
self.assertEqual(report["sampled"], 0)
|
||||
self.assertIsNone(report["recommendation"]["field"])
|
||||
self.assertEqual(report["recommendation"]["confidence"], "none")
|
||||
|
||||
def test_recommends_client_ip_over_source(self):
|
||||
messages = [message(ip) for ip in (
|
||||
"203.0.113.5", "203.0.113.5", "203.0.113.5",
|
||||
"198.51.100.7", "198.51.100.7", "192.0.2.44", "192.0.2.44",
|
||||
"10.180.2.9",
|
||||
)]
|
||||
report = detect.analyse_messages(messages)
|
||||
self.assertEqual(report["recommendation"]["field"], "client_ip")
|
||||
self.assertEqual(report["recommendation"]["confidence"], "high")
|
||||
|
||||
candidates = {row["field"]: row for row in report["candidates"]}
|
||||
self.assertIn("source", candidates)
|
||||
self.assertNotIn("message", candidates)
|
||||
self.assertLess(candidates["source"]["score"], candidates["client_ip"]["score"])
|
||||
# The constant `source` must not outrank the varying client address.
|
||||
self.assertEqual(candidates["source"]["distinct_ips"], 1)
|
||||
|
||||
def test_warns_when_source_is_a_proxy(self):
|
||||
messages = [message("203.0.113.%d" % index) for index in range(1, 7)]
|
||||
report = detect.analyse_messages(messages)
|
||||
joined = " ".join(report["recommendation"]["warnings"])
|
||||
self.assertIn("source", joined)
|
||||
|
||||
def test_private_ratio_note(self):
|
||||
messages = [message("10.180.2.%d" % index) for index in range(1, 7)]
|
||||
report = detect.analyse_messages(messages)
|
||||
best = report["recommendation"]["field"]
|
||||
self.assertEqual(best, "client_ip")
|
||||
joined = " ".join(report["recommendation"]["warnings"])
|
||||
self.assertIn("内网", joined)
|
||||
|
||||
def test_no_ip_field_at_all(self):
|
||||
messages = [{"source": "edu-master", "message": "no address here"}] * 3
|
||||
report = detect.analyse_messages(messages)
|
||||
self.assertIsNone(report["recommendation"]["field"])
|
||||
self.assertEqual(report["candidates"], [])
|
||||
|
||||
def test_http_fields_detected(self):
|
||||
report = detect.analyse_messages([message("203.0.113.5")])
|
||||
self.assertIn("status", report["http_fields"])
|
||||
self.assertIn("uri", report["http_fields"])
|
||||
|
||||
|
||||
class InternalAddressTests(unittest.TestCase):
|
||||
"""`is_internal` must not use ipaddress.is_private: that flag also covers
|
||||
documentation ranges such as 203.0.113.0/24."""
|
||||
|
||||
def test_rfc1918(self):
|
||||
self.assertTrue(detect.is_internal(detect.as_ip("10.1.2.3")))
|
||||
self.assertTrue(detect.is_internal(detect.as_ip("172.16.5.5")))
|
||||
self.assertTrue(detect.is_internal(detect.as_ip("192.168.1.1")))
|
||||
|
||||
def test_documentation_range_is_not_internal(self):
|
||||
self.assertFalse(detect.is_internal(detect.as_ip("203.0.113.5")))
|
||||
self.assertFalse(detect.is_internal(detect.as_ip("2001:db8::1")))
|
||||
|
||||
def test_loopback_link_local_and_cgnat(self):
|
||||
self.assertTrue(detect.is_internal(detect.as_ip("127.0.0.1")))
|
||||
self.assertTrue(detect.is_internal(detect.as_ip("169.254.1.1")))
|
||||
self.assertTrue(detect.is_internal(detect.as_ip("100.64.1.1")))
|
||||
self.assertTrue(detect.is_internal(detect.as_ip("::1")))
|
||||
|
||||
def test_ipv6_unique_local(self):
|
||||
self.assertTrue(detect.is_internal(detect.as_ip("fd00::1")))
|
||||
self.assertFalse(detect.is_internal(detect.as_ip("2400:cb00::1")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,212 @@
|
||||
"""End-to-end tests: run the real CLI against the fake Graylog server."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
sys.path.insert(0, ROOT)
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
from fake_graylog import MESSAGES, FakeGraylog # noqa: E402
|
||||
from graylog_ops import cli # noqa: E402
|
||||
|
||||
|
||||
class CliEndToEndTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.server = FakeGraylog().start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.stop()
|
||||
|
||||
def setUp(self):
|
||||
self.server.terms_size_param = "size"
|
||||
self.server.requests.clear()
|
||||
|
||||
def run_cli(self, *argv, json_mode=True):
|
||||
args = ["--url", self.server.url, "--token", "testtoken"]
|
||||
if json_mode:
|
||||
args.append("--json")
|
||||
args.extend(argv)
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
code = cli.main(args)
|
||||
return code, stdout.getvalue(), stderr.getvalue()
|
||||
|
||||
# ------------------------------------------------------------- probe
|
||||
def test_probe_json(self):
|
||||
code, out, err = self.run_cli("probe")
|
||||
self.assertEqual(code, 0, err)
|
||||
data = json.loads(out)
|
||||
self.assertIn("4.0.8", data["system"]["version"])
|
||||
self.assertEqual(data["indexer"]["status"], "green")
|
||||
failed = [check["name"] for check in data["checks"] if not check["ok"]]
|
||||
self.assertEqual(failed, [], "unexpected failing endpoints: %s" % failed)
|
||||
|
||||
def test_probe_human_output(self):
|
||||
code, out, err = self.run_cli("probe", json_mode=False)
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertIn("检查项", out)
|
||||
self.assertIn("4.0.8", out)
|
||||
self.assertIn("全部", out)
|
||||
|
||||
# ---------------------------------------------------------- discover
|
||||
def test_discover_finds_client_ip(self):
|
||||
code, out, err = self.run_cli("discover", "--range", "3600")
|
||||
self.assertEqual(code, 0, err)
|
||||
data = json.loads(out)
|
||||
self.assertEqual(data["sampled"], len(MESSAGES))
|
||||
self.assertEqual(data["recommendation"]["field"], "client_ip")
|
||||
self.assertEqual(data["recommendation"]["confidence"], "high")
|
||||
self.assertNotIn("message", [row["field"] for row in data["candidates"]])
|
||||
self.assertIn("status", data["http_fields"])
|
||||
self.assertIn("uri", data["http_fields"])
|
||||
|
||||
def test_discover_stream_title_resolution(self):
|
||||
code, out, err = self.run_cli("discover", "--stream", "业务访问")
|
||||
self.assertEqual(code, 0, err)
|
||||
data = json.loads(out)
|
||||
self.assertEqual(data["stream_id"], "5f00000000000000000000aa")
|
||||
|
||||
def test_discover_unknown_stream_fails_cleanly(self):
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
with self.assertRaises(SystemExit) as ctx:
|
||||
cli.main(["--url", self.server.url, "--token", "testtoken",
|
||||
"discover", "--stream", "does-not-exist"])
|
||||
self.assertNotEqual(ctx.exception.code, 0)
|
||||
|
||||
# ------------------------------------------------------------- terms
|
||||
def test_top_ips(self):
|
||||
code, out, err = self.run_cli("top-ips", "--field", "client_ip", "--size", "10")
|
||||
self.assertEqual(code, 0, err)
|
||||
data = json.loads(out)
|
||||
self.assertEqual(data["field"], "client_ip")
|
||||
self.assertEqual(len(data["terms"]), 4)
|
||||
self.assertEqual(data["terms"][0]["term"], "203.0.113.5")
|
||||
self.assertEqual(data["terms"][0]["count"], 3)
|
||||
|
||||
def test_terms_use_views_backend(self):
|
||||
"""4.x removed the legacy terms endpoint, so aggregation must POST."""
|
||||
code, out, err = self.run_cli("top-ips", "--field", "client_ip")
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertTrue(json.loads(out)["terms"])
|
||||
posts = [path for path, _ in self.server.requests
|
||||
if path == "/api/views/search/sync"]
|
||||
self.assertTrue(posts, "expected a views search POST")
|
||||
|
||||
def test_legacy_fallback_when_views_missing(self):
|
||||
"""Graylog 3.x has the legacy terms endpoint but no views API."""
|
||||
server = FakeGraylog(views_enabled=False).start()
|
||||
server.terms_size_param = "limit"
|
||||
try:
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
code = cli.main(["--url", server.url, "--token", "testtoken", "--json",
|
||||
"terms", "--field", "client_ip"])
|
||||
self.assertEqual(code, 0, stderr.getvalue())
|
||||
self.assertTrue(json.loads(stdout.getvalue())["terms"])
|
||||
term_calls = [query for path, query in server.requests
|
||||
if path.endswith("/universal/terms")]
|
||||
self.assertEqual(len(term_calls), 2)
|
||||
self.assertIn("size", term_calls[0])
|
||||
self.assertIn("limit", term_calls[1])
|
||||
self.assertNotIn("size", term_calls[1])
|
||||
finally:
|
||||
server.stop()
|
||||
|
||||
# ----------------------------------------------------------- snapshot
|
||||
def test_snapshot(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
code, out, err = self.run_cli("snapshot", "--dir", directory, "--sample", "5")
|
||||
self.assertEqual(code, 0, err)
|
||||
data = json.loads(out)
|
||||
failed = [row for row in data["resources"] if row["status"] != "ok"]
|
||||
self.assertEqual(failed, [])
|
||||
self.assertTrue(os.path.exists(os.path.join(directory, "manifest.json")))
|
||||
self.assertTrue(os.path.exists(os.path.join(directory, "system.json")))
|
||||
self.assertEqual(data["sampled_messages"], 5)
|
||||
messages_path = os.path.join(directory, "messages.jsonl")
|
||||
with open(messages_path, "r", encoding="utf-8") as handle:
|
||||
lines = [line for line in handle if line.strip()]
|
||||
self.assertEqual(len(lines), 5)
|
||||
json.loads(lines[0])
|
||||
|
||||
# -------------------------------------------------------------- raw
|
||||
def test_raw_endpoint(self):
|
||||
code, out, err = self.run_cli("raw", "/api/system/inputs")
|
||||
self.assertEqual(code, 0, err)
|
||||
data = json.loads(out)
|
||||
self.assertIn("inputs", data)
|
||||
|
||||
# ---------------------------------------------------------- messages
|
||||
def test_messages_jsonl(self):
|
||||
code, out, err = self.run_cli("messages", "--limit", "3", "--jsonl")
|
||||
self.assertEqual(code, 0, err)
|
||||
lines = [line for line in out.splitlines() if line.strip()]
|
||||
self.assertEqual(len(lines), 3)
|
||||
for line in lines:
|
||||
json.loads(line)
|
||||
|
||||
# --------------------------------------------------------------- auth
|
||||
def test_bad_token_returns_1(self):
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
code = cli.main(["--url", self.server.url, "--token", "wrong", "probe"])
|
||||
self.assertEqual(code, 1)
|
||||
self.assertIn("401", stderr.getvalue())
|
||||
|
||||
def test_missing_config_returns_2(self):
|
||||
saved = {key: os.environ.pop(key, None) for key in (
|
||||
"GRAYLOG_URL", "GRAYLOG_TOKEN", "GRAYLOG_USER", "GRAYLOG_PASSWORD")}
|
||||
original_cwd = os.getcwd()
|
||||
try:
|
||||
# Run from an empty directory: a real ./graylog-ops.json in the
|
||||
# project root must not leak into this test.
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
os.chdir(directory)
|
||||
stderr = io.StringIO()
|
||||
with redirect_stderr(stderr):
|
||||
code = cli.main(["system"])
|
||||
self.assertEqual(code, 2)
|
||||
self.assertIn("配置错误", stderr.getvalue())
|
||||
finally:
|
||||
os.chdir(original_cwd)
|
||||
for key, value in saved.items():
|
||||
if value is not None:
|
||||
os.environ[key] = value
|
||||
|
||||
# --------------------------------------------------------------- collect
|
||||
def test_collect_bundle(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
code, out, err = self.run_cli("collect", "--dir", directory, "--range", "3600")
|
||||
self.assertEqual(code, 0, err)
|
||||
for name in ("probe.json", "discover.json", "judge.json",
|
||||
"profile.json", "SUMMARY.txt"):
|
||||
self.assertTrue(os.path.exists(os.path.join(directory, name)), name)
|
||||
self.assertTrue(os.path.isdir(os.path.join(directory, "snapshot")))
|
||||
data = json.loads(out)
|
||||
self.assertTrue(all(step["ok"] for step in data["steps"]), data["steps"])
|
||||
self.assertEqual(data["recommendation"]["field"], "client_ip")
|
||||
|
||||
# ---------------------------------------------------------------- out
|
||||
def test_out_file(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = os.path.join(directory, "ips.json")
|
||||
code, out, err = self.run_cli("top-ips", "--field", "client_ip", "--out", path)
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
self.assertEqual(payload["field"], "client_ip")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,48 @@
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from graylog_ops.geo import GeoIP, build_geo_db, country_name, flag, region_name
|
||||
|
||||
|
||||
class GeoIPTests(unittest.TestCase):
|
||||
def test_build_and_lookup(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
csv_path = os.path.join(directory, "city.csv")
|
||||
with open(csv_path, "w", encoding="utf-8") as fh:
|
||||
fh.write("1.0.0.0,1.0.0.255,OC,AU,Queensland,\"South Brisbane\",0,0\n")
|
||||
fh.write("1.0.1.0,1.0.3.255,AS,CN,Fujian,Wenquan,0,0\n")
|
||||
fh.write("8.8.8.0,8.8.8.255,NA,US,California,\"Mountain View\",0,0\n")
|
||||
fh.write("2001:db8::,2001:db8::ffff,NA,US,California,X,0,0\n")
|
||||
db_path = os.path.join(directory, "geo.db")
|
||||
count = build_geo_db(csv_path, db_path)
|
||||
self.assertEqual(count, 3) # IPv6 range skipped
|
||||
|
||||
geo = GeoIP(db_path)
|
||||
try:
|
||||
cn = geo.lookup("1.0.2.1")
|
||||
self.assertEqual(cn["country_code"], "CN")
|
||||
self.assertEqual(cn["country_name"], "中国")
|
||||
self.assertEqual(cn["region"], "福建")
|
||||
self.assertEqual(geo.lookup("8.8.8.8")["country_name"], "美国")
|
||||
self.assertIsNone(geo.lookup("9.9.9.9"))
|
||||
self.assertIsNone(geo.lookup("not-an-ip"))
|
||||
finally:
|
||||
geo.close()
|
||||
|
||||
def test_helpers(self):
|
||||
self.assertEqual(flag("CN"), "\U0001F1E8\U0001F1F3")
|
||||
self.assertEqual(flag("ZZ"), "")
|
||||
self.assertEqual(flag(""), "")
|
||||
self.assertEqual(country_name("CN"), "中国")
|
||||
self.assertEqual(country_name("US"), "美国")
|
||||
self.assertEqual(country_name("XX"), "XX")
|
||||
self.assertEqual(region_name("CN", "Guangdong"), "广东")
|
||||
self.assertEqual(region_name("US", "California"), "California")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Tests for the judgement layer (ban candidate scoring)."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stderr, redirect_stdout
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
sys.path.insert(0, ROOT)
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
from fake_graylog import JUDGE_MESSAGES, FakeGraylog # noqa: E402
|
||||
from graylog_ops import cli # noqa: E402
|
||||
|
||||
|
||||
class JudgeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.server = FakeGraylog(messages=JUDGE_MESSAGES).start()
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.stop()
|
||||
|
||||
def run_cli(self, *argv, json_mode=True):
|
||||
args = ["--url", self.server.url, "--token", "testtoken"]
|
||||
if json_mode:
|
||||
args.append("--json")
|
||||
args.extend(argv)
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
code = cli.main(args)
|
||||
return code, stdout.getvalue(), stderr.getvalue()
|
||||
|
||||
def judged(self, *argv):
|
||||
code, out, err = self.run_cli("judge", "--range", "300", *argv)
|
||||
self.assertEqual(code, 0, err)
|
||||
return json.loads(out)
|
||||
|
||||
# ------------------------------------------------------------ behaviour
|
||||
def test_bruteforce_client_is_recommended_for_ban(self):
|
||||
data = self.judged()
|
||||
rows = {row["ip"]: row for row in data["candidates"]}
|
||||
self.assertIn("203.0.113.66", rows)
|
||||
row = rows["203.0.113.66"]
|
||||
self.assertEqual(row["action"], "ban_1h")
|
||||
self.assertGreaterEqual(row["score"], 75)
|
||||
self.assertIn("auth_fail", row["signals"])
|
||||
self.assertTrue(any("失败请求" in reason for reason in row["reasons"]))
|
||||
|
||||
def test_scanner_client_is_recommended_for_ban(self):
|
||||
data = self.judged()
|
||||
row = {item["ip"]: item for item in data["candidates"]}["192.0.2.99"]
|
||||
self.assertEqual(row["action"], "ban_1h")
|
||||
self.assertIn("scanner_ua", row["signals"])
|
||||
self.assertIn("sensitive_path", row["signals"])
|
||||
|
||||
def test_busy_but_legitimate_client_is_not_banned(self):
|
||||
data = self.judged("--all")
|
||||
row = {item["ip"]: item for item in data["candidates"]}["198.51.100.200"]
|
||||
self.assertIn(row["action"], ("ignore", "observe"))
|
||||
self.assertFalse(row["evidence"])
|
||||
|
||||
def test_internal_addresses_are_excluded_by_default(self):
|
||||
data = self.judged()
|
||||
reasons = {row["ip"]: row["reason"] for row in data["excluded"]}
|
||||
self.assertIn("10.180.5.5", reasons)
|
||||
self.assertIn("内部", reasons["10.180.5.5"])
|
||||
self.assertNotIn("10.180.5.5", {row["ip"] for row in data["candidates"]})
|
||||
|
||||
def test_include_internal_flag(self):
|
||||
data = self.judged("--include-internal", "--all")
|
||||
ips = {row["ip"] for row in data["candidates"]}
|
||||
self.assertIn("10.180.5.5", ips)
|
||||
|
||||
def test_allowlist_file(self):
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=False) as handle:
|
||||
handle.write("# 深信服 AD / VIP\n10.180.254.17\n203.0.113.66\n")
|
||||
path = handle.name
|
||||
try:
|
||||
data = self.judged("--allow", path)
|
||||
excluded = {row["ip"]: row["reason"] for row in data["excluded"]}
|
||||
self.assertIn("203.0.113.66", excluded)
|
||||
self.assertIn("白名单", excluded["203.0.113.66"])
|
||||
self.assertNotIn("203.0.113.66", {row["ip"] for row in data["candidates"]})
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_explain_attaches_samples(self):
|
||||
data = self.judged("--explain", "2")
|
||||
sampled = [row for row in data["candidates"] if row.get("samples")]
|
||||
self.assertTrue(sampled)
|
||||
self.assertLessEqual(len(sampled[0]["samples"]), 3)
|
||||
|
||||
def test_verify_downgrades_automation_client(self):
|
||||
"""A steady single-endpoint poller with 403s must not be auto-banned."""
|
||||
automation = [{
|
||||
"timestamp": "2026-10-17T15:%02d:%02d.000Z" % (index // 60, index % 60),
|
||||
"source": "10.180.254.17",
|
||||
"client_ip": "203.0.113.77",
|
||||
"status": 403,
|
||||
"request_method": "POST",
|
||||
"request_uri": "/prod-api/biz/liveCourseChapter/msgSubscription",
|
||||
"http_user_agent": "okhttp/4.9",
|
||||
"message": '203.0.113.77 - "POST /prod-api/biz/x" 403',
|
||||
} for index in range(40)]
|
||||
server = FakeGraylog(messages=list(JUDGE_MESSAGES) + automation).start()
|
||||
try:
|
||||
stdout, stderr = io.StringIO(), io.StringIO()
|
||||
with redirect_stdout(stdout), redirect_stderr(stderr):
|
||||
code = cli.main(["--url", server.url, "--token", "testtoken",
|
||||
"--json", "judge", "--range", "300",
|
||||
"--verify", "10", "--all"])
|
||||
self.assertEqual(code, 0, stderr.getvalue())
|
||||
data = json.loads(stdout.getvalue())
|
||||
row = {item["ip"]: item for item in data["candidates"]}["203.0.113.77"]
|
||||
self.assertEqual(row["action"], "observe")
|
||||
self.assertTrue(row["verify"]["automation_suspected"])
|
||||
self.assertFalse(row["verify"]["login_like"])
|
||||
finally:
|
||||
server.stop()
|
||||
|
||||
def test_verify_keeps_login_bruteforce(self):
|
||||
"""Failures against a login path stay bannable even if concentrated."""
|
||||
data = self.judged("--verify", "5")
|
||||
row = {item["ip"]: item for item in data["candidates"]}["203.0.113.66"]
|
||||
self.assertEqual(row["action"], "ban_1h")
|
||||
self.assertTrue(row["verify"]["login_like"])
|
||||
|
||||
def test_profile_roundtrip(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
profile = os.path.join(directory, "profile.json")
|
||||
code, _, err = self.run_cli("discover", "--range", "3600",
|
||||
"--save-profile", profile)
|
||||
self.assertEqual(code, 0, err)
|
||||
with open(profile, "r", encoding="utf-8") as handle:
|
||||
saved = json.load(handle)
|
||||
self.assertEqual(saved["fields"]["client_ip"], "client_ip")
|
||||
self.assertEqual(saved["fields"]["status"], "status")
|
||||
self.assertEqual(saved["fields"]["uri"], "request_uri")
|
||||
|
||||
data = self.judged("--profile", profile)
|
||||
self.assertTrue(data["candidates"])
|
||||
self.assertEqual(data["field"], "client_ip")
|
||||
|
||||
# -------------------------------------------------------------- output
|
||||
def test_human_output(self):
|
||||
code, out, err = self.run_cli("judge", json_mode=False)
|
||||
self.assertEqual(code, 0, err)
|
||||
self.assertIn("信号命中", out)
|
||||
self.assertIn("本命令只读", out)
|
||||
self.assertIn("ban_1h", out)
|
||||
|
||||
def test_signal_queries_are_reported(self):
|
||||
data = self.judged()
|
||||
names = {stat["name"]: stat for stat in data["signal_stats"]}
|
||||
self.assertIn("auth_fail", names)
|
||||
self.assertGreater(names["auth_fail"]["ips_matched"], 0)
|
||||
self.assertGreater(names["scanner_ua"]["ips_matched"], 0)
|
||||
self.assertIn("query", names["auth_fail"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,209 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
for path in (ROOT, HERE):
|
||||
if path not in sys.path:
|
||||
sys.path.insert(0, path)
|
||||
|
||||
from graylog_ops import watch # noqa: E402
|
||||
from graylog_ops.behavior import notify_fused # noqa: E402
|
||||
from graylog_ops.notify import Notifier, load_notifier # noqa: E402
|
||||
from test_watch import _StubClient # noqa: E402
|
||||
|
||||
|
||||
class _CaptureNotifier:
|
||||
def __init__(self):
|
||||
self.batches = []
|
||||
self.posts = []
|
||||
|
||||
def send(self, entries):
|
||||
self.batches.append(entries)
|
||||
|
||||
def link_for(self, ip):
|
||||
return ""
|
||||
|
||||
def post(self, text):
|
||||
self.posts.append(text)
|
||||
|
||||
|
||||
class _Receiver(BaseHTTPRequestHandler):
|
||||
received = []
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
body = self.rfile.read(length).decode("utf-8")
|
||||
_Receiver.received.append(body)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{"errcode":0}')
|
||||
|
||||
|
||||
class NotifierTests(unittest.TestCase):
|
||||
def _config(self, kind="generic", url="http://127.0.0.1:1/hook"):
|
||||
return {"type": kind, "url": url, "link_prefix": "http://example/panel/"}
|
||||
|
||||
def test_payload_per_kind(self):
|
||||
text = "**新增恶意 IP**\n\n1.2.3.4"
|
||||
self.assertEqual(
|
||||
Notifier(self._config("wecom")).payload(text),
|
||||
{"msgtype": "markdown", "markdown": {"content": text}})
|
||||
self.assertEqual(
|
||||
Notifier(self._config("dingtalk")).payload(text)["msgtype"], "markdown")
|
||||
self.assertEqual(
|
||||
Notifier(self._config("feishu")).payload(text)["msg_type"], "text")
|
||||
self.assertEqual(Notifier(self._config("slack")).payload(text), {"text": text})
|
||||
generic = Notifier(self._config("generic")).payload(text)
|
||||
self.assertEqual(generic["text"], text)
|
||||
self.assertEqual(generic["title"], "新增恶意 IP")
|
||||
|
||||
def test_render_includes_geo_rules_and_link(self):
|
||||
notifier = Notifier(self._config())
|
||||
text = notifier.render([{
|
||||
"ip": "203.0.113.5", "country_name": "美国", "region": "California",
|
||||
"score": 91.0, "total": 120, "first_seen": "2026-10-17T16:00:00Z",
|
||||
"rules_text": "Spring/中间件探测 4 次",
|
||||
}])
|
||||
self.assertIn("203.0.113.5", text)
|
||||
self.assertIn("美国 · California", text)
|
||||
self.assertIn("Spring/中间件探测", text)
|
||||
self.assertIn("http://example/panel/#/ip/203.0.113.5", text)
|
||||
|
||||
def test_send_to_http_endpoint(self):
|
||||
_Receiver.received = []
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), _Receiver)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
url = "http://127.0.0.1:%d/hook" % server.server_address[1]
|
||||
notifier = Notifier(self._config("generic", url))
|
||||
notifier.send_test()
|
||||
self.assertEqual(len(_Receiver.received), 1)
|
||||
payload = json.loads(_Receiver.received[0])
|
||||
self.assertIn("测试推送", payload["text"])
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
def test_dingtalk_signing(self):
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import urllib.parse
|
||||
secret = "SECtest"
|
||||
url = "https://oapi.dingtalk.com/robot/send?access_token=abc"
|
||||
notifier = Notifier({"type": "dingtalk", "url": url, "secret": secret})
|
||||
stamp = 1700000000000
|
||||
expected = base64.b64encode(hmac.new(
|
||||
secret.encode(), ("%d\n%s" % (stamp, secret)).encode(),
|
||||
hashlib.sha256).digest()).decode()
|
||||
signed = notifier.signed_url(stamp)
|
||||
self.assertIn("timestamp=1700000000000", signed)
|
||||
self.assertIn("sign=" + urllib.parse.quote_plus(expected), signed)
|
||||
self.assertTrue(signed.startswith(url + "&"))
|
||||
|
||||
def test_dingtalk_without_secret_keeps_url(self):
|
||||
url = "https://oapi.dingtalk.com/robot/send?access_token=abc"
|
||||
self.assertEqual(Notifier({"type": "dingtalk", "url": url}).signed_url(1), url)
|
||||
|
||||
def test_dingtalk_signature(self):
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import urllib.parse
|
||||
|
||||
notifier = Notifier({
|
||||
"type": "dingtalk",
|
||||
"url": "https://oapi.dingtalk.com/robot/send?access_token=abc",
|
||||
"secret": "SECtest-secret",
|
||||
})
|
||||
url = notifier.signed_url()
|
||||
self.assertIn("timestamp=", url)
|
||||
self.assertIn("sign=", url)
|
||||
query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||||
timestamp = query["timestamp"][0]
|
||||
expected = base64.b64encode(hmac.new(
|
||||
b"SECtest-secret",
|
||||
("%s\nSECtest-secret" % timestamp).encode("utf-8"),
|
||||
hashlib.sha256).digest()).decode("ascii")
|
||||
self.assertEqual(query["sign"][0], expected)
|
||||
|
||||
unsigned = Notifier({"type": "dingtalk",
|
||||
"url": "https://example/hook?access_token=abc"})
|
||||
self.assertEqual(unsigned.signed_url(),
|
||||
"https://example/hook?access_token=abc")
|
||||
|
||||
def test_load_notifier(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = os.path.join(directory, "notify.json")
|
||||
json.dump(self._config("wecom"), open(path, "w", encoding="utf-8"))
|
||||
notifier = load_notifier(path)
|
||||
self.assertEqual(notifier.kind, "wecom")
|
||||
|
||||
|
||||
class UpsertStatusTests(unittest.TestCase):
|
||||
def test_upsert_returns_new_upgraded_unchanged(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
try:
|
||||
base = {"ip": "203.0.113.9", "total": 10, "rules": {},
|
||||
"reasons": []}
|
||||
self.assertEqual(store.upsert(dict(base, tier=watch.TIER_OBSERVE,
|
||||
score=5.0),
|
||||
"2026-10-17T16:00:00Z"), "new")
|
||||
self.assertEqual(store.upsert(dict(base, tier=watch.TIER_OBSERVE,
|
||||
score=5.0),
|
||||
"2026-10-17T16:01:00Z"), "unchanged")
|
||||
self.assertEqual(store.upsert(dict(base, tier=watch.TIER_MALICIOUS,
|
||||
score=80.0),
|
||||
"2026-10-17T16:02:00Z"), "upgraded")
|
||||
self.assertEqual(store.upsert(dict(base, tier=watch.TIER_MALICIOUS,
|
||||
score=80.0),
|
||||
"2026-10-17T16:03:00Z"), "unchanged")
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
class DetectorNotificationTests(unittest.TestCase):
|
||||
def test_new_malicious_notifies_once(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
try:
|
||||
probe = watch.Detector(None, store, dict(watch.DEFAULT_FIELDS))
|
||||
queries = {rule["name"]: rule["query"] for rule in probe.rules}
|
||||
by_query = {"*": [("203.0.113.5", 40)],
|
||||
queries["probe_java"]: [("203.0.113.5", 40)]}
|
||||
for name, query in queries.items():
|
||||
by_query.setdefault(query, [])
|
||||
capture = _CaptureNotifier()
|
||||
detector = watch.Detector(_StubClient(by_query), store,
|
||||
dict(watch.DEFAULT_FIELDS), window=300)
|
||||
detector.run_once()
|
||||
store.sync_fused()
|
||||
# first sync suppresses historical rows; make it a fresh detection
|
||||
with store.lock, store.conn:
|
||||
store.conn.execute(
|
||||
"UPDATE ip_fused SET last_pushed_at=NULL")
|
||||
notify_fused(store, capture)
|
||||
self.assertEqual(len(capture.posts), 1)
|
||||
self.assertIn("203.0.113.5", capture.posts[0])
|
||||
detector.run_once()
|
||||
store.sync_fused()
|
||||
notify_fused(store, capture)
|
||||
self.assertEqual(len(capture.posts), 1) # 24h cooldown
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,436 @@
|
||||
"""Tests for the continuous detector, the SQLite store and the dashboard."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.dirname(HERE)
|
||||
sys.path.insert(0, ROOT)
|
||||
|
||||
from graylog_ops import watch # noqa: E402
|
||||
|
||||
|
||||
class ClassifyTests(unittest.TestCase):
|
||||
def test_sensitive_probe_is_malicious(self):
|
||||
tier, score, reasons = watch.classify(44, {"probe_java": 44})
|
||||
self.assertEqual(tier, watch.TIER_MALICIOUS)
|
||||
self.assertGreaterEqual(score, 60)
|
||||
self.assertTrue(any("探测" in reason for reason in reasons))
|
||||
|
||||
def test_single_sensitive_hit_is_enough(self):
|
||||
tier, score, _ = watch.classify(18, {"probe_java": 18})
|
||||
self.assertEqual(tier, watch.TIER_MALICIOUS)
|
||||
|
||||
def test_dotbot_crawler_is_malicious(self):
|
||||
tier, score, reasons = watch.classify(352, {"bot_ua": 352, "not_found": 176})
|
||||
self.assertEqual(tier, watch.TIER_MALICIOUS)
|
||||
self.assertTrue(any("404" in reason for reason in reasons))
|
||||
|
||||
def test_login_burst_alone_is_only_observed(self):
|
||||
"""Students logging in repeatedly must not look suspicious."""
|
||||
tier, score, _ = watch.classify(2253, {"login_burst": 660})
|
||||
self.assertEqual(tier, watch.TIER_OBSERVE)
|
||||
|
||||
def test_login_attack_plus_burst_is_malicious(self):
|
||||
tier, score, _ = watch.classify(666, {"login_attack": 333, "login_burst": 666})
|
||||
self.assertEqual(tier, watch.TIER_MALICIOUS)
|
||||
|
||||
def test_busy_clean_ip_scores_zero(self):
|
||||
tier, score, reasons = watch.classify(5000, {})
|
||||
self.assertEqual(score, 0.0)
|
||||
self.assertEqual(reasons, [])
|
||||
|
||||
def test_404_ratio_needs_volume(self):
|
||||
_, score, _ = watch.classify(10, {"not_found": 5})
|
||||
self.assertEqual(score, 0.0)
|
||||
|
||||
|
||||
class _StubClient:
|
||||
"""Returns canned counts keyed by the exact query string."""
|
||||
|
||||
def __init__(self, by_query, totals=None):
|
||||
self.by_query = by_query
|
||||
self.totals = totals or {}
|
||||
|
||||
def terms(self, field, query, size, range_seconds, filter_str=None):
|
||||
if query.startswith("client_ip:("):
|
||||
ips = [part.strip().strip('"')
|
||||
for part in query[len("client_ip:("):-1].split(" OR ")]
|
||||
items = [(ip, self.totals[ip]) for ip in ips if ip in self.totals]
|
||||
else:
|
||||
items = self.by_query.get(query, [])
|
||||
return {"terms": [{"term": ip, "count": count} for ip, count in items],
|
||||
"total": sum(count for _, count in items)}
|
||||
|
||||
def search_views_messages(self, query="*", range_seconds=300, limit=5, offset=0,
|
||||
filter_str=None, from_ts=None, to_ts=None,
|
||||
timeout_ms=60000, sort_order="DESC"):
|
||||
return []
|
||||
|
||||
|
||||
class StoreTests(unittest.TestCase):
|
||||
def test_upsert_list_forget(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
try:
|
||||
record = {"ip": "203.0.113.9", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 88.0, "total": 120,
|
||||
"rules": {"sensitive_probe": 4}, "reasons": ["敏感路径探测 4 次"]}
|
||||
store.upsert(record, "2026-10-17T16:00:00Z")
|
||||
store.upsert(dict(record, score=90.0), "2026-10-17T16:01:00Z")
|
||||
rows = store.list()
|
||||
self.assertEqual(len(rows), 1)
|
||||
self.assertEqual(rows[0]["score"], 90.0)
|
||||
self.assertEqual(rows[0]["hits"], 2)
|
||||
self.assertEqual(rows[0]["rules"]["sensitive_probe"], 4)
|
||||
|
||||
self.assertEqual(store.stats()["by_tier"][watch.TIER_MALICIOUS], 1)
|
||||
self.assertEqual(store.stats()["events"], 1)
|
||||
|
||||
store.upsert({"ip": "198.51.100.7", "tier": watch.TIER_SUSPICIOUS,
|
||||
"score": 40.0, "total": 10, "rules": {}, "reasons": []},
|
||||
"2026-10-17T16:02:00Z")
|
||||
self.assertEqual(len(store.list(tier=watch.TIER_SUSPICIOUS)), 1)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
class PermanentRetentionTests(unittest.TestCase):
|
||||
def _store(self, directory):
|
||||
return watch.Store(os.path.join(directory, "ips.db"))
|
||||
|
||||
def test_ever_malicious_survives_downgrade(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = self._store(directory)
|
||||
try:
|
||||
base = {"ip": "203.0.113.9", "total": 10, "rules": {}, "reasons": []}
|
||||
store.upsert(dict(base, tier=watch.TIER_MALICIOUS, score=80.0),
|
||||
"2026-10-17T16:00:00Z")
|
||||
store.upsert(dict(base, tier=watch.TIER_OBSERVE, score=5.0),
|
||||
"2026-10-17T16:01:00Z")
|
||||
row = store.get("203.0.113.9")
|
||||
self.assertEqual(row["tier"], watch.TIER_OBSERVE)
|
||||
self.assertEqual(row["ever_malicious"], 1)
|
||||
self.assertEqual(row["last_malicious_at"], "2026-10-17T16:00:00Z")
|
||||
core = store.list(scope="core")
|
||||
self.assertEqual([item["ip"] for item in core], ["203.0.113.9"])
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
def test_forget_keeps_convicted_ips(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = self._store(directory)
|
||||
try:
|
||||
store.upsert({"ip": "203.0.113.9", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 80.0, "total": 10, "rules": {},
|
||||
"reasons": []}, "2026-10-01T00:00:00Z")
|
||||
store.upsert({"ip": "198.51.100.7", "tier": watch.TIER_OBSERVE,
|
||||
"score": 5.0, "total": 10, "rules": {},
|
||||
"reasons": []}, "2026-10-01T00:00:00Z")
|
||||
store.forget(3600, time.time())
|
||||
self.assertIsNotNone(store.get("203.0.113.9"))
|
||||
self.assertIsNone(store.get("198.51.100.7"))
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
def test_sorting_modes(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = self._store(directory)
|
||||
try:
|
||||
store.upsert({"ip": "203.0.113.2", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 70.0, "total": 10, "rules": {},
|
||||
"reasons": []}, "2026-10-17T16:00:00Z")
|
||||
store.upsert({"ip": "203.0.113.1", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 90.0, "total": 5, "rules": {},
|
||||
"reasons": []}, "2026-10-17T17:00:00Z")
|
||||
first = [row["ip"] for row in store.list(sort="first_seen", order="asc")]
|
||||
self.assertEqual(first, ["203.0.113.2", "203.0.113.1"])
|
||||
recent = [row["ip"] for row in store.list(sort="last_malicious",
|
||||
order="desc")]
|
||||
self.assertEqual(recent, ["203.0.113.1", "203.0.113.2"])
|
||||
score = [row["ip"] for row in store.list(sort="score", order="desc")]
|
||||
self.assertEqual(score, ["203.0.113.1", "203.0.113.2"])
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
class TextSanitisingTests(unittest.TestCase):
|
||||
def test_strips_control_and_replacement_chars(self):
|
||||
self.assertEqual(watch._clean_text("a\x00b\ufffdc"), "a·b·c")
|
||||
|
||||
def test_scrubs_binary_header_field(self):
|
||||
self.assertEqual(watch._scrub_binary("\ufffd\ufffdk\ufffdU\ufffd\ufffd"),
|
||||
"[非 UTF-8 数据]")
|
||||
self.assertEqual(watch._scrub_binary("GET"), "GET")
|
||||
|
||||
|
||||
class DetectorTests(unittest.TestCase):
|
||||
def _detector(self, by_query, store):
|
||||
fields = dict(watch.DEFAULT_FIELDS)
|
||||
probe = watch.Detector(None, store, fields)
|
||||
queries = {rule["name"]: rule["query"] for rule in probe.rules}
|
||||
mapping = dict(by_query)
|
||||
mapping.setdefault(queries["not_found"], [])
|
||||
return watch.Detector(_StubClient(mapping), store, fields, window=300)
|
||||
|
||||
def test_automation_is_downgraded(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
try:
|
||||
probe = watch.Detector(None, store, dict(watch.DEFAULT_FIELDS))
|
||||
q = {rule["name"]: rule["query"] for rule in probe.rules}
|
||||
by_query = {
|
||||
"*": [("124.70.2.111", 666)],
|
||||
q["login_attack"]: [("124.70.2.111", 333)],
|
||||
q["login_burst"]: [("124.70.2.111", 666)],
|
||||
q["not_found"]: [],
|
||||
q["probe_java"]: [],
|
||||
q["bad_ua"]: [],
|
||||
q["bot_ua"]: [],
|
||||
q["probe_config"]: [],
|
||||
q["probe_cms"]: [],
|
||||
q["probe_iot"]: [],
|
||||
q["probe_devops"]: [],
|
||||
q["probe_webshell"]: [],
|
||||
q["exploit_payload"]: [],
|
||||
'client_ip:"124.70.2.111"': [
|
||||
("/prod-api/biz/liveCourseChapter/msgSubscription", 666)],
|
||||
}
|
||||
detector = watch.Detector(_StubClient(by_query), store,
|
||||
dict(watch.DEFAULT_FIELDS), window=300)
|
||||
result = detector.run_once()
|
||||
self.assertEqual(result["automation"], 1)
|
||||
row = store.get("124.70.2.111")
|
||||
self.assertEqual(row["tier"], watch.TIER_AUTOMATION)
|
||||
self.assertTrue(any("单一接口" in r for r in row["reasons"]))
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
def test_low_volume_scanner_is_still_caught(self):
|
||||
"""A scanner with 18 requests must not need a top-volume slot."""
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
try:
|
||||
probe = watch.Detector(None, store, dict(watch.DEFAULT_FIELDS))
|
||||
q = {rule["name"]: rule["query"] for rule in probe.rules}
|
||||
by_query = {
|
||||
"*": [("10.185.9.41", 5000), ("202.201.16.87", 4000)],
|
||||
q["probe_java"]: [("5.188.86.234", 18)],
|
||||
}
|
||||
detector = watch.Detector(
|
||||
_StubClient(by_query, totals={"5.188.86.234": 18}), store,
|
||||
dict(watch.DEFAULT_FIELDS), window=300)
|
||||
result = detector.run_once()
|
||||
self.assertEqual(result["malicious"], 1)
|
||||
self.assertEqual(store.get("5.188.86.234")["tier"], watch.TIER_MALICIOUS)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
def test_internal_and_allowlisted_are_skipped(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
try:
|
||||
probe = watch.Detector(None, store, dict(watch.DEFAULT_FIELDS))
|
||||
q = {rule["name"]: rule["query"] for rule in probe.rules}
|
||||
by_query = {
|
||||
"*": [("10.185.9.41", 5000), ("203.0.113.5", 30), ("198.51.100.9", 30)],
|
||||
q["probe_java"]: [("203.0.113.5", 30), ("198.51.100.9", 30)],
|
||||
}
|
||||
detector = watch.Detector(
|
||||
_StubClient(by_query), store, dict(watch.DEFAULT_FIELDS),
|
||||
window=300, allowlist=lambda ip: ip == "198.51.100.9")
|
||||
detector.run_once()
|
||||
self.assertIsNone(store.get("10.185.9.41"))
|
||||
self.assertIsNone(store.get("198.51.100.9"))
|
||||
self.assertEqual(store.get("203.0.113.5")["tier"], watch.TIER_MALICIOUS)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
class SampleTests(unittest.TestCase):
|
||||
def test_sample_keeps_all_source_fields(self):
|
||||
message = {
|
||||
"timestamp": "2026-10-17T05:08:58.046Z",
|
||||
"client_ip": "51.158.145.102", "client_port": 46990,
|
||||
"server_ip": "10.185.9.90", "server_port": 80,
|
||||
"host": "61.178.20.46:443", "bigip": "www.gsedu.edu.cn_2_4600",
|
||||
"vs": "/Common/SmartEDU_https", "method": "GET",
|
||||
"uri": "/vendor/phpunit/phpunit/src/Util/PHP/eval-stdin.php",
|
||||
"status": "403", "ua": "libredtail-http",
|
||||
"client_ip_country_code": "FR", "rt": "0.031",
|
||||
"message": 'event_type="req",client_ip="51.158.145.102"',
|
||||
}
|
||||
sample = watch._sample(message)
|
||||
for key in ("client_port", "server_port", "bigip", "vs", "rt",
|
||||
"client_ip_country_code", "message"):
|
||||
self.assertIn(key, sample)
|
||||
self.assertEqual(sample["client_port"], "46990")
|
||||
|
||||
def test_sample_drops_empty_and_na_values(self):
|
||||
sample = watch._sample({"client_ip": "1.2.3.4",
|
||||
"client_ip_city_name": "N/A",
|
||||
"referer": ""})
|
||||
self.assertEqual(sample, {"client_ip": "1.2.3.4"})
|
||||
|
||||
|
||||
class SnapshotTests(unittest.TestCase):
|
||||
def test_compact_evidence_is_bounded(self):
|
||||
evidence = {
|
||||
"window": 89600,
|
||||
"uris": [{"term": "/probe/%d" % i, "count": i} for i in range(30)],
|
||||
"statuses": [{"term": str(code), "count": 1} for code in range(10)],
|
||||
"methods": [{"term": "GET", "count": 1}],
|
||||
"hosts": [{"term": "h", "count": 1}],
|
||||
"server_ips": [{"term": "10.0.0.1", "count": 1}],
|
||||
"vips": [{"term": "vip", "count": 1}],
|
||||
"uas": [{"term": "ua", "count": 1}],
|
||||
"samples": [{} for _ in range(5)],
|
||||
"timeline": [{"timestamp": "t%d" % i, "count": 1} for i in range(200)],
|
||||
"peak": {"timestamp": "t", "count": 3},
|
||||
}
|
||||
compact = watch._compact_evidence(evidence)
|
||||
self.assertTrue(compact["compact"])
|
||||
self.assertLessEqual(len(compact["uris"]), 10)
|
||||
self.assertLessEqual(len(compact["timeline"]), 72)
|
||||
self.assertEqual(compact["sample_count"], 5)
|
||||
self.assertEqual(compact["window"], 89600)
|
||||
|
||||
|
||||
class PathPrefixTests(unittest.TestCase):
|
||||
def test_root_is_hidden_and_prefix_works(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
store.upsert({"ip": "203.0.113.9", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 91.0, "total": 120, "window": 3600,
|
||||
"rules": {"probe_java": 4},
|
||||
"reasons": ["探测 4 次"]}, "2026-10-17T16:00:00Z")
|
||||
store.sync_fused()
|
||||
server = ThreadingHTTPServer(
|
||||
("127.0.0.1", 0),
|
||||
watch.make_handler(store, 3600, 60, None, "/s/secret123"))
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
base = "http://127.0.0.1:%d" % server.server_address[1]
|
||||
try:
|
||||
with self.assertRaises(urllib.error.HTTPError) as ctx:
|
||||
urllib.request.urlopen(base + "/", timeout=5)
|
||||
self.assertEqual(ctx.exception.code, 404)
|
||||
|
||||
with urllib.request.urlopen(base + "/s/secret123/", timeout=5) as response:
|
||||
page = response.read().decode("utf-8")
|
||||
self.assertIn('const BASE = "/s/secret123"', page)
|
||||
|
||||
with urllib.request.urlopen(base + "/s/secret123/api/ips", timeout=5) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
self.assertEqual(payload["count"], 1)
|
||||
|
||||
with self.assertRaises(urllib.error.HTTPError):
|
||||
urllib.request.urlopen(base + "/api/ips", timeout=5)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
store.close()
|
||||
|
||||
|
||||
class StatusIndicatorTests(unittest.TestCase):
|
||||
def test_status_endpoint(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
|
||||
class DetectorStub:
|
||||
last_success = time.time() - 5
|
||||
last_error = None
|
||||
last_run = "2026-10-17T16:00:00Z"
|
||||
|
||||
server = ThreadingHTTPServer(
|
||||
("127.0.0.1", 0),
|
||||
watch.make_handler(store, 3600, 60, DetectorStub()))
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
base = "http://127.0.0.1:%d" % server.server_address[1]
|
||||
try:
|
||||
with urllib.request.urlopen(base + "/api/status", timeout=5) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
self.assertEqual(payload["graylog"], "ok")
|
||||
|
||||
DetectorStub.last_error = "connection refused"
|
||||
DetectorStub.last_success = None
|
||||
with urllib.request.urlopen(base + "/api/status", timeout=5) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
self.assertEqual(payload["graylog"], "error")
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
store.close()
|
||||
|
||||
|
||||
class DashboardTests(unittest.TestCase):
|
||||
def test_api_and_html(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = watch.Store(os.path.join(directory, "ips.db"))
|
||||
store.upsert({"ip": "203.0.113.9", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 91.0, "total": 120, "window": 3600,
|
||||
"rules": {"sensitive_probe": 4},
|
||||
"reasons": ["敏感路径探测 4 次"]}, "2026-10-17T16:00:00Z")
|
||||
store.upsert({"ip": "198.51.100.77", "tier": watch.TIER_MALICIOUS,
|
||||
"score": 70.0, "total": 653344, "window": 3600,
|
||||
"risk": "shared_egress",
|
||||
"rules": {"sensitive_probe": 4},
|
||||
"reasons": ["疑似共享出口"]}, "2026-10-17T16:00:00Z")
|
||||
store.sync_fused()
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0),
|
||||
watch.make_handler(store, 3600, 60))
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
base = "http://127.0.0.1:%d" % server.server_address[1]
|
||||
try:
|
||||
with urllib.request.urlopen(base + "/", timeout=5) as response:
|
||||
page = response.read().decode("utf-8")
|
||||
self.assertIn("恶意 IP 清单", page)
|
||||
self.assertIn("/api/ips", page)
|
||||
|
||||
with urllib.request.urlopen(base + "/api/ips", timeout=5) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
self.assertEqual(payload["count"], 2)
|
||||
self.assertEqual({item["ip"] for item in payload["items"]},
|
||||
{"203.0.113.9", "198.51.100.77"})
|
||||
self.assertTrue(all(item["sources"] == "feature"
|
||||
for item in payload["items"]))
|
||||
|
||||
with urllib.request.urlopen(base + "/api/stats", timeout=5) as response:
|
||||
stats = json.loads(response.read().decode("utf-8"))
|
||||
self.assertEqual(stats["core"], 2)
|
||||
self.assertNotIn("review", stats)
|
||||
|
||||
with urllib.request.urlopen(base + "/api/ip/203.0.113.9", timeout=5) as response:
|
||||
detail = json.loads(response.read().decode("utf-8"))
|
||||
self.assertEqual(detail["item"]["tier"], watch.TIER_MALICIOUS)
|
||||
self.assertFalse(detail["evidence"]["available"])
|
||||
|
||||
with urllib.request.urlopen(base + "/api/export.csv", timeout=5) as response:
|
||||
body = response.read().decode("utf-8")
|
||||
self.assertIn("text/csv", response.headers.get("Content-Type", ""))
|
||||
self.assertIn("ip,tier,score", body)
|
||||
self.assertIn("203.0.113.9", body)
|
||||
self.assertIn("198.51.100.77", body)
|
||||
|
||||
with urllib.request.urlopen(base + "/api/export.txt", timeout=5) as response:
|
||||
text = response.read().decode("utf-8")
|
||||
self.assertEqual(set(text.split()),
|
||||
{"203.0.113.9", "198.51.100.77"})
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user