初始提交：恶意 IP 特征+行为融合检测系统
- 特征检测（规则/签名）与行为检测（24h 统计）单进程运行 - 融合名单按首次发现时间倒序，展示位置/归属（GeoIP + ASN） - 钉钉融合推送（加签、24h 节流、去重） - 页面状态灯、CSV 导出、Docker 镜像构建文件
This commit is contained in:
DeepFal committed
commit
2d76ed6672
42 files changed
+10645
No files matched your search
@@ -0,0 +1,463 @@
|
||||
"""Thin, defensive wrappers around the Graylog 4.0 REST API.
|
||||
|
||||
Every method returns plain Python data (dict/list) that has been normalised
|
||||
just enough for callers not to care about cosmetic response differences.
|
||||
Anything that is version specific and ambiguous is passed through untouched
|
||||
and documented as such.
|
||||
|
||||
Endpoint reference for 4.0 lives in the server itself:
|
||||
``http://<graylog>:9002/api/api-browser/``
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from .http import GraylogError
|
||||
|
||||
|
||||
def as_list(data, *keys):
|
||||
"""Best effort extraction of a list from a Graylog response."""
|
||||
if isinstance(data, list):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
for key in keys:
|
||||
value = data.get(key)
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
for value in data.values():
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return []
|
||||
|
||||
|
||||
def dig(data, *keys, **kwargs):
|
||||
default = kwargs.get("default")
|
||||
current = data
|
||||
for key in keys:
|
||||
if not isinstance(current, dict):
|
||||
return default
|
||||
current = current.get(key)
|
||||
if current is None:
|
||||
return default
|
||||
return current
|
||||
|
||||
|
||||
def normalize_terms(data):
|
||||
"""Turn a terms aggregation response into ``[{"term", "count"}, ...]``.
|
||||
|
||||
Graylog has shipped a couple of shapes for this over the years, so accept
|
||||
the common ones instead of assuming a single layout.
|
||||
"""
|
||||
items = as_list(data, "terms", "values", "results")
|
||||
rows = []
|
||||
for item in items:
|
||||
if isinstance(item, dict):
|
||||
term = item.get("term", item.get("value", item.get("name")))
|
||||
count = item.get("count", item.get("doc_count", item.get("hits", 0)))
|
||||
rows.append({"term": term, "count": count})
|
||||
elif isinstance(item, (list, tuple)) and len(item) >= 2:
|
||||
rows.append({"term": item[0], "count": item[1]})
|
||||
return rows
|
||||
|
||||
|
||||
def normalize_histogram(data):
|
||||
"""Turn a histogram response into ``[{"timestamp", "count"}, ...]``."""
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
results = data.get("results")
|
||||
rows = []
|
||||
if isinstance(results, dict):
|
||||
for timestamp, value in results.items():
|
||||
if isinstance(value, dict):
|
||||
count = value.get("count", value.get("doc_count", 0))
|
||||
else:
|
||||
count = value
|
||||
rows.append({"timestamp": timestamp, "count": count or 0})
|
||||
elif isinstance(results, list):
|
||||
for item in results:
|
||||
if isinstance(item, dict):
|
||||
rows.append({
|
||||
"timestamp": item.get("timestamp", item.get("time")),
|
||||
"count": item.get("count", 0),
|
||||
})
|
||||
rows.sort(key=lambda row: str(row["timestamp"]))
|
||||
return rows
|
||||
|
||||
|
||||
class GraylogClient:
|
||||
def __init__(self, http):
|
||||
self.http = http
|
||||
# Which aggregation backend to use. Graylog 4.0 removed the legacy
|
||||
# /api/search/universal/{terms,histogram,stats} endpoints, so 4.x must
|
||||
# use the views API (POST /api/views/search/sync). 3.x has no views API.
|
||||
self.agg_backend = None
|
||||
# Legacy fallback only: some builds only accept "size" on terms,
|
||||
# others tolerate/require "limit".
|
||||
self.terms_size_param = "size"
|
||||
|
||||
# ---------------------------------------------------------------- system
|
||||
def system_info(self):
|
||||
return self.http.get("/api/system") or {}
|
||||
|
||||
def cluster_nodes(self):
|
||||
return as_list(self.http.get("/api/system/cluster/nodes"), "nodes")
|
||||
|
||||
def indexer_health(self):
|
||||
return self.http.get("/api/system/indexer/cluster/health") or {}
|
||||
|
||||
def indexer_cluster_name(self):
|
||||
return self.http.get("/api/system/indexer/cluster/name") or ""
|
||||
|
||||
def deflector(self):
|
||||
return self.http.get("/api/system/deflector") or {}
|
||||
|
||||
def field_types(self):
|
||||
data = self.http.get("/api/system/fields")
|
||||
if isinstance(data, dict):
|
||||
fields = data.get("fields")
|
||||
if isinstance(fields, list):
|
||||
return fields
|
||||
if isinstance(fields, dict):
|
||||
return [{"name": name, "type": kind} for name, kind in fields.items()]
|
||||
return as_list(data, "fields")
|
||||
|
||||
def license_info(self):
|
||||
return self.http.get("/api/system/license") or {}
|
||||
|
||||
def index_sets(self):
|
||||
data = self.http.get("/api/system/indices/index_sets", {"skip": 0, "limit": 100})
|
||||
return as_list(data, "index_sets")
|
||||
|
||||
def indices(self):
|
||||
data = self.http.get("/api/system/indexer/indices")
|
||||
if isinstance(data, dict):
|
||||
# Strict per-key access: as_list()'s "first list in the dict"
|
||||
# fallback would merge the same list three times here.
|
||||
merged = []
|
||||
for key in ("indices", "closed_indices", "reopened_indices"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, list):
|
||||
merged.extend(value)
|
||||
return merged
|
||||
return as_list(data)
|
||||
|
||||
# --------------------------------------------------------------- streams
|
||||
def streams(self):
|
||||
return as_list(self.http.get("/api/streams"), "streams")
|
||||
|
||||
def stream(self, stream_id):
|
||||
return self.http.get("/api/streams/%s" % stream_id) or {}
|
||||
|
||||
# ---------------------------------------------------------------- inputs
|
||||
def inputs(self):
|
||||
return as_list(self.http.get("/api/system/inputs"), "inputs")
|
||||
|
||||
def extractors(self, input_id):
|
||||
data = self.http.get("/api/system/inputs/%s/extractors" % input_id)
|
||||
return as_list(data, "extractors")
|
||||
|
||||
# ------------------------------------------------------------- pipelines
|
||||
def pipelines(self):
|
||||
return as_list(self.http.get("/api/system/pipelines/pipeline"), "pipelines")
|
||||
|
||||
def pipeline_rules(self):
|
||||
return as_list(self.http.get("/api/system/pipelines/rule"), "rules")
|
||||
|
||||
# ---------------------------------------------------------------- events
|
||||
def _paged(self, path, key, limit=None, per_page=50, params=None):
|
||||
items = []
|
||||
page = 1
|
||||
while True:
|
||||
query = dict(params or {})
|
||||
query.update({"page": page, "per_page": per_page})
|
||||
data = self.http.get(path, query)
|
||||
batch = as_list(data, key)
|
||||
if not batch:
|
||||
return items
|
||||
items.extend(batch)
|
||||
if limit is not None and len(items) >= limit:
|
||||
return items[:limit]
|
||||
# Trust the server's total when it tells us there is nothing left;
|
||||
# this also stops runaway pagination on servers that ignore `page`.
|
||||
total = dig(data, "total")
|
||||
if isinstance(total, int) and len(items) >= total:
|
||||
return items
|
||||
if len(batch) < per_page:
|
||||
return items
|
||||
page += 1
|
||||
if page > 500: # defensive stop
|
||||
return items
|
||||
|
||||
def event_definitions(self, limit=None, per_page=50):
|
||||
return self._paged(
|
||||
"/api/events/definitions",
|
||||
"event_definitions",
|
||||
limit=limit,
|
||||
per_page=per_page,
|
||||
params={"sort": "title", "order": "asc"},
|
||||
)
|
||||
|
||||
def notifications(self, limit=None, per_page=50):
|
||||
return self._paged(
|
||||
"/api/events/notifications", "notifications", limit=limit, per_page=per_page
|
||||
)
|
||||
|
||||
def events(self, limit=None, per_page=50):
|
||||
data = self.http.get("/api/events", {"page": 1, "per_page": per_page})
|
||||
items = as_list(data, "events")
|
||||
return items[:limit] if limit else items
|
||||
|
||||
# --------------------------------------------------------------- sidecar
|
||||
def sidecars(self):
|
||||
return as_list(self.http.get("/api/sidecars"), "sidecars")
|
||||
|
||||
# ---------------------------------------------------------------- search
|
||||
@staticmethod
|
||||
def _time_params(range_seconds=None, from_ts=None, to_ts=None):
|
||||
if from_ts and to_ts:
|
||||
return {"from": from_ts, "to": to_ts}
|
||||
return {"range": int(range_seconds if range_seconds is not None else 300)}
|
||||
|
||||
def search_relative(self, query="*", range_seconds=300, limit=100, offset=0,
|
||||
sort="timestamp:desc", fields=None, filter_str=None):
|
||||
params = {"query": query or "*", "limit": int(limit), "offset": int(offset), "sort": sort}
|
||||
params.update(self._time_params(range_seconds=range_seconds))
|
||||
if fields:
|
||||
params["fields"] = ",".join(fields)
|
||||
if filter_str:
|
||||
params["filter"] = filter_str
|
||||
data = self.http.get("/api/search/universal/relative", params)
|
||||
return self._messages(data)
|
||||
|
||||
def search_absolute(self, query="*", from_ts=None, to_ts=None, limit=100, offset=0,
|
||||
sort="timestamp:desc", fields=None, filter_str=None):
|
||||
params = {"query": query or "*", "limit": int(limit), "offset": int(offset), "sort": sort}
|
||||
params.update(self._time_params(from_ts=from_ts, to_ts=to_ts))
|
||||
if fields:
|
||||
params["fields"] = ",".join(fields)
|
||||
if filter_str:
|
||||
params["filter"] = filter_str
|
||||
data = self.http.get("/api/search/universal/absolute", params)
|
||||
return self._messages(data)
|
||||
|
||||
@staticmethod
|
||||
def _messages(data):
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
messages = data.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
return []
|
||||
result = []
|
||||
for item in messages:
|
||||
if isinstance(item, dict) and isinstance(item.get("message"), dict):
|
||||
message = dict(item["message"])
|
||||
if item.get("index") and "_index" not in message:
|
||||
message["_index"] = item["index"]
|
||||
result.append(message)
|
||||
elif isinstance(item, dict):
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------- views aggregation
|
||||
BASE_PATH = "/api/views/search"
|
||||
|
||||
@staticmethod
|
||||
def _timerange(range_seconds=None, from_ts=None, to_ts=None):
|
||||
if from_ts and to_ts:
|
||||
return {"type": "absolute", "from": from_ts, "to": to_ts}
|
||||
return {"type": "relative", "range": int(range_seconds or 300)}
|
||||
|
||||
@staticmethod
|
||||
def _stream_filter(filter_str):
|
||||
"""Translate the legacy ``streams:<id>[,<id>]`` filter to a views filter."""
|
||||
if not filter_str or not filter_str.startswith("streams:"):
|
||||
return None
|
||||
ids = [part.strip() for part in filter_str[len("streams:"):].split(",") if part.strip()]
|
||||
if not ids:
|
||||
return None
|
||||
return {"type": "or", "filters": [{"type": "stream", "id": sid} for sid in ids]}
|
||||
|
||||
def _views_pivot(self, query, timerange, row_groups, column_groups, series,
|
||||
sort, filter_str=None, timeout_ms=60000):
|
||||
search_type = {
|
||||
"type": "pivot",
|
||||
"id": "st1",
|
||||
"series": series,
|
||||
"row_groups": row_groups,
|
||||
"column_groups": column_groups,
|
||||
"sort": sort,
|
||||
"rollup": True,
|
||||
}
|
||||
query_object = {
|
||||
"id": "q1",
|
||||
"timerange": timerange,
|
||||
"query": {"type": "elasticsearch", "query_string": query or "*"},
|
||||
"search_types": [search_type],
|
||||
}
|
||||
stream_filter = self._stream_filter(filter_str)
|
||||
if stream_filter:
|
||||
query_object["filter"] = stream_filter
|
||||
body = {"queries": [query_object], "parameters": []}
|
||||
data = self.http.request(
|
||||
"POST", self.BASE_PATH + "/sync",
|
||||
params={"timeout": int(timeout_ms)}, body=body,
|
||||
)
|
||||
results = ((data or {}).get("results") or {}).get("q1") or {}
|
||||
errors = results.get("errors") or []
|
||||
if errors:
|
||||
raise GraylogError(
|
||||
"views search error: %s" % json.dumps(errors, ensure_ascii=False)[:400],
|
||||
path=self.BASE_PATH + "/sync", method="POST",
|
||||
)
|
||||
search_types = results.get("search_types") or {}
|
||||
if isinstance(search_types, list):
|
||||
search_types = {item.get("id"): item for item in search_types
|
||||
if isinstance(item, dict)}
|
||||
return search_types.get("st1") or {}
|
||||
|
||||
def _terms_views(self, field, query, size, range_seconds, from_ts, to_ts, filter_str):
|
||||
search_type = self._views_pivot(
|
||||
query=query,
|
||||
timerange=self._timerange(range_seconds, from_ts, to_ts),
|
||||
row_groups=[{"type": "values", "field": field, "limit": int(size)}],
|
||||
column_groups=[],
|
||||
series=[{"type": "count", "id": "c1"}],
|
||||
sort=[{"type": "pivot", "field": "c1", "direction": "Descending"}],
|
||||
filter_str=filter_str,
|
||||
)
|
||||
terms = []
|
||||
for row in search_type.get("rows") or []:
|
||||
key = row.get("key") or []
|
||||
if not key:
|
||||
continue # rollup row (total)
|
||||
values = row.get("values") or []
|
||||
count = values[0].get("value") if values else 0
|
||||
terms.append({"term": key[0], "count": count})
|
||||
return {
|
||||
"field": field,
|
||||
"terms": terms,
|
||||
"total": search_type.get("total", sum(row["count"] or 0 for row in terms)),
|
||||
}
|
||||
|
||||
def _histogram_views(self, query, range_seconds, interval, filter_str, from_ts, to_ts):
|
||||
search_type = self._views_pivot(
|
||||
query=query,
|
||||
timerange=self._timerange(range_seconds, from_ts, to_ts),
|
||||
row_groups=[],
|
||||
column_groups=[{"type": "time", "field": "timestamp", "interval": interval}],
|
||||
series=[{"type": "count", "id": "c1"}],
|
||||
sort=[],
|
||||
filter_str=filter_str,
|
||||
)
|
||||
results = {}
|
||||
for row in search_type.get("rows") or []:
|
||||
for item in row.get("values") or []:
|
||||
key = item.get("key") or []
|
||||
# Column keys are [timestamp, series_id]; the pivot rollup row
|
||||
# only carries [series_id] and must not become a time bucket.
|
||||
if len(key) < 2:
|
||||
continue
|
||||
results[key[0]] = {"count": item.get("value", 0)}
|
||||
return {"results": results, "interval": interval}
|
||||
|
||||
# ------------------------------------------------------------- aggregation
|
||||
def search_views_messages(self, query="*", range_seconds=300, limit=5,
|
||||
offset=0, filter_str=None, from_ts=None, to_ts=None,
|
||||
timeout_ms=60000, sort_order="DESC"):
|
||||
"""Message search through the views API (the legacy endpoint is flaky
|
||||
on long windows with complex queries and drops the connection)."""
|
||||
search_type = {
|
||||
"type": "messages",
|
||||
"id": "st1",
|
||||
"limit": int(limit),
|
||||
"offset": int(offset),
|
||||
"sort": [{"field": "timestamp",
|
||||
"order": "ASC" if str(sort_order).upper() == "ASC" else "DESC"}],
|
||||
}
|
||||
query_object = {
|
||||
"id": "q1",
|
||||
"timerange": self._timerange(range_seconds, from_ts, to_ts),
|
||||
"query": {"type": "elasticsearch", "query_string": query or "*"},
|
||||
"search_types": [search_type],
|
||||
}
|
||||
stream_filter = self._stream_filter(filter_str)
|
||||
if stream_filter:
|
||||
query_object["filter"] = stream_filter
|
||||
body = {"queries": [query_object], "parameters": []}
|
||||
data = self.http.request(
|
||||
"POST", self.BASE_PATH + "/sync",
|
||||
params={"timeout": int(timeout_ms)}, body=body,
|
||||
)
|
||||
results = ((data or {}).get("results") or {}).get("q1") or {}
|
||||
errors = results.get("errors") or []
|
||||
if errors:
|
||||
raise GraylogError(
|
||||
"views message search error: %s" % json.dumps(errors, ensure_ascii=False)[:300],
|
||||
path=self.BASE_PATH + "/sync", method="POST",
|
||||
)
|
||||
search_types = results.get("search_types") or {}
|
||||
if isinstance(search_types, list):
|
||||
search_types = {item.get("id"): item for item in search_types
|
||||
if isinstance(item, dict)}
|
||||
search_type_result = search_types.get("st1") or {}
|
||||
return self._messages({"messages": search_type_result.get("messages") or []})
|
||||
|
||||
def terms(self, field, query="*", size=50, range_seconds=300,
|
||||
from_ts=None, to_ts=None, filter_str=None):
|
||||
if self.agg_backend in (None, "views"):
|
||||
try:
|
||||
data = self._terms_views(field, query, size, range_seconds,
|
||||
from_ts, to_ts, filter_str)
|
||||
self.agg_backend = "views"
|
||||
return data
|
||||
except GraylogError as exc:
|
||||
if exc.status != 404 or self.agg_backend == "views":
|
||||
raise
|
||||
self.agg_backend = "legacy"
|
||||
return self._terms_legacy(field, query, size, range_seconds,
|
||||
from_ts, to_ts, filter_str)
|
||||
|
||||
def _terms_legacy(self, field, query, size, range_seconds, from_ts, to_ts, filter_str):
|
||||
params = {"query": query or "*", "field": field}
|
||||
params.update(self._time_params(range_seconds, from_ts, to_ts))
|
||||
if filter_str:
|
||||
params["filter"] = filter_str
|
||||
params[self.terms_size_param] = int(size)
|
||||
try:
|
||||
return self.http.get("/api/search/universal/terms", params)
|
||||
except GraylogError as exc:
|
||||
# Older/newer builds disagree on the parameter name; retry once.
|
||||
if exc.status in (400, 422) and self.terms_size_param == "size":
|
||||
params.pop("size", None)
|
||||
params["limit"] = int(size)
|
||||
data = self.http.get("/api/search/universal/terms", params)
|
||||
self.terms_size_param = "limit"
|
||||
return data
|
||||
raise
|
||||
|
||||
def histogram(self, query="*", range_seconds=3600, interval="1m",
|
||||
filter_str=None, from_ts=None, to_ts=None):
|
||||
if self.agg_backend in (None, "views"):
|
||||
try:
|
||||
data = self._histogram_views(query, range_seconds, interval,
|
||||
filter_str, from_ts, to_ts)
|
||||
self.agg_backend = "views"
|
||||
return data
|
||||
except GraylogError as exc:
|
||||
if exc.status != 404 or self.agg_backend == "views":
|
||||
raise
|
||||
self.agg_backend = "legacy"
|
||||
params = {"query": query or "*", "interval": interval}
|
||||
params.update(self._time_params(range_seconds, from_ts, to_ts))
|
||||
if filter_str:
|
||||
params["filter"] = filter_str
|
||||
return self.http.get("/api/search/universal/histogram", params)
|
||||
|
||||
def stats(self, field, query="*", range_seconds=300, stats=None,
|
||||
filter_str=None, from_ts=None, to_ts=None):
|
||||
params = {"query": query or "*", "field": field}
|
||||
params.update(self._time_params(range_seconds, from_ts, to_ts))
|
||||
if stats:
|
||||
params["stats"] = ",".join(stats)
|
||||
if filter_str:
|
||||
params["filter"] = filter_str
|
||||
return self.http.get("/api/search/universal/stats", params)
|
||||
Reference in new issue
Block a user