DeepFal
2d76ed6672
- 特征检测（规则/签名）与行为检测（24h 统计）单进程运行 - 融合名单按首次发现时间倒序，展示位置/归属（GeoIP + ASN） - 钉钉融合推送（加签、24h 节流、去重） - 页面状态灯、CSV 导出、Docker 镜像构建文件
464 lines
19 KiB
Python
464 lines
19 KiB
Python
"""Thin, defensive wrappers around the Graylog 4.0 REST API.
|
|
|
|
Every method returns plain Python data (dict/list) that has been normalised
|
|
just enough for callers not to care about cosmetic response differences.
|
|
Anything that is version specific and ambiguous is passed through untouched
|
|
and documented as such.
|
|
|
|
Endpoint reference for 4.0 lives in the server itself:
|
|
``http://<graylog>:9002/api/api-browser/``
|
|
"""
|
|
|
|
import json
|
|
|
|
from .http import GraylogError
|
|
|
|
|
|
def as_list(data, *keys):
|
|
"""Best effort extraction of a list from a Graylog response."""
|
|
if isinstance(data, list):
|
|
return data
|
|
if isinstance(data, dict):
|
|
for key in keys:
|
|
value = data.get(key)
|
|
if isinstance(value, list):
|
|
return value
|
|
for value in data.values():
|
|
if isinstance(value, list):
|
|
return value
|
|
return []
|
|
|
|
|
|
def dig(data, *keys, **kwargs):
|
|
default = kwargs.get("default")
|
|
current = data
|
|
for key in keys:
|
|
if not isinstance(current, dict):
|
|
return default
|
|
current = current.get(key)
|
|
if current is None:
|
|
return default
|
|
return current
|
|
|
|
|
|
def normalize_terms(data):
|
|
"""Turn a terms aggregation response into ``[{"term", "count"}, ...]``.
|
|
|
|
Graylog has shipped a couple of shapes for this over the years, so accept
|
|
the common ones instead of assuming a single layout.
|
|
"""
|
|
items = as_list(data, "terms", "values", "results")
|
|
rows = []
|
|
for item in items:
|
|
if isinstance(item, dict):
|
|
term = item.get("term", item.get("value", item.get("name")))
|
|
count = item.get("count", item.get("doc_count", item.get("hits", 0)))
|
|
rows.append({"term": term, "count": count})
|
|
elif isinstance(item, (list, tuple)) and len(item) >= 2:
|
|
rows.append({"term": item[0], "count": item[1]})
|
|
return rows
|
|
|
|
|
|
def normalize_histogram(data):
|
|
"""Turn a histogram response into ``[{"timestamp", "count"}, ...]``."""
|
|
if not isinstance(data, dict):
|
|
return []
|
|
results = data.get("results")
|
|
rows = []
|
|
if isinstance(results, dict):
|
|
for timestamp, value in results.items():
|
|
if isinstance(value, dict):
|
|
count = value.get("count", value.get("doc_count", 0))
|
|
else:
|
|
count = value
|
|
rows.append({"timestamp": timestamp, "count": count or 0})
|
|
elif isinstance(results, list):
|
|
for item in results:
|
|
if isinstance(item, dict):
|
|
rows.append({
|
|
"timestamp": item.get("timestamp", item.get("time")),
|
|
"count": item.get("count", 0),
|
|
})
|
|
rows.sort(key=lambda row: str(row["timestamp"]))
|
|
return rows
|
|
|
|
|
|
class GraylogClient:
|
|
def __init__(self, http):
|
|
self.http = http
|
|
# Which aggregation backend to use. Graylog 4.0 removed the legacy
|
|
# /api/search/universal/{terms,histogram,stats} endpoints, so 4.x must
|
|
# use the views API (POST /api/views/search/sync). 3.x has no views API.
|
|
self.agg_backend = None
|
|
# Legacy fallback only: some builds only accept "size" on terms,
|
|
# others tolerate/require "limit".
|
|
self.terms_size_param = "size"
|
|
|
|
# ---------------------------------------------------------------- system
|
|
def system_info(self):
|
|
return self.http.get("/api/system") or {}
|
|
|
|
def cluster_nodes(self):
|
|
return as_list(self.http.get("/api/system/cluster/nodes"), "nodes")
|
|
|
|
def indexer_health(self):
|
|
return self.http.get("/api/system/indexer/cluster/health") or {}
|
|
|
|
def indexer_cluster_name(self):
|
|
return self.http.get("/api/system/indexer/cluster/name") or ""
|
|
|
|
def deflector(self):
|
|
return self.http.get("/api/system/deflector") or {}
|
|
|
|
def field_types(self):
|
|
data = self.http.get("/api/system/fields")
|
|
if isinstance(data, dict):
|
|
fields = data.get("fields")
|
|
if isinstance(fields, list):
|
|
return fields
|
|
if isinstance(fields, dict):
|
|
return [{"name": name, "type": kind} for name, kind in fields.items()]
|
|
return as_list(data, "fields")
|
|
|
|
def license_info(self):
|
|
return self.http.get("/api/system/license") or {}
|
|
|
|
def index_sets(self):
|
|
data = self.http.get("/api/system/indices/index_sets", {"skip": 0, "limit": 100})
|
|
return as_list(data, "index_sets")
|
|
|
|
def indices(self):
|
|
data = self.http.get("/api/system/indexer/indices")
|
|
if isinstance(data, dict):
|
|
# Strict per-key access: as_list()'s "first list in the dict"
|
|
# fallback would merge the same list three times here.
|
|
merged = []
|
|
for key in ("indices", "closed_indices", "reopened_indices"):
|
|
value = data.get(key)
|
|
if isinstance(value, list):
|
|
merged.extend(value)
|
|
return merged
|
|
return as_list(data)
|
|
|
|
# --------------------------------------------------------------- streams
|
|
def streams(self):
|
|
return as_list(self.http.get("/api/streams"), "streams")
|
|
|
|
def stream(self, stream_id):
|
|
return self.http.get("/api/streams/%s" % stream_id) or {}
|
|
|
|
# ---------------------------------------------------------------- inputs
|
|
def inputs(self):
|
|
return as_list(self.http.get("/api/system/inputs"), "inputs")
|
|
|
|
def extractors(self, input_id):
|
|
data = self.http.get("/api/system/inputs/%s/extractors" % input_id)
|
|
return as_list(data, "extractors")
|
|
|
|
# ------------------------------------------------------------- pipelines
|
|
def pipelines(self):
|
|
return as_list(self.http.get("/api/system/pipelines/pipeline"), "pipelines")
|
|
|
|
def pipeline_rules(self):
|
|
return as_list(self.http.get("/api/system/pipelines/rule"), "rules")
|
|
|
|
# ---------------------------------------------------------------- events
|
|
def _paged(self, path, key, limit=None, per_page=50, params=None):
|
|
items = []
|
|
page = 1
|
|
while True:
|
|
query = dict(params or {})
|
|
query.update({"page": page, "per_page": per_page})
|
|
data = self.http.get(path, query)
|
|
batch = as_list(data, key)
|
|
if not batch:
|
|
return items
|
|
items.extend(batch)
|
|
if limit is not None and len(items) >= limit:
|
|
return items[:limit]
|
|
# Trust the server's total when it tells us there is nothing left;
|
|
# this also stops runaway pagination on servers that ignore `page`.
|
|
total = dig(data, "total")
|
|
if isinstance(total, int) and len(items) >= total:
|
|
return items
|
|
if len(batch) < per_page:
|
|
return items
|
|
page += 1
|
|
if page > 500: # defensive stop
|
|
return items
|
|
|
|
def event_definitions(self, limit=None, per_page=50):
|
|
return self._paged(
|
|
"/api/events/definitions",
|
|
"event_definitions",
|
|
limit=limit,
|
|
per_page=per_page,
|
|
params={"sort": "title", "order": "asc"},
|
|
)
|
|
|
|
def notifications(self, limit=None, per_page=50):
|
|
return self._paged(
|
|
"/api/events/notifications", "notifications", limit=limit, per_page=per_page
|
|
)
|
|
|
|
def events(self, limit=None, per_page=50):
|
|
data = self.http.get("/api/events", {"page": 1, "per_page": per_page})
|
|
items = as_list(data, "events")
|
|
return items[:limit] if limit else items
|
|
|
|
# --------------------------------------------------------------- sidecar
|
|
def sidecars(self):
|
|
return as_list(self.http.get("/api/sidecars"), "sidecars")
|
|
|
|
# ---------------------------------------------------------------- search
|
|
@staticmethod
|
|
def _time_params(range_seconds=None, from_ts=None, to_ts=None):
|
|
if from_ts and to_ts:
|
|
return {"from": from_ts, "to": to_ts}
|
|
return {"range": int(range_seconds if range_seconds is not None else 300)}
|
|
|
|
def search_relative(self, query="*", range_seconds=300, limit=100, offset=0,
|
|
sort="timestamp:desc", fields=None, filter_str=None):
|
|
params = {"query": query or "*", "limit": int(limit), "offset": int(offset), "sort": sort}
|
|
params.update(self._time_params(range_seconds=range_seconds))
|
|
if fields:
|
|
params["fields"] = ",".join(fields)
|
|
if filter_str:
|
|
params["filter"] = filter_str
|
|
data = self.http.get("/api/search/universal/relative", params)
|
|
return self._messages(data)
|
|
|
|
def search_absolute(self, query="*", from_ts=None, to_ts=None, limit=100, offset=0,
|
|
sort="timestamp:desc", fields=None, filter_str=None):
|
|
params = {"query": query or "*", "limit": int(limit), "offset": int(offset), "sort": sort}
|
|
params.update(self._time_params(from_ts=from_ts, to_ts=to_ts))
|
|
if fields:
|
|
params["fields"] = ",".join(fields)
|
|
if filter_str:
|
|
params["filter"] = filter_str
|
|
data = self.http.get("/api/search/universal/absolute", params)
|
|
return self._messages(data)
|
|
|
|
@staticmethod
|
|
def _messages(data):
|
|
if not isinstance(data, dict):
|
|
return []
|
|
messages = data.get("messages")
|
|
if not isinstance(messages, list):
|
|
return []
|
|
result = []
|
|
for item in messages:
|
|
if isinstance(item, dict) and isinstance(item.get("message"), dict):
|
|
message = dict(item["message"])
|
|
if item.get("index") and "_index" not in message:
|
|
message["_index"] = item["index"]
|
|
result.append(message)
|
|
elif isinstance(item, dict):
|
|
result.append(item)
|
|
return result
|
|
|
|
# ------------------------------------------------------- views aggregation
|
|
BASE_PATH = "/api/views/search"
|
|
|
|
@staticmethod
|
|
def _timerange(range_seconds=None, from_ts=None, to_ts=None):
|
|
if from_ts and to_ts:
|
|
return {"type": "absolute", "from": from_ts, "to": to_ts}
|
|
return {"type": "relative", "range": int(range_seconds or 300)}
|
|
|
|
@staticmethod
|
|
def _stream_filter(filter_str):
|
|
"""Translate the legacy ``streams:<id>[,<id>]`` filter to a views filter."""
|
|
if not filter_str or not filter_str.startswith("streams:"):
|
|
return None
|
|
ids = [part.strip() for part in filter_str[len("streams:"):].split(",") if part.strip()]
|
|
if not ids:
|
|
return None
|
|
return {"type": "or", "filters": [{"type": "stream", "id": sid} for sid in ids]}
|
|
|
|
def _views_pivot(self, query, timerange, row_groups, column_groups, series,
|
|
sort, filter_str=None, timeout_ms=60000):
|
|
search_type = {
|
|
"type": "pivot",
|
|
"id": "st1",
|
|
"series": series,
|
|
"row_groups": row_groups,
|
|
"column_groups": column_groups,
|
|
"sort": sort,
|
|
"rollup": True,
|
|
}
|
|
query_object = {
|
|
"id": "q1",
|
|
"timerange": timerange,
|
|
"query": {"type": "elasticsearch", "query_string": query or "*"},
|
|
"search_types": [search_type],
|
|
}
|
|
stream_filter = self._stream_filter(filter_str)
|
|
if stream_filter:
|
|
query_object["filter"] = stream_filter
|
|
body = {"queries": [query_object], "parameters": []}
|
|
data = self.http.request(
|
|
"POST", self.BASE_PATH + "/sync",
|
|
params={"timeout": int(timeout_ms)}, body=body,
|
|
)
|
|
results = ((data or {}).get("results") or {}).get("q1") or {}
|
|
errors = results.get("errors") or []
|
|
if errors:
|
|
raise GraylogError(
|
|
"views search error: %s" % json.dumps(errors, ensure_ascii=False)[:400],
|
|
path=self.BASE_PATH + "/sync", method="POST",
|
|
)
|
|
search_types = results.get("search_types") or {}
|
|
if isinstance(search_types, list):
|
|
search_types = {item.get("id"): item for item in search_types
|
|
if isinstance(item, dict)}
|
|
return search_types.get("st1") or {}
|
|
|
|
def _terms_views(self, field, query, size, range_seconds, from_ts, to_ts, filter_str):
|
|
search_type = self._views_pivot(
|
|
query=query,
|
|
timerange=self._timerange(range_seconds, from_ts, to_ts),
|
|
row_groups=[{"type": "values", "field": field, "limit": int(size)}],
|
|
column_groups=[],
|
|
series=[{"type": "count", "id": "c1"}],
|
|
sort=[{"type": "pivot", "field": "c1", "direction": "Descending"}],
|
|
filter_str=filter_str,
|
|
)
|
|
terms = []
|
|
for row in search_type.get("rows") or []:
|
|
key = row.get("key") or []
|
|
if not key:
|
|
continue # rollup row (total)
|
|
values = row.get("values") or []
|
|
count = values[0].get("value") if values else 0
|
|
terms.append({"term": key[0], "count": count})
|
|
return {
|
|
"field": field,
|
|
"terms": terms,
|
|
"total": search_type.get("total", sum(row["count"] or 0 for row in terms)),
|
|
}
|
|
|
|
def _histogram_views(self, query, range_seconds, interval, filter_str, from_ts, to_ts):
|
|
search_type = self._views_pivot(
|
|
query=query,
|
|
timerange=self._timerange(range_seconds, from_ts, to_ts),
|
|
row_groups=[],
|
|
column_groups=[{"type": "time", "field": "timestamp", "interval": interval}],
|
|
series=[{"type": "count", "id": "c1"}],
|
|
sort=[],
|
|
filter_str=filter_str,
|
|
)
|
|
results = {}
|
|
for row in search_type.get("rows") or []:
|
|
for item in row.get("values") or []:
|
|
key = item.get("key") or []
|
|
# Column keys are [timestamp, series_id]; the pivot rollup row
|
|
# only carries [series_id] and must not become a time bucket.
|
|
if len(key) < 2:
|
|
continue
|
|
results[key[0]] = {"count": item.get("value", 0)}
|
|
return {"results": results, "interval": interval}
|
|
|
|
# ------------------------------------------------------------- aggregation
|
|
def search_views_messages(self, query="*", range_seconds=300, limit=5,
|
|
offset=0, filter_str=None, from_ts=None, to_ts=None,
|
|
timeout_ms=60000, sort_order="DESC"):
|
|
"""Message search through the views API (the legacy endpoint is flaky
|
|
on long windows with complex queries and drops the connection)."""
|
|
search_type = {
|
|
"type": "messages",
|
|
"id": "st1",
|
|
"limit": int(limit),
|
|
"offset": int(offset),
|
|
"sort": [{"field": "timestamp",
|
|
"order": "ASC" if str(sort_order).upper() == "ASC" else "DESC"}],
|
|
}
|
|
query_object = {
|
|
"id": "q1",
|
|
"timerange": self._timerange(range_seconds, from_ts, to_ts),
|
|
"query": {"type": "elasticsearch", "query_string": query or "*"},
|
|
"search_types": [search_type],
|
|
}
|
|
stream_filter = self._stream_filter(filter_str)
|
|
if stream_filter:
|
|
query_object["filter"] = stream_filter
|
|
body = {"queries": [query_object], "parameters": []}
|
|
data = self.http.request(
|
|
"POST", self.BASE_PATH + "/sync",
|
|
params={"timeout": int(timeout_ms)}, body=body,
|
|
)
|
|
results = ((data or {}).get("results") or {}).get("q1") or {}
|
|
errors = results.get("errors") or []
|
|
if errors:
|
|
raise GraylogError(
|
|
"views message search error: %s" % json.dumps(errors, ensure_ascii=False)[:300],
|
|
path=self.BASE_PATH + "/sync", method="POST",
|
|
)
|
|
search_types = results.get("search_types") or {}
|
|
if isinstance(search_types, list):
|
|
search_types = {item.get("id"): item for item in search_types
|
|
if isinstance(item, dict)}
|
|
search_type_result = search_types.get("st1") or {}
|
|
return self._messages({"messages": search_type_result.get("messages") or []})
|
|
|
|
def terms(self, field, query="*", size=50, range_seconds=300,
|
|
from_ts=None, to_ts=None, filter_str=None):
|
|
if self.agg_backend in (None, "views"):
|
|
try:
|
|
data = self._terms_views(field, query, size, range_seconds,
|
|
from_ts, to_ts, filter_str)
|
|
self.agg_backend = "views"
|
|
return data
|
|
except GraylogError as exc:
|
|
if exc.status != 404 or self.agg_backend == "views":
|
|
raise
|
|
self.agg_backend = "legacy"
|
|
return self._terms_legacy(field, query, size, range_seconds,
|
|
from_ts, to_ts, filter_str)
|
|
|
|
def _terms_legacy(self, field, query, size, range_seconds, from_ts, to_ts, filter_str):
|
|
params = {"query": query or "*", "field": field}
|
|
params.update(self._time_params(range_seconds, from_ts, to_ts))
|
|
if filter_str:
|
|
params["filter"] = filter_str
|
|
params[self.terms_size_param] = int(size)
|
|
try:
|
|
return self.http.get("/api/search/universal/terms", params)
|
|
except GraylogError as exc:
|
|
# Older/newer builds disagree on the parameter name; retry once.
|
|
if exc.status in (400, 422) and self.terms_size_param == "size":
|
|
params.pop("size", None)
|
|
params["limit"] = int(size)
|
|
data = self.http.get("/api/search/universal/terms", params)
|
|
self.terms_size_param = "limit"
|
|
return data
|
|
raise
|
|
|
|
def histogram(self, query="*", range_seconds=3600, interval="1m",
|
|
filter_str=None, from_ts=None, to_ts=None):
|
|
if self.agg_backend in (None, "views"):
|
|
try:
|
|
data = self._histogram_views(query, range_seconds, interval,
|
|
filter_str, from_ts, to_ts)
|
|
self.agg_backend = "views"
|
|
return data
|
|
except GraylogError as exc:
|
|
if exc.status != 404 or self.agg_backend == "views":
|
|
raise
|
|
self.agg_backend = "legacy"
|
|
params = {"query": query or "*", "interval": interval}
|
|
params.update(self._time_params(range_seconds, from_ts, to_ts))
|
|
if filter_str:
|
|
params["filter"] = filter_str
|
|
return self.http.get("/api/search/universal/histogram", params)
|
|
|
|
def stats(self, field, query="*", range_seconds=300, stats=None,
|
|
filter_str=None, from_ts=None, to_ts=None):
|
|
params = {"query": query or "*", "field": field}
|
|
params.update(self._time_params(range_seconds, from_ts, to_ts))
|
|
if stats:
|
|
params["stats"] = ",".join(stats)
|
|
if filter_str:
|
|
params["filter"] = filter_str
|
|
return self.http.get("/api/search/universal/stats", params)
|