From 749419533d5292edb99c1fddde0d819b1e7f8493 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pavel=20K=C3=A1cha?= <ph@cesnet.cz>
Date: Thu, 26 Feb 2015 15:08:24 +0100
Subject: [PATCH] * warden_filer: now able to filter outgoing messages in
sender mode * filenames now have .idea extension (Mentat requires that :) ) *
bugfix: use list range instead of generator xrange on event index list
generation (also in warden_client)
---
warden3/contrib/warden_filer/README | 3 ++
warden3/contrib/warden_filer/warden_filer.cfg | 11 ++++-
warden3/contrib/warden_filer/warden_filer.py | 49 +++++++++++++++++--
warden3/warden_client/warden_client.py | 2 +-
4 files changed, 59 insertions(+), 6 deletions(-)
diff --git a/warden3/contrib/warden_filer/README b/warden3/contrib/warden_filer/README
index 12f500a..a33de1f 100644
--- a/warden3/contrib/warden_filer/README
+++ b/warden3/contrib/warden_filer/README
@@ -71,6 +71,9 @@ JSON object, containing configuration. See also warden_filer.cfg as example.
sender - configuration section for sender mode
dir - directory, whose "incoming" subdir will be checked for Idea
events to send out
+ filter - filter fields (same as in Warden query, see Warden and Idea
+ doc, possible keys: cat, nocat, group, nogroup, tag, notag),
+ unmatched events get discarded and deleted
node - o information about detector to be prepended into event Node
array (see Idea doc)
receiver - configuration section for receiver mode
diff --git a/warden3/contrib/warden_filer/warden_filer.cfg b/warden3/contrib/warden_filer/warden_filer.cfg
index 1c7ee0c..402f518 100644
--- a/warden3/contrib/warden_filer/warden_filer.cfg
+++ b/warden3/contrib/warden_filer/warden_filer.cfg
@@ -15,10 +15,19 @@
// Maildir like directory, whose "incoming" subdir will be checked
// for Idea events to send out
"dir": "warden_sender",
+ // Optional filter fields, unmatched events are discarded (and removed)
+ "filter": {
+ "cat": ["Test", "Recon.Scanning"],
+ "nocat": null,
+ "group": ["cz.example"],
+ "nogroup": null,
+ "tag": null,
+ "notag": ["Honeypot"]
+ },
// Optional information about detector to be prepended into Idea Node array
"node": {
"Name": "cz.example.warden.test",
- "Type": ["External"],
+ "Type": ["Relay"],
"SW": ["warden_filer"],
"AggrWin": "00:05:00",
"Note": "Test warden_filer sender"
diff --git a/warden3/contrib/warden_filer/warden_filer.py b/warden3/contrib/warden_filer/warden_filer.py
index 84b1c56..b4c19c8 100644
--- a/warden3/contrib/warden_filer/warden_filer.py
+++ b/warden3/contrib/warden_filer/warden_filer.py
@@ -95,7 +95,7 @@ class SafeDir(object):
def _get_new_name(self, device=0, inode=0):
- return "%s.%d.%f.%d.%d" % (
+ return "%s.%d.%f.%d.%d.idea" % (
self.hostname, self.pid, time.time(), device, inode)
@@ -177,10 +177,46 @@ def receiver(config, wclient, sdir, oneshot):
+def match_event(event, cat=None, nocat=None, tag=None, notag=None, group=None, nogroup=None):
+
+ cat_match = tag_match = group_match = True
+
+ if cat or nocat:
+ event_cats = event.get("Category")
+ event_full_cats = set(event_cats) | set(cat.split(".", 1)[0] for cat in event_cats)
+ cat_match = set(cat or nocat) & event_full_cats
+ cat_match = not cat_match if nocat else cat_match
+
+ try:
+ event_node = event.get("Node", [])[0]
+ except IndexError:
+ event_node = {}
+
+ if tag or notag:
+ event_tags = set(event_node.get("Type", []))
+ tag_match = set(tag or notag) & event_tags
+ tag_match = not tag_match if notag else tag_match
+
+ if group or nogroup:
+ event_name = event_node.get("Name")
+ namesplit = event_name.split(".")
+ allnames = set([".".join(namesplit[0:l]) for l in range(1, len(namesplit)+1)])
+ group_match = set(group or nogroup) & allnames
+ group_match = not group_match if nogroup else group_match
+
+ return cat_match and tag_match and group_match
+
+
+
def sender(config, wclient, sdir, oneshot):
send_events_limit = config.get("send_events_limit", 500)
poll_time = config.get("poll_time", 5)
node = config.get("node", None)
+ conf_filt = config.get("filter", {})
+ filt = {}
+ # Extract filter explicitly to be sure we have right param names for match_event
+ for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"):
+ filt[s] = conf_filt.get(s, None)
while running_flag:
nflist = sdir.get_incoming()
@@ -194,6 +230,7 @@ def sender(config, wclient, sdir, oneshot):
events = []
nf_sent = []
+ count_ok = count_err = count_unmatched = 0
for nf in nflist:
# prepare event array from files
try:
@@ -204,6 +241,11 @@ def sender(config, wclient, sdir, oneshot):
with nf.open("rb") as fd:
data = fd.read()
event = json.loads(data)
+ if not match_event(event, **filt):
+ wclient.logger.debug("Unmatched event: %s" % data)
+ count_unmatched += 1
+ nf.remove()
+ continue
if node:
nodelist = event.setdefault("Node", [])
nodelist.insert(0, node)
@@ -216,11 +258,10 @@ def sender(config, wclient, sdir, oneshot):
res = wclient.sendEvents(events)
- count_ok = count_err = count_retry = 0
if isinstance(res, Error):
for e in res.errors:
errno = e["error"]
- evlist = e.get("events", xrange(len(nf_sent))) # None means all
+ evlist = e.get("events", range(len(nf_sent))) # None means all
for i in evlist:
if nf_sent[i]:
nf_sent[i].moveto(dest_dir)
@@ -233,7 +274,7 @@ def sender(config, wclient, sdir, oneshot):
name.remove()
count_ok += 1
wclient.logger.info(
- "warden_filer: saved %d, errors %d" % (count_ok, count_err))
+ "warden_filer: saved %d, errors %d, unmatched %d" % (count_ok, count_err, count_unmatched))
diff --git a/warden3/warden_client/warden_client.py b/warden3/warden_client/warden_client.py
index c177682..f1a9b21 100644
--- a/warden3/warden_client/warden_client.py
+++ b/warden3/warden_client/warden_client.py
@@ -525,7 +525,7 @@ class Client(object):
res.errors.sort(key=itemgetter("error"))
for e in res.errors:
errno = e["error"]
- evlist = e.get("events", xrange(len(ev))) # none means all
+ evlist = e.get("events", range(len(ev))) # none means all
if errno < 500 or not attempt:
# Fatal error or last try, translate indices
# to original and prepare for returning to caller
--
GitLab