* warden_filer: now able to filter outgoing messages in sender mode

* filenames now have .idea extension (Mentat requires that :) )
* bugfix: use list range instead of generator xrange on event index list generation (also in warden_client)

warden3/contrib/warden_filer/README

@@ -71,6 +71,9 @@ JSON object, containing configuration. See also warden_filer.cfg as example.
    sender - configuration section for sender mode
       dir - directory, whose "incoming" subdir will be checked for Idea
             events to send out
+      filter - filter fields (same as in Warden query, see Warden and Idea
+               doc, possible keys: cat, nocat, group, nogroup, tag, notag),
+               unmatched events get discarded and deleted
       node - o information about detector to be prepended into event Node
              array (see Idea doc)
    receiver - configuration section for receiver mode

warden3/contrib/warden_filer/warden_filer.cfg

@@ -15,10 +15,19 @@
         // Maildir like directory, whose "incoming" subdir will be checked
         // for Idea events to send out
         "dir": "warden_sender",
+        // Optional filter fields, unmatched events are discarded (and removed)
+        "filter": {
+            "cat": ["Test", "Recon.Scanning"],
+            "nocat": null,
+            "group": ["cz.example"],
+            "nogroup": null,
+            "tag": null,
+            "notag": ["Honeypot"]
+        },
         // Optional information about detector to be prepended into Idea Node array
         "node": {
             "Name": "cz.example.warden.test",
-            "Type": ["External"],
+            "Type": ["Relay"],
             "SW": ["warden_filer"],
             "AggrWin": "00:05:00",
             "Note": "Test warden_filer sender"

warden3/contrib/warden_filer/warden_filer.py

@@ -95,7 +95,7 @@ class SafeDir(object):
 
 
     def _get_new_name(self, device=0, inode=0):
-        return "%s.%d.%f.%d.%d" % (
+        return "%s.%d.%f.%d.%d.idea" % (
             self.hostname, self.pid, time.time(), device, inode)
 
 
@@ -177,10 +177,46 @@ def receiver(config, wclient, sdir, oneshot):
 
 
 
+def match_event(event, cat=None, nocat=None, tag=None, notag=None, group=None, nogroup=None):
+
+    cat_match = tag_match = group_match = True
+
+    if cat or nocat:
+        event_cats = event.get("Category")
+        event_full_cats = set(event_cats) | set(cat.split(".", 1)[0] for cat in event_cats)
+        cat_match = set(cat or nocat) & event_full_cats
+        cat_match = not cat_match if nocat else cat_match
+
+    try:
+        event_node = event.get("Node", [])[0]
+    except IndexError:
+        event_node = {}
+
+    if tag or notag:
+        event_tags = set(event_node.get("Type", []))
+        tag_match = set(tag or notag) & event_tags
+        tag_match = not tag_match if notag else tag_match
+
+    if group or nogroup:
+        event_name = event_node.get("Name")
+        namesplit = event_name.split(".")
+        allnames = set([".".join(namesplit[0:l]) for l in range(1, len(namesplit)+1)])
+        group_match = set(group or nogroup) & allnames
+        group_match = not group_match if nogroup else group_match
+
+    return cat_match and tag_match and group_match
+
+
+
 def sender(config, wclient, sdir, oneshot):
     send_events_limit = config.get("send_events_limit", 500)
     poll_time = config.get("poll_time", 5)
     node = config.get("node", None)
+    conf_filt = config.get("filter", {})
+    filt = {}
+    # Extract filter explicitly to be sure we have right param names for match_event
+    for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"):
+        filt[s] = conf_filt.get(s, None)
 
     while running_flag:
         nflist = sdir.get_incoming()
@@ -194,6 +230,7 @@ def sender(config, wclient, sdir, oneshot):
 
         events = []
         nf_sent = []
+        count_ok = count_err = count_unmatched = 0
         for nf in nflist:
             # prepare event array from files
             try:
@@ -204,6 +241,11 @@ def sender(config, wclient, sdir, oneshot):
                 with nf.open("rb") as fd:
                     data = fd.read()
                     event = json.loads(data)
+                    if not match_event(event, **filt):
+                        wclient.logger.debug("Unmatched event: %s" % data)
+                        count_unmatched += 1
+                        nf.remove()
+                        continue
                     if node:
                         nodelist = event.setdefault("Node", [])
                         nodelist.insert(0, node)
@@ -216,11 +258,10 @@ def sender(config, wclient, sdir, oneshot):
 
         res = wclient.sendEvents(events)
 
-        count_ok = count_err = count_retry = 0
         if isinstance(res, Error):
             for e in res.errors:
                 errno = e["error"]
-                evlist = e.get("events", xrange(len(nf_sent)))  # None means all
+                evlist = e.get("events", range(len(nf_sent)))  # None means all
                 for i in evlist:
                     if nf_sent[i]:
                         nf_sent[i].moveto(dest_dir)
@@ -233,7 +274,7 @@ def sender(config, wclient, sdir, oneshot):
                 name.remove()
                 count_ok += 1
         wclient.logger.info(
-            "warden_filer: saved %d, errors %d" % (count_ok, count_err))
+            "warden_filer: saved %d, errors %d, unmatched %d" % (count_ok, count_err, count_unmatched))
 
 
 

warden3/warden_client/warden_client.py

@@ -525,7 +525,7 @@ class Client(object):
                 res.errors.sort(key=itemgetter("error"))
                 for e in res.errors:
                     errno = e["error"]
-                    evlist = e.get("events", xrange(len(ev)))   # none means all
+                    evlist = e.get("events", range(len(ev)))   # none means all
                     if errno < 500 or not attempt:
                         # Fatal error or last try, translate indices
                         # to original and prepare for returning to caller