Filer: Filter keys in sent events. (Redmine issue: #6799)

warden_filer/test_warden_filer.py

+#!/usr/bin/python
+"""Warden3 Filer Test Suite"""
+
+import unittest2 as unittest
+import warden_filer
+
+idea_raw_1 = {
+        'ID': '4dd7cf5e-4a95-49f6-8f04-947de998012c',
+        'Format': 'IDEA0',
+        'DetectTime': '2016-06-21T13:08:27Z',
+        'WinStartTime': '2016-06-21T11:55:02Z',
+        'WinEndTime': '2016-06-21T12:00:02Z',
+        'Source': [
+            {
+                'IP4': ['188.14.166.39']
+            }
+        ],
+        'Target': [
+            {
+                'IP4': ['195.113.165.128/25']
+            }
+        ],
+        '_TO_DELETE': {
+            'key1' : 'value',
+            'key2' : 2
+        },
+        'Node': [
+            {
+                'Type': ['Relay'],
+                'Name': 'cz.cesnet.mentat.warden_filer'
+            }
+        ],
+        '_CESNET': {
+            'StorageTime': '2016-06-21T14:00:07Z'
+        }
+    }
+    
+idea_filtered_1 = {
+        'ID': '4dd7cf5e-4a95-49f6-8f04-947de998012c',
+        'Format': 'IDEA0',
+        'DetectTime': '2016-06-21T13:08:27Z',
+        'WinStartTime': '2016-06-21T11:55:02Z',
+        'WinEndTime': '2016-06-21T12:00:02Z',
+        'Source': [
+            {
+                'IP4': ['188.14.166.39']
+            }
+        ],
+        'Target': [
+            {
+                'IP4': ['195.113.165.128/25']
+            }
+        ],
+        'Node': [
+            {
+                'Type': ['Relay'],
+                'Name': 'cz.cesnet.mentat.warden_filer'
+            }
+        ]
+    }
+
+class Warden3FilerTest(unittest.TestCase):
+    """Warden3 Filer unit tests"""
+    
+    def test_filter_by_regexp(self):
+        regexp = '^_+'
+        filtered = warden_filer.filter_by_regexp(idea_raw_1, regexp)
+        self.assertEquals(filtered, idea_filtered_1)
+        
+        event = {
+            'ID' : '1',
+            'Node' : {
+            	'_INTERNAL' : 'data'
+            }
+        }
+        
+        filtered = warden_filer.filter_by_regexp(event, regexp)
+        
+        # only first level keys are filtered
+        self.assertEquals(filtered, event) 
+        
+if __name__ == "__main__":
+    unittest.main()

warden_filer/warden_filer.cfg.dist

@@ -27,6 +27,8 @@
         //    "tag": null,
         //    "notag": ["Honeypot"]
         //},
+        // Optional regexp filter for keys, matched keys are removed from events
+        //"key_filter" : "^_+",
         // Optional information about detector to be prepended into Idea Node array
         //"node": {
         //    "Name": "cz.example.warden.test_sender",

warden_filer/warden_filer.py

@@ -17,6 +17,7 @@ import signal
 import resource
 import atexit
 import argparse
+import re
 from os import path, mkdir
 from random import choice, randint;
 
@@ -249,7 +250,14 @@ def get_dir_list(sdir, owait_poll_time, owait_timeout, nfchunk, oneshot):
         nflist = sdir.get_incoming()
     return nflist
 
-
+def filter_by_regexp(event, regexp):
+    """
+    :param dict event: event where the keys should be filtered.
+    :param regexp: regular expression defining keys which should be left out.
+    :return: dictionary which does NOT contain keys matching regexp.
+    :rtype: dict
+    """
+    return {k:event.get(k) for k in event.keys() if not re.match(regexp, k)}
 
 def sender(config, wclient, sdir, oneshot):
     poll_time = config.get("poll_time", 5)
@@ -258,6 +266,8 @@ def sender(config, wclient, sdir, oneshot):
     node = config.get("node", None)
     done_dir = config.get("done_dir", None)
     conf_filt = config.get("filter", {})
+    # If no filter for keys is set then the filter which matches nothing is used
+    key_filter = config.get("key_filter", "a^")
     filt = {}
     # Extract filter explicitly to be sure we have right param names for match_event
     for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"):
@@ -301,7 +311,8 @@ def sender(config, wclient, sdir, oneshot):
                         if node:
                             nodelist = event.setdefault("Node", [])
                             nodelist.insert(0, node)
-                        events.append(event)
+                        # filter keys based on regular expression before appending to the list
+                        events.append(filter_by_regexp(event, key_filter))
                         nf_sent.append(nf)
                 except Exception as e:
                     Error(message="Error loading event", exc=sys.exc_info(), file=str(nf),