diff --git a/warden_filer/test_warden_filer.py b/warden_filer/test_warden_filer.py new file mode 100755 index 0000000000000000000000000000000000000000..58ce0865976988f4d0488c326dab1f579f62f4cd --- /dev/null +++ b/warden_filer/test_warden_filer.py @@ -0,0 +1,83 @@ +#!/usr/bin/python +"""Warden3 Filer Test Suite""" + +import unittest2 as unittest +import warden_filer + +idea_raw_1 = { + 'ID': '4dd7cf5e-4a95-49f6-8f04-947de998012c', + 'Format': 'IDEA0', + 'DetectTime': '2016-06-21T13:08:27Z', + 'WinStartTime': '2016-06-21T11:55:02Z', + 'WinEndTime': '2016-06-21T12:00:02Z', + 'Source': [ + { + 'IP4': ['188.14.166.39'] + } + ], + 'Target': [ + { + 'IP4': ['195.113.165.128/25'] + } + ], + '_TO_DELETE': { + 'key1' : 'value', + 'key2' : 2 + }, + 'Node': [ + { + 'Type': ['Relay'], + 'Name': 'cz.cesnet.mentat.warden_filer' + } + ], + '_CESNET': { + 'StorageTime': '2016-06-21T14:00:07Z' + } + } + +idea_filtered_1 = { + 'ID': '4dd7cf5e-4a95-49f6-8f04-947de998012c', + 'Format': 'IDEA0', + 'DetectTime': '2016-06-21T13:08:27Z', + 'WinStartTime': '2016-06-21T11:55:02Z', + 'WinEndTime': '2016-06-21T12:00:02Z', + 'Source': [ + { + 'IP4': ['188.14.166.39'] + } + ], + 'Target': [ + { + 'IP4': ['195.113.165.128/25'] + } + ], + 'Node': [ + { + 'Type': ['Relay'], + 'Name': 'cz.cesnet.mentat.warden_filer' + } + ] + } + +class Warden3FilerTest(unittest.TestCase): + """Warden3 Filer unit tests""" + + def test_filter_by_regexp(self): + regexp = '^_+' + filtered = warden_filer.filter_by_regexp(idea_raw_1, regexp) + self.assertEquals(filtered, idea_filtered_1) + + event = { + 'ID' : '1', + 'Node' : { + '_INTERNAL' : 'data' + } + } + + filtered = warden_filer.filter_by_regexp(event, regexp) + + # only first level keys are filtered + self.assertEquals(filtered, event) + +if __name__ == "__main__": + unittest.main() diff --git a/warden_filer/warden_filer.cfg.dist b/warden_filer/warden_filer.cfg.dist index ab6b1e1947e0a3b5e642ff0f74dca8cda1285367..091ba25497f3156c60f99fdab5562ece7128d7d2 100644 --- a/warden_filer/warden_filer.cfg.dist +++ b/warden_filer/warden_filer.cfg.dist @@ -27,6 +27,8 @@ // "tag": null, // "notag": ["Honeypot"] //}, + // Optional regexp filter for keys, matched keys are removed from events + //"key_filter" : "^_+", // Optional information about detector to be prepended into Idea Node array //"node": { // "Name": "cz.example.warden.test_sender", diff --git a/warden_filer/warden_filer.py b/warden_filer/warden_filer.py index 9ed1fbc573cc8f0aa2469f2d6e8c05bb604c42fe..05a6c65c7b4a13e3a15325a22ddc646a64db11d3 100755 --- a/warden_filer/warden_filer.py +++ b/warden_filer/warden_filer.py @@ -17,6 +17,7 @@ import signal import resource import atexit import argparse +import re from os import path, mkdir from random import choice, randint; @@ -249,7 +250,14 @@ def get_dir_list(sdir, owait_poll_time, owait_timeout, nfchunk, oneshot): nflist = sdir.get_incoming() return nflist - +def filter_by_regexp(event, regexp): + """ + :param dict event: event where the keys should be filtered. + :param regexp: regular expression defining keys which should be left out. + :return: dictionary which does NOT contain keys matching regexp. + :rtype: dict + """ + return {k:event.get(k) for k in event.keys() if not re.match(regexp, k)} def sender(config, wclient, sdir, oneshot): poll_time = config.get("poll_time", 5) @@ -258,6 +266,8 @@ def sender(config, wclient, sdir, oneshot): node = config.get("node", None) done_dir = config.get("done_dir", None) conf_filt = config.get("filter", {}) + # If no filter for keys is set then the filter which matches nothing is used + key_filter = config.get("key_filter", "a^") filt = {} # Extract filter explicitly to be sure we have right param names for match_event for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"): @@ -301,7 +311,8 @@ def sender(config, wclient, sdir, oneshot): if node: nodelist = event.setdefault("Node", []) nodelist.insert(0, node) - events.append(event) + # filter keys based on regular expression before appending to the list + events.append(filter_by_regexp(event, key_filter)) nf_sent.append(nf) except Exception as e: Error(message="Error loading event", exc=sys.exc_info(), file=str(nf),