Merge from main Warden repo

cznichaas2warden/README.md

+# haas2warden
+
+Warden connector for data of [CZ.NIC HaaS project](https://haas.nic.cz/).
+
+It downloads daily [HaaS data dumps](https://haas.nic.cz/stats/export/),
+converts them to IDEA messages and sends them to CESNET's Warden server.
+
+It should be run from `cron` every night when data from previous day are
+available (at 3:30).
+
+The script just writes IDEA messages as files into a "filer" directory.
+A _warden_filer_ daemon must be configured to pick up the messages
+and send them to Warden server.
+There is a systemd file which can be used to run the warden_filer.

cznichaas2warden/haas2warden.cron

+# Run every day at 03:30
+30 03 * * * haas2warden python3 /data/haas2warden/haas2warden.py -p /data/haas2warden/warden_filer/ -n org.example.ext.cznic_haas -t >> /data/haas2warden/haas2warden.log 2>&1 

cznichaas2warden/haas2warden.py

+#!/usr/bin/env python3
+from gzip import decompress
+from json import loads
+from datetime import datetime, timedelta
+import argparse
+import logging
+import uuid
+import json
+import os
+import requests
+
+
+data_date = datetime.date(datetime.utcnow()) - timedelta(days=1)
+
+LOGFORMAT = "%(asctime)-15s,%(name)s [%(levelname)s] %(message)s"
+LOGDATEFORMAT = "%Y-%m-%dT%H:%M:%S"
+logging.basicConfig(level=logging.INFO, format=LOGFORMAT, datefmt=LOGDATEFORMAT)
+
+logger = logging.getLogger('haas2warden')
+
+def createIDEAFile(idea_id, idea_msg):
+    """
+    Creates file for IDEA message in .../tmp folder, then move it to .../incoming folder
+    """
+    tmp_dir_path = os.path.join(args.path, "tmp")
+    idea_file_path = os.path.join(tmp_dir_path, idea_id+".idea")
+    os.makedirs(tmp_dir_path, exist_ok=True)
+    idea_file = open(idea_file_path, "w")
+    idea_file.write(idea_msg)
+    idea_file.close()
+
+    incoming_dir_path = os.path.join(args.path, "incoming")
+    incoming_file_path = os.path.join(incoming_dir_path,idea_id+".idea")
+    os.makedirs(incoming_dir_path, exist_ok=True) 
+    os.rename(idea_file_path,incoming_file_path)
+
+
+def createIDEA(time, time_closed, ip, login_successful, commands):
+    """
+    Creates IDEA message 
+    """ 
+    idea_id = str(uuid.uuid4())
+
+    if login_successful:
+        category = "[\"Intrusion.UserCompromise\"]" 
+        description = "SSH login on honeypot (HaaS)"
+        if args.test:
+            category = "[\"Intrusion.UserCompromise\", \"Test\"]"
+        attach = f''',
+   "Attach": [
+        {{
+            "Note": "commands",
+            "Type": ["ShellCode"],
+            "ContentType": "application/json",
+            "Content": {json.dumps(commands)}
+        }}
+    ]''' #              ^-- "commands" is already serialiezed into a json string, we want to include it into a bigger JSON so we must encode it again (to escape quotes and any other special charaters)
+
+    else:
+        category = "[\"Attempt.Login\"]" 
+        description = "Unsuccessful SSH login attempt on honeypot (HaaS)"
+        if args.test:
+            category = "[\"Attempt.Login\", \"Test\"]"
+        attach = ""
+
+    if time_closed: # sometimes time_closed is empty, in such case we must omit CeaseTime completely from IDEA msg
+        cease_time = f'"CeaseTime": "{time_closed}",'
+    else:
+        cease_time = ""
+
+    idea_msg = f"""\
+{{
+    "Format": "IDEA0",
+    "ID": "{idea_id}",
+    "Category": {category},
+    "Description": "{description}",
+    "Note": "Extracted from data of CZ.NIC HaaS project",
+    "DetectTime": "{time}",
+    "EventTime": "{time}",
+    {cease_time}
+    "CreateTime": "{datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')}",
+    "Source": [
+        {{
+            "IP4": ["{ip}"],
+            "Proto": ["tcp", "ssh"]
+        }}
+    ],
+    "Node": [
+        {{
+            "Name": "{args.name}",
+            "SW": ["CZ.NIC HaaS"],
+            "Type": ["Connection", "Auth", "Honeypot"],
+            "Note": "A script converting daily HaaS data dumps from https://haas.nic.cz/stats/export/"
+        }}
+    ]{attach}
+}}
+"""
+    createIDEAFile(idea_id, idea_msg)
+
+    
+def processJSON():
+    """
+    Downloads data from https://haas.nic.cz/stats/export/ and process json files.
+    """
+    date = datetime.strptime(args.date, '%Y-%m-%d').date()
+    # get url
+    url = "https://haas.nic.cz/stats/export/{}/{}/{}.json.gz".format(str(date).split('-')[0],str(date).split('-')[1], str(date))
+    # get data 
+    logger.info("Downloading {}".format(url))
+    response = requests.get(url)
+    if response.status_code == 200:
+        # unzip and read json file
+        json_objects = loads(decompress(response.content))
+        logger.info("Found {} records, converting to IDEA messages".format(len(json_objects)))
+        # go through all json objects
+        for json_object in json_objects:
+            createIDEA(json_object["time"], json_object["time_closed"], json_object["ip"], json_object["login_successful"], json.dumps(json_object["commands"]))
+
+if __name__ == "__main__":
+    
+    # parse arguments
+    parser = argparse.ArgumentParser(
+        prog="haas_receiver.py",
+        description="A script converting daily HaaS data dumps from https://haas.nic.cz/stats/export/"
+    )
+
+    parser.add_argument('-d', '--date', metavar='DATE', default = str(data_date),
+                        help='To download data from date YYYY-MM-DD, use date + 1 day (default: utcnow - 1 day)')
+    parser.add_argument('-p', '--path', metavar='DIRPATH', default = "/data/haas2warden/warden_filer/",
+                        help='Target folder for Idea messages (default: "/data/haas2warden/warden_filer/")')
+    parser.add_argument('-n', '--name', metavar='NODENAME', default = "undefined",
+                        help='Name of the node (default: undefined)')
+    parser.add_argument('-t', '--test', action="store_true",
+                        help='Test category')                                        
+
+    args = parser.parse_args()
+    
+    processJSON()
+    logger.info("Done")

cznichaas2warden/warden-filer.service

+# Template of Systemd unit for Warden filer daemon
+#
+# TODO: set paths, username and mode (receiver/sender) in the last two lines
+# and uncomment them. Then copy the file to:
+#   /etc/systemd/system/warden-filer.service
+# and run:
+#   systemctl daemon-reload
+
+[Unit]
+Description=Warden filer for haas2warden
+After=syslog.target network.target
+
+[Service]
+Type=forking
+User=haas2warden
+PIDFile=/data/haas2warden/warden_filer.pid
+ExecStart=/opt/warden_filer/warden_filer.py --daemon -c "/data/haas2warden/warden_filer.cfg" --pid_file "/data/haas2warden/warden_filer.pid" sender
+

cznichaas2warden/warden_filer.cfg

+{
+    // Warden config can be also referenced as:
+    // "warden": "/path/to/warden_client.cfg"
+    "warden": {
+        "url": "https://warden-hub.cesnet.cz/warden3",
+        "cafile": "/etc/pki/tls/certs/ca-bundle.crt",
+        "keyfile": "/data/haas2warden/key.pem",
+        "certfile": "/data/haas2warden/cert.pem",
+        "timeout": 10,
+        "errlog": {"level": "warning"},
+        "filelog": {"level": "info", "file": "/data/haas2warden/warden_filer.log"},
+        "idstore": "/data/haas2warden/warden_filer.id",
+        "name": "org.example.cznic_haas"
+    },
+    "sender": {
+        // Maildir like directory, whose "incoming" subdir will be checked
+        // for Idea events to send out
+        "dir": "/data/haas2warden/warden_filer",
+        "poll_time": 60
+    }
+}

fail2ban/README

+Support scripts for fail2ban
+============================
+
+Introduction
+------------
+
+Fail2ban is a logfile watcher, which is able to run various actions,
+based on too many patterns occured in the log file.
+Those helper shell scripts can be used as action to report events to
+Warden_.
+
+Dependencies
+------------
+
+ 1. Python packages
+
+    warden_filer 3.0+
+
+Usage
+-----
+
+ * f2ban_spam.sh is meant to be used in cooperation with the default
+   "postfix" rule.
+ * f2ban_ssh.sh is meant to be used in cooperation with the default
+   "ssh" rule. 
+
+In the corresponding action following invocation can be used:
+
+   actionban = /usr/local/bin/f2ban_XXX.sh <ip> <failures> <time>
+
+Please, edit corresponding paths and Warden names in the corresponding
+script preamble and check/edit contents of the IDEA template (e.g. Target IP 
+address in f2ban_ssh.sh).
+
+Scripts write generated Idea_ events into warden_filer compatible
+directory, so you will need to run properly configured (and registered
+into Warden server) warden_filer instance, which will take care for
+picking up the events and submitting them.
+
+.. _Warden: https://warden.cesnet.cz/
+.. _Idea: https://idea.cesnet.cz/
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

fail2ban/f2ban_spam.sh

+#!/bin/bash
+
+umask 0111
+
+filer_dir="/var/mentat/spool/_wardenout"
+src_ip=$1
+failures=$2
+detect_time=$(date --date="@$3" --rfc-3339=seconds)
+create_time=$(date --rfc-3339=seconds)
+node_name="org.example.fail2ban.blacklist"
+
+uuid() {
+        for ((n=0; n<16; n++)); do  
+                read -n1 c < /dev/urandom
+                LC_CTYPE=C d=$(printf '%d' "'$c")
+                s=''
+                case $n in
+                        6) ((d = d & 79 | 64));;   
+                        8) ((d = d & 191 | 128));;  
+                        3|5|9|7) s='-';; 
+                esac
+                printf '%02x%s' $d "$s"
+        done
+}
+
+event_id=$(uuid)
+
+cat >"$filer_dir/tmp/$event_id" <<EOF
+{
+   "Format" : "IDEA0",
+   "ID" : "$event_id",
+   "DetectTime" : "$detect_time",
+   "CreateTime" : "$create_time",
+   "Category" : ["Abusive.Spam"],
+   "Description" : "Blacklisted host",
+   "Note" : "Block duration: 3600. IP was blacklisted, is listed on more than 5 public blacklists",
+   "Source" : [{
+      "Type": ["Spam"],
+      "IP4" : ["$src_ip"],
+      "Proto": ["tcp", "smtp"]
+   }],
+   "Node" : [{
+         "Name" : "$node_name",
+         "SW" : ["Fail2Ban"],
+         "Type" : ["Log", "Statistical"]
+   }],
+   "_CESNET" : {
+      "Impact" : "IP was blacklisted, is listed on more than 5 public blacklists",
+      "EventTemplate" : "f2b-001"
+   }
+}
+EOF
+
+mv "$filer_dir/tmp/$event_id" "$filer_dir/incoming"

fail2ban/f2ban_ssh.sh

+#!/bin/bash
+
+umask 0111
+
+filer_dir="/var/spool/warden_sender"
+src_ip=$1
+failures=$2
+detect_time=$(date --date="@$3" --rfc-3339=seconds)
+create_time=$(date --rfc-3339=seconds)
+node_name="org.example.fail2ban.ssh"
+
+uuid() {
+        for ((n=0; n<16; n++)); do  
+                read -n1 c < /dev/urandom
+                LC_CTYPE=C d=$(printf '%d' "'$c")
+                s=''
+                case $n in
+                        6) ((d = d & 79 | 64));;   
+                        8) ((d = d & 191 | 128));;  
+                        3|5|9|7) s='-';; 
+                esac
+                printf '%02x%s' $d "$s"
+        done
+}
+
+event_id=$(uuid)
+
+cat >"$filer_dir/tmp/$event_id" <<EOF
+{
+   "Format": "IDEA0",
+   "ID": "$event_id",
+   "DetectTime": "$detect_time",
+   "CreateTime": "$create_time",
+   "Category": ["Attempt.Login"],
+   "Description": "SSH dictionary/bruteforce attack",
+   "ConnCount": $failures,
+   "Note": "IP attempted $failures logins to SSH service",
+   "Source": [{
+      "IP4": ["$src_ip"],
+      "Proto": ["tcp", "ssh"]
+   }],
+   "Target": [{
+       "Type": ["Anonymised"],
+       "IP4": ["192.0.2.0/24"],
+       "Anonymised": true,
+       "Proto": ["tcp", "ssh"],
+       "Port": [22]
+   }],
+   "Node": [{
+         "Name": "$node_name",
+         "SW": ["Fail2Ban"],
+         "Type": ["Log", "Statistical"]
+   }]
+}
+EOF
+
+mv "$filer_dir/tmp/$event_id" "$filer_dir/incoming"

hp-labrea/README

+Warden LaBrea connector 0.1 for Warden 3.X
+==========================================
+
+Introduction
+------------
+
+labrea-idea.py is a daemon, meant for continuous watching of LaBrea log files
+and generation of Idea_ format of corresponding security events. It is
+usually run in correspondence with warden_filer daemon, which picks the
+resulting events up and feeds them to the Warden_ server. Connector supports
+sliding window aggregation, so sets of connections with the same source are
+reported as one event (within aggregation window).
+
+
+Dependencies
+------------
+
+ 1. Platform
+
+    Python 2.7+
+
+ 2. Python packages
+
+    warden_filer 3.0+ (recommended)
+
+
+Usage
+-----
+
+        ./labrea-idea.py [options] logfile ...
+
+        Options:
+          -h, --help            show this help message and exit
+          -w WINDOW, --window=WINDOW
+                                max detection window (default: 900)
+          -t TIMEOUT, --timeout=TIMEOUT
+                                detection timeout (default: 300)
+          -n NAME, --name=NAME  Warden client name
+          --test                Add Test category
+          -o, --oneshot         process files and quit (do not daemonize)
+          --poll=POLL           log file polling interval
+          -d DIR, --dir=DIR     Target directory (mandatory)
+          -p PID, --pid=PID     create PID file with this name (default: /var/run
+                                /labrea-idea.pid)
+          -u UID, --uid=UID     user id to run under
+          -g GID, --gid=GID     group id to run under
+          -v, --verbose         turn on debug logging
+          --log=LOG             syslog facility or log file name (default: local7)
+          --realtime            use system time along with log timestamps (default)
+          --norealtime          don't system time, use solely log timestamps
+
+
+Configuration
+-------------
+
+However, the daemon is usually run by init script (example one is a part of
+the distribution, along with sample logrotate definition). Options then can
+be configured by /etc/sysconfig/labrea-idea or /etc/defaults/labrea-idea,
+depending on your distribution custom, where at least PARAMS variable has
+to be specified (for others, see the init script).
+    
+.. _Warden: https://warden.cesnet.cz/
+.. _Idea: https://idea.cesnet.cz/
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

hp-labrea/labrea-idea

+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:          labrea-idea
+# Required-Start:    $local_fs $syslog
+# Required-Stop:     $local_fs $syslog
+# Should-Start:      $network
+# Should-Stop:       $network
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Labrea-Idea aggregator/converter
+### END INIT INFO
+
+DAEMON_NAME=labrea-idea
+DAEMON_PATH=/usr/local/bin/"$DAEMON_NAME".py
+PID=/var/run/"$DAEMON_NAME".pid
+
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$DAEMON_NAME" ] && . /etc/$n/"$DAEMON_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
+
+ACTION="$1"
+
+case "$ACTION" in
+	start)
+		if [ -z "$PARAMS" ]; then
+			log_daemon_msg "Unconfigured $DAEMON_NAME, not starting."
+			exit 2
+		fi
+		mkdir -p "${PID%/*}"
+		log_daemon_msg "Starting $DAEMON_NAME"
+		start_daemon -p "$PID" "$DAEMON_PATH" --pid "$PID" $PARAMS
+		log_end_msg $?
+		;;
+	stop)
+		log_daemon_msg "Stopping $DAEMON_NAME"
+		killproc -p "$PID" "$DAEMON_PATH"
+		log_end_msg $?
+		;;
+	restart|force-reload)
+		$0 stop && sleep 2 && exec $0 start
+		;;
+	status)
+		status_of_proc -p "$PID" "$DAEMON_PATH"
+		;;
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 2
+		;;
+esac

hp-labrea/labrea-idea.logrotate

+/var/log/labrea-idea.log
+{
+	rotate 52
+	weekly
+	missingok
+	notifempty
+	compress
+	delaycompress
+	dateext
+	create 640 mentat mentat
+}

hp-labrea/labrea-idea.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import re
+import time
+import optparse
+import signal
+import contextlib
+import uuid
+import json
+import socket
+import resource
+import atexit
+import logging
+import logging.handlers
+import os.path as pth
+from itertools import izip
+from collections import namedtuple
+try:
+    from collections import OrderedDict
+except ImportError:
+    from ordereddict import OrderedDict
+
+
+class WindowContextMgr(object):
+
+    def __init__(self, window=60*10, timeout=60*5, ideagen=None):
+        self.contexts = {}
+        self.update_timestamp = 0
+        self.window = window
+        self.timeout = timeout
+        self.ideagen = ideagen
+        self.first_update_queue = OrderedDict()
+        self.last_update_queue = OrderedDict()
+        # Hammer to mitigate too big events
+        self.max_count = 2000
+        self.max_src_ports = 1024
+
+    def expire_queue(self, queue, window):
+        aggr_events = []
+        ctx_to_del = []
+        for ctx, timestamp in queue.iteritems():
+            if timestamp >= self.update_timestamp - window:
+                break
+            closed = self.ctx_close(self.contexts[ctx])
+            if closed is not None:
+                aggr_events.append(closed)
+            ctx_to_del.append(ctx)
+        for ctx in ctx_to_del:
+            del self.contexts[ctx]
+            del self.first_update_queue[ctx]
+            del self.last_update_queue[ctx]
+        return aggr_events
+
+    def process(self, event=None, timestamp=None):
+        if timestamp > self.update_timestamp:
+            self.update_timestamp = timestamp
+
+        aggr_events = []
+
+        aggr_events.extend(self.expire_queue(self.first_update_queue, self.window))
+        aggr_events.extend(self.expire_queue(self.last_update_queue, self.timeout))
+
+        if event is not None:
+            ctx = self.match(event)
+            if ctx is not None:
+                if ctx not in self.contexts:
+                    self.contexts[ctx] = self.ctx_create(event)
+                    self.first_update_queue[ctx] = self.update_timestamp
+                    self.last_update_queue[ctx] = self.update_timestamp
+                else:
+                    if not self.ctx_append(self.contexts[ctx], event):
+                        closed = self.ctx_close(self.contexts[ctx])
+                        if closed is not None:
+                            aggr_events.append(closed)
+                        del self.contexts[ctx]
+                        del self.first_update_queue[ctx]
+                        del self.last_update_queue[ctx]
+                    else:
+                        del self.last_update_queue[ctx]
+                        self.last_update_queue[ctx] = self.update_timestamp
+
+        return aggr_events
+
+    def close(self, timestamp):
+        if timestamp is not None and timestamp > self.update_timestamp:
+            self.update_timestamp = timestamp
+        aggr_events = []
+        for context in self.contexts.values():
+            closed = self.ctx_close(context)
+            if closed is not None:
+                aggr_events.append(closed)
+        self.contexts = {}
+        self.first_update_queue = OrderedDict()
+        self.last_update_queue = OrderedDict()
+        return aggr_events
+
+
+class PingContextMgr(WindowContextMgr):
+
+    def match(self, event):
+        return event.src_ip if 'Ping' in event.message else None
+
+    def ctx_create(self, event):
+        ctx = dict(
+            src_ip=event.src_ip,
+            tgt_ips=set([event.tgt_ip]),
+            count=1,
+            first_update=self.update_timestamp,
+            last_update=self.update_timestamp
+        )
+        return ctx
+
+    def ctx_append(self, ctx, event):
+        ctx["tgt_ips"].add(event.tgt_ip)
+        ctx["count"] += 1
+        ctx["last_update"] = self.update_timestamp
+        return ctx["count"] < self.max_count
+
+    def ctx_close(self, ctx):
+        return self.ideagen.gen_idea(
+            src=ctx["src_ip"],
+            src_ports=None,
+            targets=[(ip, []) for ip in ctx["tgt_ips"]],
+            detect_time=self.update_timestamp,
+            event_time=ctx["first_update"],
+            cease_time=ctx["last_update"],
+            count=ctx["count"],
+            template="ping"
+        )
+
+
+class ConnectContextMgr(WindowContextMgr):
+
+    def match(self, event):
+        return event.src_ip if 'Connect' in event.message else None
+
+    def ctx_create(self, event):
+        ctx = dict(
+            src_ip=event.src_ip,
+            src_ports=set([event.src_port]),
+            tgt_ips_ports={event.tgt_ip: set([event.tgt_port])},
+            count=1,
+            first_update=self.update_timestamp,
+            last_update=self.update_timestamp
+        )
+        return ctx
+
+    def ctx_append(self, ctx, event):
+        tgt_ports = ctx["tgt_ips_ports"].setdefault(event.tgt_ip, set())
+        tgt_ports.add(event.tgt_port)
+        ctx["src_ports"].add(event.src_port)
+        ctx["count"] += 1
+        ctx["last_update"] = self.update_timestamp
+        return ctx["count"] < self.max_count
+
+    def ctx_close(self, ctx):
+        src_ports = ctx["src_ports"] if len(ctx["src_ports"]) <= self.max_src_ports else None
+        return self.ideagen.gen_idea(
+            src=ctx["src_ip"],
+            src_ports=src_ports,
+            targets=ctx["tgt_ips_ports"].items(),
+            detect_time=self.update_timestamp,
+            event_time=ctx["first_update"],
+            cease_time=ctx["last_update"],
+            count=ctx["count"],
+            template="connect"
+        )
+
+
+class InboundContextMgr(ConnectContextMgr):
+
+    def match(self, event):
+        return event.src_ip if 'Inbound' in event.message else None
+
+    def ctx_close(self, ctx):
+        return self.ideagen.gen_idea(
+            src=ctx["src_ip"],
+            src_ports=ctx["src_ports"],
+            targets=ctx["tgt_ips_ports"].items(),
+            detect_time=self.update_timestamp,
+            event_time=ctx["first_update"],
+            cease_time=ctx["last_update"],
+            count=ctx["count"],
+            template="synack"
+        )
+
+
+class FileWatcher(object):
+
+    def __init__(self, filename, tail=True):
+        self.filename = filename
+        self.open()
+        self.line_buffer = ""
+        if tail and self.f:
+            self.f.seek(0, os.SEEK_END)
+
+    def open(self):
+        try:
+            self.f = open(self.filename, "r")
+            st = os.fstat(self.f.fileno())
+            self.inode, self.size = st.st_ino, st.st_size
+        except IOError:
+            self.f = None
+            self.inode = -1
+            self.size = -1
+
+    def _check_reopen(self):
+        try:
+            st = os.stat(self.filename)
+            cur_inode, cur_size = st.st_ino, st.st_size
+        except OSError as e:
+            cur_inode = -1
+            cur_size = -1
+        if cur_inode != self.inode or cur_size < self.size:
+            self.close()
+            self.open()
+
+    def readline(self):
+        if not self.f:
+            self.open()
+            if not self.f:
+                return self.line_buffer
+        res = self.f.readline()
+        if not res:
+            self._check_reopen()
+            if not self.f:
+                return self.line_buffer
+            res = self.f.readline()
+        if not res.endswith("\n"):
+            self.line_buffer += res
+        else:
+            res = self.line_buffer + res
+            self.line_buffer = ""
+            return res
+
+    def close(self):
+        try:
+            if self.f:
+                self.f.close()
+        except IOError:
+            pass
+        self.inode = -1
+        self.size = -1
+
+    def __repr__(self):
+        return '%s("%s")' % (type(self).__name__, self.filename)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+        return False
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        return self.readline()
+
+
+class IdeaGen(object):
+
+    def __init__(self, name, test=False):
+        self.name = name
+        self.test = test
+        self.template = {
+            "connect": {
+                "category": ["Recon.Scanning"],
+                "description": "TCP connections/scan",
+                "template": "labrea-001",
+                "note": "Connections from remote host to never assigned IP",
+                "proto": ["tcp"]
+            },
+            "ping": {
+                "category": ["Recon.Scanning"],
+                "description": "Ping scan",
+                "template": "labrea-002",
+                "note": "Ping requests from remote host to never assigned IP",
+                "proto": ["icmp"]
+            },
+            "synack": {
+                "category": ["Availability.DoS"],
+                "description": "Unsolicited TCP SYN/ACK connections/scan",
+                "template": "labrea-003",
+                "note": "Unsolicited SYN/ACK packet received from remote host to never assigned IP",
+                "source_type": ["Backscatter"],
+                "source_to_target": True,
+                "proto": ["tcp"]
+            }
+        }
+
+    def format_timestamp(self, epoch):
+        return "%04d-%02d-%02dT%02d:%02d:%02dZ" % time.gmtime(epoch)[:6]
+
+    def gen_idea(self, src, src_ports, targets, detect_time, event_time, cease_time, count, template):
+        tmpl = self.template[template]
+        isource = {
+            "IP6" if ":" in src else "IP4": [src],
+            "Proto": tmpl["proto"]
+        }
+        if "source_type" in tmpl:
+            isource["Type"] = tmpl["source_type"]
+        if src_ports:
+            isource["Port"] = [int(port) for port in src_ports]
+        # Fold multiple IPs with the same portset
+        folded_tgt = {}
+        for tgt, ports in targets:
+            folded_tgt.setdefault(frozenset(ports), []).append(tgt)
+        itargets = []
+        for ports, tgt in folded_tgt.items():
+            itarget = {"Proto": tmpl["proto"]}
+            tgts4 = [ip for ip in tgt if ":" not in ip]
+            tgts6 = [ip for ip in tgt if ":" in ip]
+            if tgts4:
+                itarget["IP4"] = tgts4
+            if tgts6:
+                itarget["IP6"] = tgts6
+            if ports:
+                itarget["Port"] = [int(port) for port in ports]
+            itargets.append(itarget)
+        inode = {
+            "SW": ["LaBrea"],
+            "Type": ["Connection", "Tarpit"],
+            "Name": self.name
+        }
+        idea = {
+            "Format": "IDEA0",
+            "ID": str(uuid.uuid4()),
+            "Category": tmpl["category"] + (["Test"] if self.test else []),
+            "Description": tmpl["description"],
+            "DetectTime": self.format_timestamp(detect_time),
+            "EventTime": self.format_timestamp(event_time),
+            "CeaseTime": self.format_timestamp(cease_time),
+            "ConnCount": count,
+            "Note": tmpl["note"],
+            "_CESNET": {
+                "EventTemplate": tmpl["template"],
+            },
+            "Target": itargets,
+            "Node": [inode]
+        }
+        if tmpl.get("source_to_target", False):
+            idea["Target"].append(isource)
+        else:
+            idea["Source"] = [isource]
+        return idea
+
+
+class Filer(object):
+
+    def __init__(self, directory):
+        self.basedir = self._ensure_path(directory)
+        self.tmp = self._ensure_path(pth.join(self.basedir, "tmp"))
+        self.incoming = self._ensure_path(pth.join(self.basedir, "incoming"))
+        self.hostname = socket.gethostname()
+        self.pid = os.getpid()
+
+    def _ensure_path(self, p):
+        try:
+            os.mkdir(p)
+        except OSError:
+            if not pth.isdir(p):
+                raise
+        return p
+
+    def _get_new_name(self, fd=None):
+        (inode, device) = os.fstat(fd)[1:3] if fd else (0, 0)
+        return "%s.%d.%f.%d.%d" % (
+            self.hostname, self.pid, time.time(), device, inode)
+
+    def create_unique_file(self):
+        # First find and open name unique within tmp
+        tmpname = None
+        while not tmpname:
+            tmpname = self._get_new_name()
+            try:
+                fd = os.open(pth.join(self.tmp, tmpname), os.O_CREAT | os.O_RDWR | os.O_EXCL)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise   # other errors than duplicates should get noticed
+                tmpname = None
+        # Now we know the device/inode, rename to make unique within system
+        newname = self._get_new_name(fd)
+        os.rename(pth.join(self.tmp, tmpname), pth.join(self.tmp, newname))
+        nf = os.fdopen(fd, "w")
+        return nf, newname
+
+    def publish_file(self, short_name):
+        os.rename(pth.join(self.tmp, short_name), pth.join(self.incoming, short_name))
+
+
+def daemonize(
+        work_dir=None, chroot_dir=None,
+        umask=None, uid=None, gid=None,
+        pidfile=None, files_preserve=[], signals={}):
+    # Dirs, limits, users
+    if chroot_dir is not None:
+        os.chdir(chroot_dir)
+        os.chroot(chroot_dir)
+    if umask is not None:
+        os.umask(umask)
+    if work_dir is not None:
+        os.chdir(work_dir)
+    if gid is not None:
+        os.setgid(gid)
+    if uid is not None:
+        os.setuid(uid)
+    # Doublefork, split session
+    if os.fork() > 0:
+        os._exit(0)
+    try:
+        os.setsid()
+    except OSError:
+        pass
+    if os.fork() > 0:
+        os._exit(0)
+    # Setup signal handlers
+    for (signum, handler) in signals.items():
+        signal.signal(signum, handler)
+    # Close descriptors
+    descr_preserve = set(f.fileno() for f in files_preserve)
+    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    if maxfd == resource.RLIM_INFINITY:
+        maxfd = 65535
+    for fd in range(maxfd, 3, -1):  # 3 means omit stdin, stdout, stderr
+        if fd not in descr_preserve:
+            try:
+                os.close(fd)
+            except Exception:
+                pass
+    # PID file
+    if pidfile is not None:
+        pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
+        os.write(pidd, str(os.getpid())+"\n")
+        os.close(pidd)
+        # Define and setup atexit closure
+
+        @atexit.register
+        def unlink_pid():
+            try:
+                os.unlink(pidfile)
+            except Exception:
+                pass
+    # Redirect stdin, stdout, stderr to /dev/null
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for fd in range(3):
+        os.dup2(devnull, fd)
+
+
+def save_events(aggr, filer):
+    for event in aggr:
+        f, name = filer.create_unique_file()
+        with f:
+            f.write(json.dumps(event, ensure_ascii=True))
+        filer.publish_file(name)
+        logging.info(
+            "Saved %s %s \"%s\" (%d) as file %s" % (
+                event["ID"], "+".join(event["Category"]), event["Description"], event["ConnCount"], name))
+
+
+RE_LIST = (
+    # 1493035442 Initial Connect - tarpitting: 89.163.242.15 56736 -> 195.113.254.182 9898
+    # 1493037991 Inbound SYN/ACK: 185.62.190.15 21001 -> 195.113.252.222 15584
+    (
+        re.compile(r'([0-9]+) ([^:]*:) ([^ ]+) ([0-9]+) -> ([^ ]+) ([0-9]+).*'),
+        namedtuple("connect_tuple", ("timestamp", "message", "src_ip", "src_port", "tgt_ip", "tgt_port"))
+    ),
+    # 1493035442 Responded to a Ping: 88.86.96.25 -> 195.113.253.87 *
+    (
+        re.compile(r'([0-9]+) ([^:]*:) ([^ ]+) -> ([^ ]+).*'),
+        namedtuple("ping_tuple", ("timestamp", "message", "src_ip", "tgt_ip"))
+    )
+)
+
+
+def match_event(line):
+    for labrea_re, event_tuple in RE_LIST:
+        match = labrea_re.match(line)
+        if match:
+            return event_tuple(*match.groups())
+    logging.info("Unmatched line: \"%s\"" % line.replace("\n", r"\n"))
+    return None
+
+
+def get_args():
+    optp = optparse.OptionParser(
+        usage="usage: %prog [options] logfile ...",
+        description="Watch LaBrea logfiles and generate Idea events into directory")
+    optp.add_option(
+        "-w", "--window",
+        default=900,
+        dest="window",
+        type="int",
+        action="store",
+        help="max detection window (default: %default)")
+    optp.add_option(
+        "-t", "--timeout",
+        default=300,
+        dest="timeout",
+        type="int",
+        action="store",
+        help="detection timeout (default: %default)")
+    optp.add_option(
+        "-n", "--name",
+        default=None,
+        dest="name",
+        type="string",
+        action="store",
+        help="Warden client name")
+    optp.add_option(
+        "--test",
+        default=False,
+        dest="test",
+        action="store_true",
+        help="Add Test category")
+    optp.add_option(
+        "-o", "--oneshot",
+        default=False,
+        dest="oneshot",
+        action="store_true",
+        help="process files and quit (do not daemonize)")
+    optp.add_option(
+        "--poll",
+        default=1,
+        dest="poll",
+        type="int",
+        action="store",
+        help="log file polling interval")
+    optp.add_option(
+        "-d", "--dir",
+        default=None,
+        dest="dir",
+        type="string",
+        action="store",
+        help="Target directory (mandatory)"),
+    optp.add_option(
+        "-p", "--pid",
+        default=pth.join("/var/run", pth.splitext(pth.basename(sys.argv[0]))[0] + ".pid"),
+        dest="pid",
+        type="string",
+        action="store",
+        help="create PID file with this name (default: %default)")
+    optp.add_option(
+        "-u", "--uid",
+        default=None,
+        dest="uid",
+        type="int",
+        action="store",
+        help="user id to run under")
+    optp.add_option(
+        "-g", "--gid",
+        default=None,
+        dest="gid",
+        type="int",
+        action="store",
+        help="group id to run under")
+    optp.add_option(
+        "-v", "--verbose",
+        dest="verbose",
+        action="store_true",
+        help="turn on debug logging")
+    optp.add_option(
+        "--log",
+        default="local7",
+        dest="log",
+        type="string",
+        action="store",
+        help="syslog facility or log file name (default: %default)")
+    optp.add_option(
+        "--realtime",
+        default=True,
+        dest="realtime",
+        action="store_true",
+        help="use system time along with log timestamps (default)")
+    optp.add_option(
+        "--norealtime",
+        dest="realtime",
+        action="store_false",
+        help="don't system time, use solely log timestamps")
+    return optp
+
+
+running_flag = True
+reload_flag = False
+
+
+def terminate_me(signum, frame):
+    global running_flag
+    running_flag = False
+
+
+def reload_me(signum, frame):
+    global reload_flag
+    reload_flag = True
+
+
+def main():
+    global reload_flag
+    optp = get_args()
+    opts, args = optp.parse_args()
+    if not args or opts.name is None or opts.dir is None:
+        optp.print_help()
+        sys.exit()
+
+    logformat = "%(filename)s[%(process)d] %(message)s"
+    logger = logging.getLogger()
+    if opts.oneshot:
+        handler = logging.StreamHandler()
+        handler.setFormatter(logging.Formatter("%(asctime)s " + logformat))
+    else:
+        if "/" in opts.log:
+            handler = logging.handlers.WatchedFileHandler(opts.log)
+            handler.setFormatter(logging.Formatter("%(asctime)s " + logformat))
+        else:
+            handler = logging.handlers.SysLogHandler(address="/dev/log", facility=opts.log)
+            handler.setFormatter(logging.Formatter(logformat))
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG if opts.verbose else logging.INFO)
+
+    try:
+        if opts.oneshot:
+            signal.signal(signal.SIGINT, terminate_me)
+            signal.signal(signal.SIGTERM, terminate_me)
+            files = [open(arg) for arg in args]
+        else:
+            daemonize(
+                pidfile=opts.pid,
+                uid=opts.uid,
+                gid=opts.gid,
+                signals={
+                    signal.SIGINT: terminate_me,
+                    signal.SIGTERM: terminate_me,
+                    signal.SIGHUP: reload_me
+                })
+            files = [FileWatcher(arg) for arg in args]
+
+        ideagen = IdeaGen(name=opts.name, test=opts.test)
+        filer = Filer(opts.dir)
+        contexts = [
+            context(window=opts.window, timeout=opts.timeout, ideagen=ideagen)
+            for context in [PingContextMgr, ConnectContextMgr, InboundContextMgr]
+        ]
+
+        with contextlib.nested(*files):
+            for line_set in izip(*files):
+                for line in line_set:
+                    if not line:
+                        break
+
+                    event = match_event(line)
+
+                    if event or opts.realtime:
+                        timestamp = int(event.timestamp) if event else int(time.time())
+                        for context in contexts:
+                            save_events(context.process(event, timestamp), filer)
+
+                if not running_flag:
+                    break
+                if reload_flag:
+                    for f in files:
+                        f.close()
+                        f.open()
+                    reload_flag = False
+                if not any(line_set):
+                    time.sleep(opts.poll)
+
+        for context in contexts:
+            timestamp = int(time.time()) if opts.realtime else None
+            save_events(context.close(timestamp), filer)
+
+    except Exception:
+        logging.exception("Exception")
+
+main()