Merge from main Warden repo

cznichaas2warden/README.md

+# haas2warden
+
+Warden connector for data of [CZ.NIC HaaS project](https://haas.nic.cz/).
+
+It downloads daily [HaaS data dumps](https://haas.nic.cz/stats/export/),
+converts them to IDEA messages and sends them to CESNET's Warden server.
+
+It should be run from `cron` every night when data from previous day are
+available (at 3:30).
+
+The script just writes IDEA messages as files into a "filer" directory.
+A _warden_filer_ daemon must be configured to pick up the messages
+and send them to Warden server.
+There is a systemd file which can be used to run the warden_filer.

cznichaas2warden/haas2warden.cron

+# Run every day at 03:30
+30 03 * * * haas2warden python3 /data/haas2warden/haas2warden.py -p /data/haas2warden/warden_filer/ -n org.example.ext.cznic_haas -t >> /data/haas2warden/haas2warden.log 2>&1 

cznichaas2warden/haas2warden.py

+#!/usr/bin/env python3
+from gzip import decompress
+from json import loads
+from datetime import datetime, timedelta
+import argparse
+import logging
+import uuid
+import json
+import os
+import requests
+
+
+data_date = datetime.date(datetime.utcnow()) - timedelta(days=1)
+
+LOGFORMAT = "%(asctime)-15s,%(name)s [%(levelname)s] %(message)s"
+LOGDATEFORMAT = "%Y-%m-%dT%H:%M:%S"
+logging.basicConfig(level=logging.INFO, format=LOGFORMAT, datefmt=LOGDATEFORMAT)
+
+logger = logging.getLogger('haas2warden')
+
+def createIDEAFile(idea_id, idea_msg):
+    """
+    Creates file for IDEA message in .../tmp folder, then move it to .../incoming folder
+    """
+    tmp_dir_path = os.path.join(args.path, "tmp")
+    idea_file_path = os.path.join(tmp_dir_path, idea_id+".idea")
+    os.makedirs(tmp_dir_path, exist_ok=True)
+    idea_file = open(idea_file_path, "w")
+    idea_file.write(idea_msg)
+    idea_file.close()
+
+    incoming_dir_path = os.path.join(args.path, "incoming")
+    incoming_file_path = os.path.join(incoming_dir_path,idea_id+".idea")
+    os.makedirs(incoming_dir_path, exist_ok=True) 
+    os.rename(idea_file_path,incoming_file_path)
+
+
+def createIDEA(time, time_closed, ip, login_successful, commands):
+    """
+    Creates IDEA message 
+    """ 
+    idea_id = str(uuid.uuid4())
+
+    if login_successful:
+        category = "[\"Intrusion.UserCompromise\"]" 
+        description = "SSH login on honeypot (HaaS)"
+        if args.test:
+            category = "[\"Intrusion.UserCompromise\", \"Test\"]"
+        attach = f''',
+   "Attach": [
+        {{
+            "Note": "commands",
+            "Type": ["ShellCode"],
+            "ContentType": "application/json",
+            "Content": {json.dumps(commands)}
+        }}
+    ]''' #              ^-- "commands" is already serialiezed into a json string, we want to include it into a bigger JSON so we must encode it again (to escape quotes and any other special charaters)
+
+    else:
+        category = "[\"Attempt.Login\"]" 
+        description = "Unsuccessful SSH login attempt on honeypot (HaaS)"
+        if args.test:
+            category = "[\"Attempt.Login\", \"Test\"]"
+        attach = ""
+
+    if time_closed: # sometimes time_closed is empty, in such case we must omit CeaseTime completely from IDEA msg
+        cease_time = f'"CeaseTime": "{time_closed}",'
+    else:
+        cease_time = ""
+
+    idea_msg = f"""\
+{{
+    "Format": "IDEA0",
+    "ID": "{idea_id}",
+    "Category": {category},
+    "Description": "{description}",
+    "Note": "Extracted from data of CZ.NIC HaaS project",
+    "DetectTime": "{time}",
+    "EventTime": "{time}",
+    {cease_time}
+    "CreateTime": "{datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')}",
+    "Source": [
+        {{
+            "IP4": ["{ip}"],
+            "Proto": ["tcp", "ssh"]
+        }}
+    ],
+    "Node": [
+        {{
+            "Name": "{args.name}",
+            "SW": ["CZ.NIC HaaS"],
+            "Type": ["Connection", "Auth", "Honeypot"],
+            "Note": "A script converting daily HaaS data dumps from https://haas.nic.cz/stats/export/"
+        }}
+    ]{attach}
+}}
+"""
+    createIDEAFile(idea_id, idea_msg)
+
+    
+def processJSON():
+    """
+    Downloads data from https://haas.nic.cz/stats/export/ and process json files.
+    """
+    date = datetime.strptime(args.date, '%Y-%m-%d').date()
+    # get url
+    url = "https://haas.nic.cz/stats/export/{}/{}/{}.json.gz".format(str(date).split('-')[0],str(date).split('-')[1], str(date))
+    # get data 
+    logger.info("Downloading {}".format(url))
+    response = requests.get(url)
+    if response.status_code == 200:
+        # unzip and read json file
+        json_objects = loads(decompress(response.content))
+        logger.info("Found {} records, converting to IDEA messages".format(len(json_objects)))
+        # go through all json objects
+        for json_object in json_objects:
+            createIDEA(json_object["time"], json_object["time_closed"], json_object["ip"], json_object["login_successful"], json.dumps(json_object["commands"]))
+
+if __name__ == "__main__":
+    
+    # parse arguments
+    parser = argparse.ArgumentParser(
+        prog="haas_receiver.py",
+        description="A script converting daily HaaS data dumps from https://haas.nic.cz/stats/export/"
+    )
+
+    parser.add_argument('-d', '--date', metavar='DATE', default = str(data_date),
+                        help='To download data from date YYYY-MM-DD, use date + 1 day (default: utcnow - 1 day)')
+    parser.add_argument('-p', '--path', metavar='DIRPATH', default = "/data/haas2warden/warden_filer/",
+                        help='Target folder for Idea messages (default: "/data/haas2warden/warden_filer/")')
+    parser.add_argument('-n', '--name', metavar='NODENAME', default = "undefined",
+                        help='Name of the node (default: undefined)')
+    parser.add_argument('-t', '--test', action="store_true",
+                        help='Test category')                                        
+
+    args = parser.parse_args()
+    
+    processJSON()
+    logger.info("Done")

cznichaas2warden/warden-filer.service

+# Template of Systemd unit for Warden filer daemon
+#
+# TODO: set paths, username and mode (receiver/sender) in the last two lines
+# and uncomment them. Then copy the file to:
+#   /etc/systemd/system/warden-filer.service
+# and run:
+#   systemctl daemon-reload
+
+[Unit]
+Description=Warden filer for haas2warden
+After=syslog.target network.target
+
+[Service]
+Type=forking
+User=haas2warden
+PIDFile=/data/haas2warden/warden_filer.pid
+ExecStart=/opt/warden_filer/warden_filer.py --daemon -c "/data/haas2warden/warden_filer.cfg" --pid_file "/data/haas2warden/warden_filer.pid" sender
+

cznichaas2warden/warden_filer.cfg

+{
+    // Warden config can be also referenced as:
+    // "warden": "/path/to/warden_client.cfg"
+    "warden": {
+        "url": "https://warden-hub.cesnet.cz/warden3",
+        "cafile": "/etc/pki/tls/certs/ca-bundle.crt",
+        "keyfile": "/data/haas2warden/key.pem",
+        "certfile": "/data/haas2warden/cert.pem",
+        "timeout": 10,
+        "errlog": {"level": "warning"},
+        "filelog": {"level": "info", "file": "/data/haas2warden/warden_filer.log"},
+        "idstore": "/data/haas2warden/warden_filer.id",
+        "name": "org.example.cznic_haas"
+    },
+    "sender": {
+        // Maildir like directory, whose "incoming" subdir will be checked
+        // for Idea events to send out
+        "dir": "/data/haas2warden/warden_filer",
+        "poll_time": 60
+    }
+}

fail2ban/README

+Support scripts for fail2ban
+============================
+
+Introduction
+------------
+
+Fail2ban is a logfile watcher, which is able to run various actions,
+based on too many patterns occured in the log file.
+Those helper shell scripts can be used as action to report events to
+Warden_.
+
+Dependencies
+------------
+
+ 1. Python packages
+
+    warden_filer 3.0+
+
+Usage
+-----
+
+ * f2ban_spam.sh is meant to be used in cooperation with the default
+   "postfix" rule.
+ * f2ban_ssh.sh is meant to be used in cooperation with the default
+   "ssh" rule. 
+
+In the corresponding action following invocation can be used:
+
+   actionban = /usr/local/bin/f2ban_XXX.sh <ip> <failures> <time>
+
+Please, edit corresponding paths and Warden names in the corresponding
+script preamble and check/edit contents of the IDEA template (e.g. Target IP 
+address in f2ban_ssh.sh).
+
+Scripts write generated Idea_ events into warden_filer compatible
+directory, so you will need to run properly configured (and registered
+into Warden server) warden_filer instance, which will take care for
+picking up the events and submitting them.
+
+.. _Warden: https://warden.cesnet.cz/
+.. _Idea: https://idea.cesnet.cz/
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

fail2ban/f2ban_spam.sh

+#!/bin/bash
+
+umask 0111
+
+filer_dir="/var/mentat/spool/_wardenout"
+src_ip=$1
+failures=$2
+detect_time=$(date --date="@$3" --rfc-3339=seconds)
+create_time=$(date --rfc-3339=seconds)
+node_name="org.example.fail2ban.blacklist"
+
+uuid() {
+        for ((n=0; n<16; n++)); do  
+                read -n1 c < /dev/urandom
+                LC_CTYPE=C d=$(printf '%d' "'$c")
+                s=''
+                case $n in
+                        6) ((d = d & 79 | 64));;   
+                        8) ((d = d & 191 | 128));;  
+                        3|5|9|7) s='-';; 
+                esac
+                printf '%02x%s' $d "$s"
+        done
+}
+
+event_id=$(uuid)
+
+cat >"$filer_dir/tmp/$event_id" <<EOF
+{
+   "Format" : "IDEA0",
+   "ID" : "$event_id",
+   "DetectTime" : "$detect_time",
+   "CreateTime" : "$create_time",
+   "Category" : ["Abusive.Spam"],
+   "Description" : "Blacklisted host",
+   "Note" : "Block duration: 3600. IP was blacklisted, is listed on more than 5 public blacklists",
+   "Source" : [{
+      "Type": ["Spam"],
+      "IP4" : ["$src_ip"],
+      "Proto": ["tcp", "smtp"]
+   }],
+   "Node" : [{
+         "Name" : "$node_name",
+         "SW" : ["Fail2Ban"],
+         "Type" : ["Log", "Statistical"]
+   }],
+   "_CESNET" : {
+      "Impact" : "IP was blacklisted, is listed on more than 5 public blacklists",
+      "EventTemplate" : "f2b-001"
+   }
+}
+EOF
+
+mv "$filer_dir/tmp/$event_id" "$filer_dir/incoming"

fail2ban/f2ban_ssh.sh

+#!/bin/bash
+
+umask 0111
+
+filer_dir="/var/spool/warden_sender"
+src_ip=$1
+failures=$2
+detect_time=$(date --date="@$3" --rfc-3339=seconds)
+create_time=$(date --rfc-3339=seconds)
+node_name="org.example.fail2ban.ssh"
+
+uuid() {
+        for ((n=0; n<16; n++)); do  
+                read -n1 c < /dev/urandom
+                LC_CTYPE=C d=$(printf '%d' "'$c")
+                s=''
+                case $n in
+                        6) ((d = d & 79 | 64));;   
+                        8) ((d = d & 191 | 128));;  
+                        3|5|9|7) s='-';; 
+                esac
+                printf '%02x%s' $d "$s"
+        done
+}
+
+event_id=$(uuid)
+
+cat >"$filer_dir/tmp/$event_id" <<EOF
+{
+   "Format": "IDEA0",
+   "ID": "$event_id",
+   "DetectTime": "$detect_time",
+   "CreateTime": "$create_time",
+   "Category": ["Attempt.Login"],
+   "Description": "SSH dictionary/bruteforce attack",
+   "ConnCount": $failures,
+   "Note": "IP attempted $failures logins to SSH service",
+   "Source": [{
+      "IP4": ["$src_ip"],
+      "Proto": ["tcp", "ssh"]
+   }],
+   "Target": [{
+       "Type": ["Anonymised"],
+       "IP4": ["192.0.2.0/24"],
+       "Anonymised": true,
+       "Proto": ["tcp", "ssh"],
+       "Port": [22]
+   }],
+   "Node": [{
+         "Name": "$node_name",
+         "SW": ["Fail2Ban"],
+         "Type": ["Log", "Statistical"]
+   }]
+}
+EOF
+
+mv "$filer_dir/tmp/$event_id" "$filer_dir/incoming"

hp-labrea/README

+Warden LaBrea connector 0.1 for Warden 3.X
+==========================================
+
+Introduction
+------------
+
+labrea-idea.py is a daemon, meant for continuous watching of LaBrea log files
+and generation of Idea_ format of corresponding security events. It is
+usually run in correspondence with warden_filer daemon, which picks the
+resulting events up and feeds them to the Warden_ server. Connector supports
+sliding window aggregation, so sets of connections with the same source are
+reported as one event (within aggregation window).
+
+
+Dependencies
+------------
+
+ 1. Platform
+
+    Python 2.7+
+
+ 2. Python packages
+
+    warden_filer 3.0+ (recommended)
+
+
+Usage
+-----
+
+        ./labrea-idea.py [options] logfile ...
+
+        Options:
+          -h, --help            show this help message and exit
+          -w WINDOW, --window=WINDOW
+                                max detection window (default: 900)
+          -t TIMEOUT, --timeout=TIMEOUT
+                                detection timeout (default: 300)
+          -n NAME, --name=NAME  Warden client name
+          --test                Add Test category
+          -o, --oneshot         process files and quit (do not daemonize)
+          --poll=POLL           log file polling interval
+          -d DIR, --dir=DIR     Target directory (mandatory)
+          -p PID, --pid=PID     create PID file with this name (default: /var/run
+                                /labrea-idea.pid)
+          -u UID, --uid=UID     user id to run under
+          -g GID, --gid=GID     group id to run under
+          -v, --verbose         turn on debug logging
+          --log=LOG             syslog facility or log file name (default: local7)
+          --realtime            use system time along with log timestamps (default)
+          --norealtime          don't system time, use solely log timestamps
+
+
+Configuration
+-------------
+
+However, the daemon is usually run by init script (example one is a part of
+the distribution, along with sample logrotate definition). Options then can
+be configured by /etc/sysconfig/labrea-idea or /etc/defaults/labrea-idea,
+depending on your distribution custom, where at least PARAMS variable has
+to be specified (for others, see the init script).
+    
+.. _Warden: https://warden.cesnet.cz/
+.. _Idea: https://idea.cesnet.cz/
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

hp-labrea/labrea-idea

+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:          labrea-idea
+# Required-Start:    $local_fs $syslog
+# Required-Stop:     $local_fs $syslog
+# Should-Start:      $network
+# Should-Stop:       $network
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Labrea-Idea aggregator/converter
+### END INIT INFO
+
+DAEMON_NAME=labrea-idea
+DAEMON_PATH=/usr/local/bin/"$DAEMON_NAME".py
+PID=/var/run/"$DAEMON_NAME".pid
+
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$DAEMON_NAME" ] && . /etc/$n/"$DAEMON_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
+
+ACTION="$1"
+
+case "$ACTION" in
+	start)
+		if [ -z "$PARAMS" ]; then
+			log_daemon_msg "Unconfigured $DAEMON_NAME, not starting."
+			exit 2
+		fi
+		mkdir -p "${PID%/*}"
+		log_daemon_msg "Starting $DAEMON_NAME"
+		start_daemon -p "$PID" "$DAEMON_PATH" --pid "$PID" $PARAMS
+		log_end_msg $?
+		;;
+	stop)
+		log_daemon_msg "Stopping $DAEMON_NAME"
+		killproc -p "$PID" "$DAEMON_PATH"
+		log_end_msg $?
+		;;
+	restart|force-reload)
+		$0 stop && sleep 2 && exec $0 start
+		;;
+	status)
+		status_of_proc -p "$PID" "$DAEMON_PATH"
+		;;
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 2
+		;;
+esac

hp-labrea/labrea-idea.logrotate

+/var/log/labrea-idea.log
+{
+	rotate 52
+	weekly
+	missingok
+	notifempty
+	compress
+	delaycompress
+	dateext
+	create 640 mentat mentat
+}