From 7cfc939004a50d29331f0cbcb674f79cd161f658 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=A1clav=20Barto=C5=A1?= <bartos@cesnet.cz>
Date: Tue, 8 Mar 2022 11:10:29 +0100
Subject: [PATCH] Added haas2warden connector
---
cznichaas2warden/README.md | 14 +++
cznichaas2warden/haas2warden.cron | 2 +
cznichaas2warden/haas2warden.py | 139 ++++++++++++++++++++++++++
cznichaas2warden/warden-filer.service | 18 ++++
cznichaas2warden/warden_filer.cfg | 21 ++++
5 files changed, 194 insertions(+)
create mode 100644 cznichaas2warden/README.md
create mode 100644 cznichaas2warden/haas2warden.cron
create mode 100644 cznichaas2warden/haas2warden.py
create mode 100644 cznichaas2warden/warden-filer.service
create mode 100644 cznichaas2warden/warden_filer.cfg
diff --git a/cznichaas2warden/README.md b/cznichaas2warden/README.md
new file mode 100644
index 0000000..42972f5
--- /dev/null
+++ b/cznichaas2warden/README.md
@@ -0,0 +1,14 @@
+# haas2warden
+
+Warden connector for data of [CZ.NIC HaaS project](https://haas.nic.cz/).
+
+It downloads daily [HaaS data dumps](https://haas.nic.cz/stats/export/),
+converts them to IDEA messages and sends them to CESNET's Warden server.
+
+It should be run from `cron` every night when data from previous day are
+available (at 3:30).
+
+The script just writes IDEA messages as files into a "filer" directory.
+A _warden_filer_ daemon must be configured to pick up the messages
+and send them to Warden server.
+There is a systemd file which can be used to run the warden_filer.
diff --git a/cznichaas2warden/haas2warden.cron b/cznichaas2warden/haas2warden.cron
new file mode 100644
index 0000000..dd0dbb6
--- /dev/null
+++ b/cznichaas2warden/haas2warden.cron
@@ -0,0 +1,2 @@
+# Run every day at 03:30
+30 03 * * * haas2warden python3 /data/haas2warden/haas2warden.py -p /data/haas2warden/warden_filer/ -n org.example.ext.cznic_haas -t >> /data/haas2warden/haas2warden.log 2>&1
diff --git a/cznichaas2warden/haas2warden.py b/cznichaas2warden/haas2warden.py
new file mode 100644
index 0000000..130b82f
--- /dev/null
+++ b/cznichaas2warden/haas2warden.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+from gzip import decompress
+from json import loads
+from datetime import datetime, timedelta
+import argparse
+import logging
+import uuid
+import json
+import os
+import requests
+
+
+data_date = datetime.date(datetime.utcnow()) - timedelta(days=1)
+
+LOGFORMAT = "%(asctime)-15s,%(name)s [%(levelname)s] %(message)s"
+LOGDATEFORMAT = "%Y-%m-%dT%H:%M:%S"
+logging.basicConfig(level=logging.INFO, format=LOGFORMAT, datefmt=LOGDATEFORMAT)
+
+logger = logging.getLogger('haas2warden')
+
+def createIDEAFile(idea_id, idea_msg):
+ """
+ Creates file for IDEA message in .../tmp folder, then move it to .../incoming folder
+ """
+ tmp_dir_path = os.path.join(args.path, "tmp")
+ idea_file_path = os.path.join(tmp_dir_path, idea_id+".idea")
+ os.makedirs(tmp_dir_path, exist_ok=True)
+ idea_file = open(idea_file_path, "w")
+ idea_file.write(idea_msg)
+ idea_file.close()
+
+ incoming_dir_path = os.path.join(args.path, "incoming")
+ incoming_file_path = os.path.join(incoming_dir_path,idea_id+".idea")
+ os.makedirs(incoming_dir_path, exist_ok=True)
+ os.rename(idea_file_path,incoming_file_path)
+
+
+def createIDEA(time, time_closed, ip, login_successful, commands):
+ """
+ Creates IDEA message
+ """
+ idea_id = str(uuid.uuid4())
+
+ if login_successful:
+ category = "[\"Intrusion.UserCompromise\"]"
+ description = "SSH login on honeypot (HaaS)"
+ if args.test:
+ category = "[\"Intrusion.UserCompromise\", \"Test\"]"
+ attach = f''',
+ "Attach": [
+ {{
+ "Note": "commands",
+ "Type": ["ShellCode"],
+ "ContentType": "application/json",
+ "Content": {json.dumps(commands)}
+ }}
+ ]''' # ^-- "commands" is already serialiezed into a json string, we want to include it into a bigger JSON so we must encode it again (to escape quotes and any other special charaters)
+
+ else:
+ category = "[\"Attempt.Login\"]"
+ description = "Unsuccessful SSH login attempt on honeypot (HaaS)"
+ if args.test:
+ category = "[\"Attempt.Login\", \"Test\"]"
+ attach = ""
+
+ if time_closed: # sometimes time_closed is empty, in such case we must omit CeaseTime completely from IDEA msg
+ cease_time = f'"CeaseTime": "{time_closed}",'
+ else:
+ cease_time = ""
+
+ idea_msg = f"""\
+{{
+ "Format": "IDEA0",
+ "ID": "{idea_id}",
+ "Category": {category},
+ "Description": "{description}",
+ "Note": "Extracted from data of CZ.NIC HaaS project",
+ "DetectTime": "{time}",
+ "EventTime": "{time}",
+ {cease_time}
+ "CreateTime": "{datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')}",
+ "Source": [
+ {{
+ "IP4": ["{ip}"],
+ "Proto": ["tcp", "ssh"]
+ }}
+ ],
+ "Node": [
+ {{
+ "Name": "{args.name}",
+ "SW": ["CZ.NIC HaaS"],
+ "Type": ["Connection", "Auth", "Honeypot"],
+ "Note": "A script converting daily HaaS data dumps from https://haas.nic.cz/stats/export/"
+ }}
+ ]{attach}
+}}
+"""
+ createIDEAFile(idea_id, idea_msg)
+
+
+def processJSON():
+ """
+ Downloads data from https://haas.nic.cz/stats/export/ and process json files.
+ """
+ date = datetime.strptime(args.date, '%Y-%m-%d').date()
+ # get url
+ url = "https://haas.nic.cz/stats/export/{}/{}/{}.json.gz".format(str(date).split('-')[0],str(date).split('-')[1], str(date))
+ # get data
+ logger.info("Downloading {}".format(url))
+ response = requests.get(url)
+ if response.status_code == 200:
+ # unzip and read json file
+ json_objects = loads(decompress(response.content))
+ logger.info("Found {} records, converting to IDEA messages".format(len(json_objects)))
+ # go through all json objects
+ for json_object in json_objects:
+ createIDEA(json_object["time"], json_object["time_closed"], json_object["ip"], json_object["login_successful"], json.dumps(json_object["commands"]))
+
+if __name__ == "__main__":
+
+ # parse arguments
+ parser = argparse.ArgumentParser(
+ prog="haas_receiver.py",
+ description="A script converting daily HaaS data dumps from https://haas.nic.cz/stats/export/"
+ )
+
+ parser.add_argument('-d', '--date', metavar='DATE', default = str(data_date),
+ help='To download data from date YYYY-MM-DD, use date + 1 day (default: utcnow - 1 day)')
+ parser.add_argument('-p', '--path', metavar='DIRPATH', default = "/data/haas2warden/warden_filer/",
+ help='Target folder for Idea messages (default: "/data/haas2warden/warden_filer/")')
+ parser.add_argument('-n', '--name', metavar='NODENAME', default = "undefined",
+ help='Name of the node (default: undefined)')
+ parser.add_argument('-t', '--test', action="store_true",
+ help='Test category')
+
+ args = parser.parse_args()
+
+ processJSON()
+ logger.info("Done")
diff --git a/cznichaas2warden/warden-filer.service b/cznichaas2warden/warden-filer.service
new file mode 100644
index 0000000..ca81c2a
--- /dev/null
+++ b/cznichaas2warden/warden-filer.service
@@ -0,0 +1,18 @@
+# Template of Systemd unit for Warden filer daemon
+#
+# TODO: set paths, username and mode (receiver/sender) in the last two lines
+# and uncomment them. Then copy the file to:
+# /etc/systemd/system/warden-filer.service
+# and run:
+# systemctl daemon-reload
+
+[Unit]
+Description=Warden filer for haas2warden
+After=syslog.target network.target
+
+[Service]
+Type=forking
+User=haas2warden
+PIDFile=/data/haas2warden/warden_filer.pid
+ExecStart=/opt/warden_filer/warden_filer.py --daemon -c "/data/haas2warden/warden_filer.cfg" --pid_file "/data/haas2warden/warden_filer.pid" sender
+
diff --git a/cznichaas2warden/warden_filer.cfg b/cznichaas2warden/warden_filer.cfg
new file mode 100644
index 0000000..21b46cf
--- /dev/null
+++ b/cznichaas2warden/warden_filer.cfg
@@ -0,0 +1,21 @@
+{
+ // Warden config can be also referenced as:
+ // "warden": "/path/to/warden_client.cfg"
+ "warden": {
+ "url": "https://warden-hub.cesnet.cz/warden3",
+ "cafile": "/etc/pki/tls/certs/ca-bundle.crt",
+ "keyfile": "/data/haas2warden/key.pem",
+ "certfile": "/data/haas2warden/cert.pem",
+ "timeout": 10,
+ "errlog": {"level": "warning"},
+ "filelog": {"level": "info", "file": "/data/haas2warden/warden_filer.log"},
+ "idstore": "/data/haas2warden/warden_filer.id",
+ "name": "org.example.cznic_haas"
+ },
+ "sender": {
+ // Maildir like directory, whose "incoming" subdir will be checked
+ // for Idea events to send out
+ "dir": "/data/haas2warden/warden_filer",
+ "poll_time": 60
+ }
+}
--
GitLab