diff --git a/censys/README.txt b/censys/README.txt
new file mode 100644
index 0000000000000000000000000000000000000000..abc09184694de38870744b32b26a73230d8547b8
--- /dev/null
+++ b/censys/README.txt
@@ -0,0 +1,68 @@
+=========================
+Censys2warden connector
+=========================
+
+This connector uses the internet-wide scanning service Censys.io to search for
+potentially vulnerable open services in given network (ASN) and reports them to
+Warden (via Warden filer daemon).
+
+The script sends a set of preconfigured queries to Censys API to search for all
+matching IPs in given ASN. The queries correspond to various potentially
+vulnerable hosts or other problems with open services.
+
+Since Censys' free account has quite strict limits on number of queries and
+results obtained, it is recommended to have at least the lowest tier of a paid
+account.
+
+Currently implemented queries:
+
+- Publicly accessible IPMI protocol
+- Publicly accessible SCADA (BACnet) system
+- Publicly accessible printer via IPP protocol
+- Open MongoDB database
+- Publicly accessible PCAnywhere (an unsupported vulnerable remote-access SW)
+- Open Elastic database
+- Web page or other service with a "hacked by" message
+- Web running on an old (unsupported) PHP version
+
+Important: These rules may need to be changed according to needs, interests and
+security policies of each organization.
+
+For each host in given ASN found to be matching a query, a corresponding IDEA
+message is created and put to an output directory. Messages from this directory
+should be sent to Warden by a Warden filer daemon.
+
+The script is assumed to be run periodically (e.g. once a day) by cron.
+
+
+-------------------------
+Installation:
+
+1. create a directory for the script and resulting IDEA files, e.g.
+ `/data/censys2warden/` and `/data/censys2warden/warden_sender`
+2. put the censys2warden.py script into the first directory
+3. set up warden_filer daemon to read IDEA messages from
+ `/data/censys2warden/warden_sender` and send them to Warden
+ (see README of Warden filer)
+4. set up cron to run the scirpt every day (use censys2warden.cron as an
+ example)
+
+
+-------------------------
+Configuration:
+
+The script takes the following arguments:
+
+ -h, --help show this help message and exit
+ -i APIID, --apiid APIID
+ Censys API ID
+ -s APISECRET, --apisecret APISECRET
+ Censys API secret
+ -a ASN, --asn ASN ASN to query
+ -n NODE, --node NODE Node name to fill into IDEA messages
+ -d PATH, --destdir PATH
+ Path to destination directory (with 'incoming' and
+ 'temp' subdirectories) (default: CWD)
+ -t, --test Add 'Test' category to IDEA messages.
+ -v, --verbose Print information about progress and results
+
diff --git a/censys/censys2warden.cron b/censys/censys2warden.cron
new file mode 100644
index 0000000000000000000000000000000000000000..df55515ba31be5928226212e370ce9d6f6e1dd18
--- /dev/null
+++ b/censys/censys2warden.cron
@@ -0,0 +1,2 @@
+# Run every day at 9:00
+0 9 * * * shodan2warden python3 /data/censys2warden/censys2warden.py $(cat /data/censys2warden/api_params) -a 2852 -n cz.cesnet.ext.censys -d /data/censys2warden/warden_sender --test -v >>/data/censys2warden/censys2warden.log 2>&1
\ No newline at end of file
diff --git a/censys/censys2warden.py b/censys/censys2warden.py
new file mode 100644
index 0000000000000000000000000000000000000000..cab2bc6c826950590ec4fe3e4a0c46693d12c456
--- /dev/null
+++ b/censys/censys2warden.py
@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Author: Pavla Hlučková
+# Václav Bartoš <bartos@cesnet.cz>
+
+import censys.ipv4
+import json
+import os
+from datetime import datetime
+from uuid import uuid4
+import argparse
+
+def vprint(*args, **kwargs):
+ # Verbose print
+ if VERBOSE:
+ print("[{}] ".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")), end="")
+ print(*args, **kwargs)
+
+IPV4_FIELDS = ['ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
+
+MAX_RESULTS_PER_QUERY = 1000
+
+def generateIdeaEvent(detect_time, category, description,ip_string,ports,proto,content_type,content,note):
+ create_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+ # if there's no timezone in detect_time, assume UTC (which Shodan normally uses) and append 'Z'
+ if 'Z' not in detect_time and '+' not in detect_time and '-' not in detect_time:
+ detect_time += 'Z'
+ event = {
+ "Format": "IDEA0",
+ "ID": str(uuid4()),
+ "Category": [category],
+ "CreateTime": create_time,
+ "DetectTime": detect_time,
+ "Description": description,
+ "Ref": ["https://www.censys.io/ipv4/" + ip_string],
+ "Source": [
+ {
+ "IP4": [ip_string],
+ "Port": ports,
+ "Proto": proto
+ }
+ ],
+ "Node": [
+ {
+ "Name": node_name,
+ "SW": ["censys2warden"],
+ "Type": ["External", "Recon"]
+ }
+ ],
+ "Attach": [
+ {
+ "ContentType": content_type,
+ "Content": json.dumps(content) if content_type == 'application/json' else content,
+ "Note": note
+ }
+ ]
+ }
+
+ if test_category:
+ event["Category"].append("Test")
+
+ filename = "{}_{}_{}_{}.json".format(
+ datetime.utcnow().strftime('%Y%m%d%H%M%S%f'),
+ category.replace('.','').lower(),
+ ip_string,
+ event['ID'][:8])
+ tmp_destination = os.path.join(default_directory, 'tmp', filename)
+ inc_destination = os.path.join(default_directory, 'incoming', filename)
+
+ with open(tmp_destination, 'w') as json_file:
+ json.dump(event, json_file)
+
+ os.rename(tmp_destination, inc_destination)
+
+def IPMI():
+ query = asnString + "protocols: \"623/ipmi\" tags: ipmi"
+ category = "Vulnerable.Config"
+ description = "Publicly accessible insecure protocol: IPMI"
+ proto = ["udp", "ipmi"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for IMPI:", query)
+ for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[623],proto,content_type,banner,note)
+
+def SCADA():
+ query = asnString + "scada"
+ category = "Vulnerable.Config"
+ description = "Publicly accessible SCADA (BACnet) system"
+ proto = ["udp", "bacnet"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for SCADA:", query)
+ for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
+ # TODO: find out the port of the scada protocol(s)
+ # (sometimes there are multiple services running on the IP)
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),banner.get('ports'),proto,content_type,banner,note)
+
+def printerIPP():
+ query = asnString + "protocols: \"631/ipp\""
+ category = "Vulnerable.Config"
+ description = "Potentially vulnerable IPP printer"
+ proto = ["tcp", "ipp"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for printer IPP:", query)
+ for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[631],proto,content_type,banner,note)
+
+def mongoDB():
+ query = asnString + "protocols: \"27017/mongodb\""
+ category = "Vulnerable.Config"
+ description = "Potentially vulnerable MongoDB database"
+ proto = ["tcp"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for MongoDB:", query)
+ for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+ # TODO: try to find a field with DB size, but it seems there's none
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[27017],proto,content_type,banner,note)
+
+def PCA():
+ query = asnString + "tags: pca"
+ category = "Vulnerable.Config"
+ description = "An old unsupported service 'PCAnywhere' open to internet"
+ proto = ["tcp","pca"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for PCAnywhere:", query)
+ for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+def elasticSearch():
+ query = asnString + "protocols: \"9200/elasticsearch\""
+ category = "Vulnerable.Config"
+ description = "Possibly vulnerable data displayed - Elastic Search"
+ proto = ["tcp"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for Elastic Indices:", query)
+ for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+
+def hacked():
+ IPV4_FIELDS_HACKED = ['80.http.get.body','443.https.get.body','ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
+ query = asnString + "\"hacked\""
+ category = "Information.UnauthorizedModification"
+ description = "Service probably hacked (\"hacked\" string found in banner)"
+ proto = []
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for \"hacked\" string:", query)
+ for banner in c.search(query, IPV4_FIELDS_HACKED,max_records=MAX_RESULTS_PER_QUERY):
+ if "Test Page" in str(banner.get('80.http.get.body')) or "Test Page" in str(banner.get('443.https.get.body')):
+ continue
+ if "Cyber Security" in str(banner.get('80.http.get.body')) or "Cyber Security" in str(banner.get('443.https.get.body')):
+ # this was added to filter a false positive on a cybersecurity page mentioning hacking, see http://158.196.109.174/
+ continue
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+def unsupportedPHP():
+ query = asnString + "(80.http.get.headers.x_powered_by: PHP\\/5.* OR 8080.http.get.headers.x_powered_by: PHP\\/5.* OR 443.https.get.headers.x_powered_by: PHP\\/5.*)"
+ category = "Vulnerable.Open"
+ description = "Web running on old (unsupported) PHP version"
+ proto = ["tcp", "http"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for unsupported PHP:", query)
+ for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+
+def parse_args():
+ # command line argument parser
+ parser = argparse.ArgumentParser(
+ description="Searches Censys for potential problems with open services in given ASN. For each such problem generates an IDEA message into gievn directory (to be sent to Warden by warden_filer). This script is assumed to be run daily by cron.")
+ parser.add_argument('-i', '--apiid', required=True,
+ help="Censys API ID")
+ parser.add_argument('-s', '--apisecret', required=True,
+ help="Censys API secret")
+ parser.add_argument('-a', '--asn', type=int, required=True,
+ help="ASN to query")
+ parser.add_argument('-n', '--node', required=True,
+ help="Node name to fill into IDEA messages")
+ parser.add_argument('-d', '--destdir', dest="path", default=os.getcwd(),
+ help="Path to destination directory (with 'incoming' and 'temp' subdirectories) (default: CWD)")
+ parser.add_argument('-t', '--test', action="store_true",
+ help="Add 'Test' category to IDEA messages.")
+ parser.add_argument('-v', '--verbose', action="store_true",
+ help="Print information about progress and results")
+ return parser.parse_args()
+
+
+def main():
+ IPMI()
+ SCADA()
+ printerIPP()
+ mongoDB()
+ PCA()
+ elasticSearch()
+ hacked()
+ unsupportedPHP()
+
+
+if __name__ == "__main__":
+ # getting arguments from argparse
+ args = parse_args()
+ VERBOSE = args.verbose
+ default_directory = args.path
+ node_name = args.node
+ test_category = args.test
+ asnString = "autonomous_system.asn:" + str(args.asn) + " AND "
+
+ c = censys.ipv4.CensysIPv4(api_id=str(args.apiid),
+ api_secret=str(args.apisecret))
+
+ # incoming directory creation
+ directory = "incoming"
+ path = os.path.join(default_directory, directory)
+ os.makedirs(path, exist_ok=True)
+
+ # tmp directory creation
+ directory = "tmp"
+ path = os.path.join(default_directory, directory)
+ os.makedirs(path, exist_ok=True)
+
+ main()
+
diff --git a/cowrie/warden_client_cowrie.cfg b/cowrie/warden_client_cowrie.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..5668548946e964bee92ae0705d758418ee639ab9
--- /dev/null
+++ b/cowrie/warden_client_cowrie.cfg
@@ -0,0 +1,15 @@
+{
+ "warden": "warden_client.cfg",
+ "name": "cz.cesnet.server.kippo",
+ "secret": "",
+
+ "anonymised": "no",
+ "target_net": "195.113.0.0/16",
+
+ "dbhost": "localhost",
+ "dbuser": "kippo",
+ "dbpass": "kippopass",
+ "dbname": "kippo",
+ "dbport": 3306,
+ "awin": 5
+}
diff --git a/cowrie/warden_sender_cowrie.py b/cowrie/warden_sender_cowrie.py
new file mode 100755
index 0000000000000000000000000000000000000000..db056aaa18cc2a1e75477e24fffc6c6e950e2872
--- /dev/null
+++ b/cowrie/warden_sender_cowrie.py
@@ -0,0 +1,305 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+
+from warden_client import Client, Error, read_cfg, format_timestamp
+from time import time, gmtime, strftime
+from math import trunc
+from uuid import uuid4
+import MySQLdb as my
+import MySQLdb.cursors as mycursors
+import tempfile, subprocess, base64
+import json
+import string
+import os
+import sys
+
+#warden client startup
+aconfig = read_cfg('warden_client_cowrie.cfg')
+wconfig = read_cfg('warden_client.cfg')
+aclient_name = aconfig['name']
+wconfig['name'] = aclient_name
+aanonymised = aconfig['anonymised']
+aanonymised_net = aconfig['target_net']
+aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
+awin = aconfig['awin'] * 60
+atest = aconfig['test_mode']
+
+wclient = Client(**wconfig)
+
+def idea_fill_addresses(event, source_ip, destination_ip, anonymised, anonymised_net):
+ af = "IP4" if not ':' in source_ip else "IP6"
+ event['Source'][0][af] = [source_ip]
+ if anonymised != 'omit':
+ if anonymised == 'yes':
+ event['Target'][0]['Anonymised'] = True
+ event['Target'][0][af] = [anonymised_net]
+ else:
+ event['Target'][0][af] = [destination_ip]
+ return event
+
+def gen_event_idea_cowrie_info(detect_time, src_ip, dst_ip, win_start_time, win_end_time, aggr_win, conn_count):
+ event = {
+ "Format": "IDEA0",
+ "ID": str(uuid4()),
+ "DetectTime": detect_time,
+ "WinStartTime": win_start_time,
+ "WinEndTime": win_end_time,
+ "Category": ["Attempt.Login"],
+ "Note": "SSH login attempt",
+ "ConnCount": conn_count,
+ "Source": [{}],
+ "Target": [{ "Proto": ["tcp", "ssh"], "Port": [22]}],
+ "Node": [
+ {
+ "Name": aclient_name,
+ "Type": ["Connection","Honeypot","Recon"],
+ "SW": ["Cowrie"],
+ "AggrWin": strftime("%H:%M:%S", gmtime(aggr_win))
+ }
+ ]
+ }
+ # Test if we're testing
+ if atest == "true":
+ event["Category"].append('Test')
+
+ event = idea_fill_addresses(event, src_ip, dst_ip, aanonymised, aanonymised_net)
+ return event
+
+
+def gen_event_idea_cowrie_auth(detect_time, src_ip, dst_ip, username, password, sessionid):
+
+ event = {
+ "Format": "IDEA0",
+ "ID": str(uuid4()),
+ "DetectTime": detect_time,
+ "Category": ["Information.UnauthorizedAccess"],
+ "Note": "SSH successfull attempt",
+ "ConnCount": 1,
+ "Source": [{}],
+ "Target": [{ "Proto": ["tcp", "ssh"], "Port" : [22] }],
+ "Node": [
+ {
+ "Name": aclient_name,
+ "Type": ["Honeypot", "Connection", "Auth"],
+ "SW": ["Cowrie"],
+ }
+ ],
+ "Attach": [{ "sessionid": sessionid, "username": username, "password": password }]
+ }
+ # Test if we're testing
+ if atest == "true":
+ event["Category"].append('Test')
+
+ event = idea_fill_addresses(event, src_ip, dst_ip, aanonymised, aanonymised_net)
+
+ return event
+
+
+def gen_event_idea_cowrie_ttylog(detect_time, src_ip, dst_ip, sessionid, ttylog, iinput):
+
+ event = {
+ "Format": "IDEA0",
+ "ID": str(uuid4()),
+ "DetectTime": detect_time,
+ "Category": ["Information.UnauthorizedAccess"],
+ "Note": "Cowrie ttylog",
+ "ConnCount": 1,
+ "Source": [{}],
+ "Target": [{ "Proto": ["tcp", "ssh"], "Port" : [22] }],
+ "Node": [
+ {
+ "Name": aclient_name,
+ "Type": ["Honeypot", "Data"],
+ "SW": ["Cowrie"],
+ }
+ ],
+ "Attach": [ { "sessionid": sessionid, "ttylog": ttylog, "iinput": iinput, "smart": iinput } ]
+ }
+ # Test if we're testing
+ if atest == "true":
+ event["Category"].append('Test')
+
+ event = idea_fill_addresses(event, src_ip, dst_ip, aanonymised, aanonymised_net)
+
+ return event
+
+
+def gen_event_idea_cowrie_download(detect_time, src_ip, dst_ip, sessionid, url, outfile):
+
+ event = {
+ "Format": "IDEA0",
+ "ID": str(uuid4()),
+ "DetectTime": detect_time,
+ "Category": ["Malware"],
+ "Note": "Cowrie download",
+ "ConnCount": 1,
+ "Source": [{}],
+ "Target": [{ "Proto": ["tcp", "ssh"], "Port" : [22]}],
+ "Node": [
+ {
+ "Name": aclient_name,
+ "Type": ["Honeypot", "Data"],
+ "SW": ["Cowrie"],
+ }
+ ],
+ "Attach": [{ "sessionid": sessionid, "url": url, "outfile": outfile, "smart": url }]
+ }
+ # Test if we're testing
+ if atest == "true":
+ event["Category"].append('Test')
+
+ event = idea_fill_addresses(event, src_ip, dst_ip, aanonymised, aanonymised_net)
+
+ return event
+
+
+def get_iinput(sessionid):
+ ret = []
+ query = "SELECT GROUP_CONCAT(input SEPARATOR '--SEP--') as i FROM input WHERE session=%s GROUP BY session;"
+ crs.execute(query, (sessionid,))
+ rows = crs.fetchall()
+ for row in rows:
+ ret.append(row["i"])
+ return ''.join(ret)
+
+
+def get_ttylog(sessionid):
+ ret = ""
+ query = "SELECT id, session, ttylog FROM ttylog WHERE session=%s;"
+ crs.execute(query, (sessionid,))
+ rows = crs.fetchall()
+ for row in rows:
+ try:
+ tf = tempfile.NamedTemporaryFile(delete=False)
+ with open(tf.name, 'w') as f:
+ f.write(row['ttylog'])
+ ret = subprocess.check_output(["/opt/cowrie/bin/playlog", "-m0", tf.name])
+ finally:
+ os.remove(tf.name)
+
+ #try to dumpit to json to see if there are some binary input and perhaps wrap it to base64
+ try:
+ a = json.dumps(ret)
+ except UnicodeDecodeError as e:
+ wclient.logger.warning("wraping binary content")
+ ret = base64.b64encode(ret)
+
+ return ret
+
+
+
+con = my.connect( host=aconfig['dbhost'], user=aconfig['dbuser'], passwd=aconfig['dbpass'],
+ db=aconfig['dbname'], port=aconfig['dbport'], cursorclass=mycursors.DictCursor)
+crs = con.cursor()
+events = []
+
+#kippo vs cowrie
+#cowrie/core/dblog.py: def nowUnix(self):
+#cowrie/core/dblog.py- """return the current UTC time as an UNIX timestamp"""
+#cowrie/core/dblog.py- return int(time.time())
+#kippo/core/dblog.py: def nowUnix(self):
+#kippo/core/dblog.py- """return the current UTC time as an UNIX timestamp"""
+#kippo/core/dblog.py- return int(time.mktime(time.gmtime()[:-1] + (-1,)))
+# k sozalenju
+# >>> int(time.mktime(time.gmtime()[:-1] + (-1,)))-int(time.time()) != 0
+
+
+#old senders data
+query = "SELECT UNIX_TIMESTAMP(s.starttime) as starttime, s.ip, COUNT(s.id) as attack_scale, sn.ip as sensor \
+ FROM sessions s \
+ LEFT JOIN sensors sn ON s.sensor=sn.id \
+ WHERE s.starttime > DATE_SUB(UTC_TIMESTAMP(), INTERVAL + %s SECOND) \
+ GROUP BY s.ip ORDER BY s.starttime ASC;"
+
+crs.execute(query, (awin,))
+etime = format_timestamp(time())
+stime = format_timestamp(time() - awin)
+rows = crs.fetchall()
+for row in rows:
+ a = gen_event_idea_cowrie_info(
+ detect_time = format_timestamp(row['starttime']),
+ src_ip = row['ip'],
+ dst_ip = row['sensor'],
+
+ win_start_time = stime,
+ win_end_time = etime,
+ aggr_win = awin,
+ conn_count = row['attack_scale']
+ )
+ events.append(a)
+
+
+
+#success login
+query = "SELECT UNIX_TIMESTAMP(a.timestamp) as timestamp, s.ip as sourceip, sn.ip as sensor, a.session as sessionid, a.username as username, a.password as password \
+ FROM auth a JOIN sessions s ON s.id=a.session JOIN sensors sn ON s.sensor=sn.id \
+ WHERE a.success=1 AND a.timestamp > DATE_SUB(UTC_TIMESTAMP(), INTERVAL + %s SECOND) \
+ ORDER BY a.timestamp ASC;"
+
+crs.execute(query, (awin,))
+rows = crs.fetchall()
+for row in rows:
+ a = gen_event_idea_cowrie_auth(
+ detect_time = format_timestamp(row['timestamp']),
+ src_ip = row['sourceip'],
+ dst_ip = row['sensor'],
+
+ username = row['username'],
+ password = row['password'],
+ sessionid = row['sessionid']
+ )
+ events.append(a)
+
+#ttylog+iinput reporter
+query = "SELECT UNIX_TIMESTAMP(s.starttime) as starttime, s.ip as sourceip, sn.ip as sensor, t.session as sessionid \
+ FROM ttylog t JOIN sessions s ON s.id=t.session JOIN sensors sn ON s.sensor=sn.id \
+ WHERE s.starttime > DATE_SUB(UTC_TIMESTAMP(), INTERVAL + %s SECOND) \
+ ORDER BY s.starttime ASC;"
+
+crs.execute(query, (awin,))
+rows = crs.fetchall()
+for row in rows:
+ a = gen_event_idea_cowrie_ttylog(
+ detect_time = format_timestamp(row['starttime']),
+ src_ip = row['sourceip'],
+ dst_ip = row['sensor'],
+
+ sessionid = row['sessionid'],
+ ttylog = get_ttylog(row['sessionid']),
+ iinput = get_iinput(row['sessionid'])
+ )
+ events.append(a)
+
+
+#download
+query = "SELECT UNIX_TIMESTAMP(s.starttime) as starttime, s.ip as sourceip, sn.ip as sensor, d.session as sessionid, d.url as url, d.outfile as ofile \
+ FROM downloads d JOIN sessions s ON s.id=d.session JOIN sensors sn ON s.sensor=sn.id \
+ WHERE s.starttime > DATE_SUB(UTC_TIMESTAMP(), INTERVAL + %s SECOND) \
+ ORDER BY s.starttime ASC;"
+
+crs.execute(query, (awin,))
+rows = crs.fetchall()
+for row in rows:
+ a = gen_event_idea_cowrie_download(
+ detect_time = format_timestamp(row['starttime']),
+ src_ip = row['sourceip'],
+ dst_ip = row['sensor'],
+
+ sessionid = row['sessionid'],
+ url = row['url'],
+ outfile = row['ofile']
+ )
+ events.append(a)
+
+
+print "=== Sending ==="
+start = time()
+ret = wclient.sendEvents(events)
+
+if 'saved' in ret:
+ wclient.logger.info("%d event(s) successfully delivered." % ret['saved'])
+
+print "Time: %f" % (time() - start)
+
+
diff --git a/dionaea/warden3-dio-sender.py b/dionaea/warden3-dio-sender.py
new file mode 100755
index 0000000000000000000000000000000000000000..b992ad495c2bad5a742968758b206231799c017f
--- /dev/null
+++ b/dionaea/warden3-dio-sender.py
@@ -0,0 +1,246 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2018 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+from warden_client import Client, Error, read_cfg, format_timestamp
+import json
+import string
+import urllib
+from time import time, gmtime, strftime, sleep
+from math import trunc
+from uuid import uuid4
+from os import path, remove
+import base64
+import sqlite3
+import sys
+import re
+
+aconfig = read_cfg('warden_client_dio.cfg')
+wconfig = read_cfg('warden_client.cfg')
+aclient_name = aconfig['name']
+wconfig['name'] = aclient_name
+aanonymised = aconfig['anonymised']
+aanonymised_net = aconfig['target_net']
+aanonymised = aanonymised if (aanonymised_net != '0.0.0.0/0') or (aanonymised_net == 'omit') else '0.0.0.0/0'
+atest = aconfig['test_mode']
+
+awin = aconfig['awin'] * 60
+abinpath = aconfig['binaries_path']
+adbfile = aconfig['dbfile']
+aconattempts = aconfig['con_attempts']
+aretryinterval = aconfig['con_retry_interval']
+areportbinaries = aconfig['report_binaries']
+apurgebinaries = aconfig['purge_binaries']
+
+wconfig['secret'] = aconfig.get('secret', '')
+wclient = Client(**wconfig)
+
+def idea_fill_addresses(event, source_ip, destination_ip, anonymised, anonymised_net):
+ af = "IP4" if not ':' in source_ip else "IP6"
+ event['Source'][0][af] = [source_ip]
+ if anonymised != 'omit':
+ if anonymised == 'yes':
+ event['Target'][0]['Anonymised'] = True
+ event['Target'][0][af] = [anonymised_net]
+ else:
+ event['Target'][0][af] = [destination_ip]
+
+ return event
+
+def gen_attach_idea_smb(logger, report_binaries, purge_binaries, binaries_path, filename, hashtype, hashdigest, vtpermalink, avref):
+
+ refs = []
+ attach = {
+ "Handle": 'att1',
+ "FileName": [filename],
+ "Type": ["Malware"],
+ "Hash": ["%s:%s" % (hashtype, hashdigest)],
+ }
+
+ if vtpermalink is not None:
+ refs.append('url:' + vtpermalink)
+
+ if avref is not None:
+ refs.extend(avref.split(';'))
+
+ if refs:
+ refs = [urllib.quote(ref, safe=':') for ref in refs]
+ refs = list(set(refs))
+ attach['Ref'] = refs
+
+ if report_binaries == 'true':
+ try:
+ fpath = path.join(binaries_path, hashdigest)
+ with open(fpath, "r") as f:
+ fdata = f.read()
+ attach['ContentType'] = 'application/octet-stream'
+ attach['ContentEncoding'] = 'base64'
+ attach['Size'] = len(fdata)
+ attach['Content'] = base64.b64encode(fdata)
+ except (IOError) as e:
+ logger.info("Reading id file \"%s\" with malware failed, information will not be attached." % (fpath))
+ if purge_binaries == 'true':
+ try:
+ remove(filename)
+ except OSError:
+ pass
+
+ return attach
+
+def gen_attach_idea_db(logger, data):
+
+ attach = {}
+ attach["Handle"] = 'att1'
+ attach["Type"] = ["Malware"]
+ attach['ContentType'] = 'application/octet-stream'
+ attach['ContentEncoding'] = 'base64'
+ attach['Size'] = len(data)
+ attach['Content'] = base64.b64encode(data)
+
+ return attach
+
+def gen_event_idea_dio(logger, binaries_path, report_binaries, purge_binaries, client_name, anonymised, target_net, detect_time, win_start_time, win_end_time, aggr_win, data):
+
+ category = []
+ event = {
+ "Format": "IDEA0",
+ "ID": str(uuid4()),
+ "DetectTime": detect_time,
+ "WinStartTime": win_start_time,
+ "WinEndTime": win_end_time,
+ "ConnCount": data['attack_scale'],
+ "Source": [{}],
+ "Target": [{}],
+ "Node": [
+ {
+ "Name": client_name,
+ "Type": ["Connection", "Protocol", "Honeypot"],
+ "SW": ["Dionaea"],
+ "AggrWin": strftime("%H:%M:%S", gmtime(aggr_win))
+ }
+ ]
+ }
+
+ # Save TCP/UDP proto
+ proto = [data['proto']]
+
+ # smbd allows save malware
+ if data['service'] == 'smbd' and data['download_md5_hash'] is not None:
+ category.append('Attempt.Exploit')
+ category.append('Malware')
+ proto.append('smb')
+
+ if data['download_url'] != '':
+ event['Source'][0]['URL'] = [data['download_url']]
+ filename = data['download_url'].split('/')[-1]
+
+ if filename != '' and data['download_md5_hash'] != '':
+ # Generate "SMB Attach" part of IDEA
+ a = gen_attach_idea_smb(logger, report_binaries, binaries_path, filename, "md5", data['download_md5_hash'], data['virustotal_permalink'], data['scan_result'])
+
+ event['Source'][0]['AttachHand'] = ['att1']
+ event['Attach'] = [a]
+
+ if data['service'] == 'mysqld':
+ #Clean exported data
+ if data['mysql_query'] is not None:
+ mysql_data = re.sub("select @@version_comment limit 1,?", "", data['mysql_query'])
+ if mysql_data != "":
+ # Generate "MySQL Attach" part of IDEA
+ a = gen_attach_idea_db(logger, mysql_data)
+
+ category.append('Attempt.Exploit')
+ proto.append('mysql')
+ event['Source'][0]['AttachHand'] = ['att1']
+ event['Attach'] = [a]
+
+ if data['service'] == 'mssqld':
+ #Clean exported data
+ if data['mssql_query'] is not None:
+ mssql_data = data['mssql_query']
+ if mssql_data != "":
+ # Generate "MSSQL Attach" part of IDEA
+ a = gen_attach_idea_db(logger, mssql_data)
+
+ category.append('Attempt.Exploit')
+ proto.append('mssql')
+ event['Source'][0]['AttachHand'] = ['att1']
+ event['Attach'] = [a]
+
+ event['Source'][0]['Port'] = [data['src_port']]
+ event['Target'][0]['Port'] = [data['dst_port']]
+ event['Source'][0]['Proto'] = proto
+ event['Target'][0]['Proto'] = proto
+
+ idea_fill_addresses(event, data['src_ip'], data['dst_ip'], aanonymised, aanonymised_net)
+
+ # Add default category
+ if not category:
+ category.append('Recon.Scanning')
+
+ # Test if we're testing
+ if atest == "true":
+ category.append('Test')
+
+ event['Category'] = category
+
+ return event
+
+def main():
+
+ con = sqlite3.connect(adbfile)
+ con.row_factory = sqlite3.Row
+ con.text_factory = str
+ crs = con.cursor()
+
+ events = []
+
+ query = "SELECT c.connection_timestamp AS timestamp, c.remote_host AS src_ip, c.remote_port AS src_port, c.connection_transport AS proto, \
+ c.local_host AS dst_ip, c.local_port AS dst_port, COUNT(c.connection) as attack_scale, c.connection_protocol AS service, d.download_url, d.download_md5_hash, \
+ v.virustotal_permalink, GROUP_CONCAT('urn:' || vt.virustotalscan_scanner || ':' || vt.virustotalscan_result,';') AS scan_result, \
+ group_concat(mca.mysql_command_arg_data) as mysql_query, \
+ group_concat(msc.mssql_command_cmd) as mssql_query \
+ FROM connections AS c LEFT JOIN downloads AS d ON c.connection = d.connection \
+ LEFT JOIN virustotals AS v ON d.download_md5_hash = v.virustotal_md5_hash \
+ LEFT JOIN virustotalscans vt ON v.virustotal = vt.virustotal \
+ LEFT JOIN mysql_commands mc ON c.connection = mc.connection \
+ LEFT JOIN mysql_command_args mca ON mc.mysql_command = mca.mysql_command \
+ LEFT JOIN mssql_commands msc ON c.connection = msc.connection \
+ WHERE datetime(connection_timestamp,'unixepoch') > datetime('now','-%d seconds') AND c.remote_host != '' \
+ GROUP BY c.remote_host, c.local_port ORDER BY c.connection_timestamp ASC;" % (awin)
+
+ attempts = 0
+ while attempts < aconattempts:
+ try:
+ crs.execute(query)
+ break
+ except sqlite3.Error, e:
+ attempts += 1
+ wclient.logger.info("Info: %s - attempt %d/%d." % (e.args[0], attempts, aconattempts))
+ if attempts == aconattempts:
+ wclient.logger.error("Error: %s (dbfile: %s)" % (e.args[0], adbfile))
+
+ sleep(aretryinterval)
+
+ rows = crs.fetchall()
+
+ if con:
+ con.close
+
+ etime = format_timestamp(time())
+ stime = format_timestamp(time() - awin)
+
+ for row in rows:
+ dtime = format_timestamp(row['timestamp'])
+ events.append(gen_event_idea_dio(logger = wclient.logger, binaries_path = abinpath, report_binaries = areportbinaries, purge_binaries = apurgebinaries, client_name = aclient_name, anonymised = aanonymised, target_net = aanonymised_net, detect_time = dtime, win_start_time = stime, win_end_time = etime, aggr_win = awin, data = row))
+
+ start = time()
+ ret = wclient.sendEvents(events)
+ if 'saved' in ret:
+ wclient.logger.info("%d event(s) successfully delivered in %d seconds" % (ret['saved'], (time() - start)))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/dionaea/warden_client-dio.cfg b/dionaea/warden_client-dio.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..537d96ab12e92c58163b9c2c1a34e8962bc6f714
--- /dev/null
+++ b/dionaea/warden_client-dio.cfg
@@ -0,0 +1,15 @@
+{
+ "warden": "warden_client.cfg",
+ "name": "cz.cesnet.server.dionaea",
+ "secret": "",
+
+ "anonymised": "no",
+ "target_net": "195.113.0.0/16",
+
+ "dbfile": "/opt/dionaea/var/dionaea/logsql.sqlite",
+ "binaries_path" : "/opt/dionaea/var/dionaea/binaries",
+ "report_binaries" : "true",
+ "con_attempts" : 3,
+ "con_retry_interval" : 5,
+ "awin": 5
+}
diff --git a/shodan/README.txt b/shodan/README.txt
new file mode 100644
index 0000000000000000000000000000000000000000..247893ca2e1aaab9a7836254faafe5eddb584700
--- /dev/null
+++ b/shodan/README.txt
@@ -0,0 +1,63 @@
+=========================
+Shodan2warden connector
+=========================
+
+This connector uses the internet-wide scanning service Shodan to search for
+potentially vulnerable open services in given network (ASN) and reports them to
+Warden (via Warden filer daemon).
+
+The script sends a set of preconfigured queries to Shodan API to search for all
+matching IPs in given ASN. The queries correspond to various potentially
+vulnerable hosts or other problems with open services.
+
+Currently implemented queries:
+
+- Publicly accessible IPMI protocol
+- Publicly accessible SCADA (BACnet) system
+- Publicly accessible printer via incesure PJL protocol
+- Publicly accessible printer via IPP protocol
+- Open MongoDB database
+- Open Elastic database
+- Open anonymous FTP
+- Web page or other service with a "hacked by" message
+- Web running on an old (unsupported) PHP version
+
+Important: These rules may need to be changed according to needs, interests and
+security policies of each organization.
+
+For each host in given ASN found to be matching a query, a corresponding IDEA
+message is created and put to an output directory. Messages from this directory
+should be sent to Warden by a Warden filer daemon.
+
+The script is assumed to be run periodically (e.g. once a day) by cron.
+
+
+-------------------------
+Installation:
+
+1. create a directory for the script and resulting IDEA files, e.g.
+ `/data/shodan2warden/` and `/data/shodan2warden/warden_sender`
+2. put the shodan2warden.py script into the first directory
+3. set up warden_filer daemon to read IDEA messages from
+ `/data/shodan2warden/warden_sender` and send them to Warden
+ (see README of Warden filer)
+4. set up cron to run the scirpt every day (use shodan2warden.cron as an
+ example)
+
+
+-------------------------
+Configuration:
+
+The script takes the following arguments:
+
+ -h, --help show help message and exit
+ -k APIKEY, --apikey APIKEY
+ Shodan API key
+ -a ASN, --asn ASN ASN to query
+ -n NODE, --node NODE Node name to fill into IDEA messages
+ -d PATH, --destdir PATH
+ Path to destination directory (with 'incoming' and
+ 'temp' subdirectories) (default: CWD)
+ -t, --test Add 'Test' category to IDEA messages.
+ -v, --verbose Print information about progress and results
+
diff --git a/shodan/shodan2warden.cron b/shodan/shodan2warden.cron
new file mode 100644
index 0000000000000000000000000000000000000000..ea09bb7e2b21e1b6ec94404c1df048c3cec24b9c
--- /dev/null
+++ b/shodan/shodan2warden.cron
@@ -0,0 +1,2 @@
+# Run every day at 9:00
+0 9 * * * shodan2warden python3 /data/shodan2warden/shodan2warden.py -k $(cat /data/shodan2warden/shodan_key) -a 2852 -n cz.cesnet.ext.shodan -d /data/shodan2warden/warden_sender --test -v >>/data/shodan2warden/shodan2warden.log 2>&1
\ No newline at end of file
diff --git a/shodan/shodan2warden.py b/shodan/shodan2warden.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8af304ee386dbad3c54c24ba5d58e04ff821dab
--- /dev/null
+++ b/shodan/shodan2warden.py
@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Author: Pavla Hlučková
+# Václav Bartoš <bartos@cesnet.cz>
+
+from shodan import Shodan
+from datetime import datetime
+from uuid import uuid4
+import json
+import os
+import argparse
+
+# Global variables
+VERBOSE = False
+test_category = False
+node_name = 'undefined'
+
+def vprint(*args, **kwargs):
+ # Verbose print
+ if VERBOSE:
+ print("[{}] ".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")), end="")
+ print(*args, **kwargs)
+
+
+def generateIdeaEvent(detect_time, category, description,ip_string,port_num,proto,content_type,content,note):
+ create_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+ # if there's no timezone in detect_time, assume UTC (which Shodan normally uses) and append 'Z'
+ if 'Z' not in detect_time and '+' not in detect_time and '-' not in detect_time:
+ detect_time += 'Z'
+ event = {
+ "Format": "IDEA0",
+ "ID": str(uuid4()),
+ "Category": [category],
+ "CreateTime": create_time,
+ "DetectTime": detect_time,
+ "Description": description,
+ "Ref": ["https://www.shodan.io/host/" + ip_string],
+ "Source": [
+ {
+ "IP4": [ip_string],
+ "Port": [port_num],
+ "Proto": proto
+ }
+ ],
+ "Node": [
+ {
+ "Name": node_name,
+ "SW": ["shodan2warden"],
+ "Type": ["External", "Recon"]
+ }
+ ],
+ "Attach": [
+ {
+ "ContentType": content_type,
+ "Content": content,
+ "Note": note
+ }
+ ]
+ }
+
+ if test_category:
+ event["Category"].append("Test")
+
+ filename = "{}_{}_{}.json".format(
+ datetime.utcnow().strftime('%Y%m%d%H%M%S%f'),
+ ip_string,
+ event['ID'][:8])
+ tmp_destination = os.path.join(default_directory, 'tmp', filename)
+ inc_destination = os.path.join(default_directory, 'incoming', filename)
+
+ with open(tmp_destination, 'w') as json_file:
+ json.dump(event, json_file)
+
+ os.rename(tmp_destination, inc_destination)
+
+
+def IPMI():
+ query = asnString+" port:623"
+ category = "Vulnerable.Config"
+ description = "Publicly accessible insecure protocol: IPMI"
+ proto = ["udp", "ipmi"]
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for IMPI:", query)
+ for banner in api.search_cursor(query):
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+def SCADA():
+ query = asnString+" port:47808"
+ category = "Vulnerable.Config"
+ description = "Publicly accessible SCADA (BACnet) system"
+ proto = ["udp", "bacnet"]
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for SCADA:", query)
+ for banner in api.search_cursor(query):
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+def printerPJL():
+ query = asnString + " port:9100 PJL INFO STATUS"
+ category = "Vulnerable.Config"
+ description = "Vulnerable PJL printer"
+ proto = []
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for printer PLJ:", query)
+ for banner in api.search_cursor(query):
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+def printerIPP():
+ query = asnString+" port:631"
+ category = "Vulnerable.Config"
+ description = "Potentially vulnerable IPP printer"
+ proto = ["ipp"]
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for printer IPP:", query)
+ for banner in api.search_cursor(query):
+ if "close" in banner.get('data'):
+ continue
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+def mongoDB():
+ query = asnString+" \"mongodb metrics\""
+ category = "Vulnerable.Config"
+ description = "Potentially vulnerable MongoDB database"
+ proto = ["tcp"]
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for MongoDB:", query)
+ for banner in api.search_cursor(query):
+ if "\"totalSize\": 0.0" in banner.get('data'):
+ continue # skip if database is empty
+ if "hacked" in banner.get('data'):
+ category = "Information.UnauthorizedModification"
+ description = "Potentially exploited mongoDB database"
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+def elasticIndices():
+ query = asnString+" \"Elastic Indices\""
+ category = "Vulnerable.Config"
+ description = "Possibly vulnerable data displayed - Elastic Indices"
+ proto = ["tcp"]
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for Elastic Indices:", query)
+ for banner in api.search_cursor(query):
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+def anonFTP():
+ query = asnString+" port:21 \"anonymous logged in\""
+ category = "Vulnerable.Config"
+ description = "Open anonymous FTP"
+ proto = ["tcp", "ftp"]
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for anonymous FTP:", query)
+ for banner in api.search_cursor(query):
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+def hacked():
+ query = asnString+" \"hacked\""
+ category = "Information.UnauthorizedModification"
+ description = "Service probably hacked (\"hacked\" string found in banner)"
+ proto = []
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for \"hacked\" string:", query)
+ for banner in api.search_cursor(query):
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+def unsupportedPHP():
+ query = asnString+" PHP"
+ category = "Vulnerable.Open"
+ description = "Web running on old (unsupported) PHP version"
+ proto = ["tcp", "http"]
+ content_type = "text/plain"
+ note = "Original service banner from Shodan"
+ vprint("Querying for unsupported PHP:", query)
+ for banner in api.search_cursor(query):
+ if "test page" in banner.get('data'):
+ continue
+ if "PHP/5" in banner.get('data') or "PHP/4" in banner.get('data'):
+ vprint("Found problematic IP:", banner.get('ip_str'))
+ generateIdeaEvent(banner.get('timestamp'), category, description,
+ banner.get('ip_str'), banner.get('port'),
+ proto, content_type, banner.get('data'), note)
+
+
+def parse_args():
+ # command line argument parser
+ parser = argparse.ArgumentParser(description="Searches Shodan for potential problems with open services in given ASN. For each such problem generates an IDEA message into gievn directory (to be sent to Warden by warden_filer). This script is assumed to be run daily by cron.")
+ parser.add_argument('-k', '--apikey', required=True,
+ help="Shodan API key")
+ parser.add_argument('-a', '--asn', type=int, required=True,
+ help="ASN to query")
+ parser.add_argument('-n', '--node', required=True,
+ help="Node name to fill into IDEA messages")
+ parser.add_argument('-d', '--destdir', dest="path", default=os.getcwd(),
+ help="Path to destination directory (with 'incoming' and 'temp' subdirectories) (default: CWD)")
+ parser.add_argument('-t', '--test', action="store_true",
+ help="Add 'Test' category to IDEA messages.")
+ parser.add_argument('-v', '--verbose', action="store_true",
+ help="Print information about progress and results")
+ return parser.parse_args()
+
+def main():
+ IPMI()
+ SCADA()
+ printerPJL()
+ printerIPP()
+ mongoDB()
+ elasticIndices()
+ anonFTP()
+ hacked()
+ unsupportedPHP()
+
+if __name__ == "__main__":
+ #getting arguments from argparse
+ args = parse_args()
+ VERBOSE = args.verbose
+ default_directory = args.path
+ node_name = args.node
+ test_category = args.test
+ asnString = "asn:as" + str(args.asn)
+
+ api = Shodan(args.apikey)
+
+ # incoming directory creation
+ directory = "incoming"
+ path = os.path.join(default_directory, directory)
+ os.makedirs(path, exist_ok=True)
+
+ # tmp directory creation
+ directory = "tmp"
+ path = os.path.join(default_directory, directory)
+ os.makedirs(path, exist_ok=True)
+
+ main()
diff --git a/tippingpoint/tpToIdea.py b/tippingpoint/tpToIdea.py
new file mode 100644
index 0000000000000000000000000000000000000000..52b9bacb46e477a368945a52650772d2d15bc961
--- /dev/null
+++ b/tippingpoint/tpToIdea.py
@@ -0,0 +1,642 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2018 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+import json
+from uuid import uuid4
+import re
+import socket
+import optparse
+import sys
+import os
+import signal
+import resource
+import os.path as pth
+import atexit
+import time
+from datetime import datetime
+import logging
+
+
+class FileWatcher(object):
+
+ def __init__(self, filename, tail=True):
+ self.filename = filename
+ self.open()
+ self.line_buffer = ""
+ if tail and self.f:
+ self.f.seek(0, os.SEEK_END)
+
+ def open(self):
+ try:
+ self.f = open(self.filename, "r")
+ st = os.fstat(self.f.fileno())
+ self.inode, self.size = st.st_ino, st.st_size
+ except IOError:
+ self.f = None
+ self.inode = -1
+ self.size = -1
+
+ def _check_reopen(self):
+ try:
+ st = os.stat(self.filename)
+ cur_inode, cur_size = st.st_ino, st.st_size
+ except OSError as e:
+ cur_inode = -1
+ cur_size = -1
+ if cur_inode != self.inode or cur_size < self.size:
+ self.close()
+ self.open()
+
+ def readline(self):
+ if not self.f:
+ self.open()
+ if not self.f:
+ return self.line_buffer
+ res = self.f.readline()
+ if not res:
+ self._check_reopen()
+ if not self.f:
+ return self.line_buffer
+ res = self.f.readline()
+ if not res.endswith("\n"):
+ self.line_buffer += res
+ else:
+ res = self.line_buffer + res
+ self.line_buffer = ""
+ return res
+
+ def close(self):
+ try:
+ if self.f:
+ self.f.close()
+ except IOError:
+ pass
+ self.inode = -1
+ self.size = -1
+
+ def __repr__(self):
+ return '%s("%s")' % (type(self).__name__, self.filename)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+ return False
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ return self.readline()
+
+
+class Filer(object):
+
+ def __init__(self, directory):
+ self.basedir = self._ensure_path(directory)
+ self.tmp = self._ensure_path(pth.join(self.basedir, "tmp"))
+ self.incoming = self._ensure_path(pth.join(self.basedir, "incoming"))
+ self.hostname = socket.gethostname()
+ self.pid = os.getpid()
+
+ def _ensure_path(self, p):
+ try:
+ os.mkdir(p)
+ except OSError:
+ if not pth.isdir(p):
+ raise
+ return p
+
+ def _get_new_name(self, fd=None):
+ (inode, device) = os.fstat(fd)[1:3] if fd else (0, 0)
+ return "%s.%d.%f.%d.%d" % (
+ self.hostname, self.pid, time.time(), device, inode)
+
+ def create_unique_file(self):
+ # First find and open name unique within tmp
+ tmpname = None
+ while not tmpname:
+ tmpname = self._get_new_name()
+ try:
+ fd = os.open(pth.join(self.tmp, tmpname), os.O_CREAT | os.O_RDWR | os.O_EXCL)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise # other errors than duplicates should get noticed
+ tmpname = None
+ # Now we know the device/inode, rename to make unique within system
+ newname = self._get_new_name(fd)
+ os.rename(pth.join(self.tmp, tmpname), pth.join(self.tmp, newname))
+ nf = os.fdopen(fd, "w")
+ return nf, newname
+
+ def publish_file(self, short_name):
+ os.rename(pth.join(self.tmp, short_name), pth.join(self.incoming, short_name))
+
+
+class IdeaGen(object):
+
+ tp_tax_to_idea = {
+ 1: {
+ 1: ["Attempt.Exploit"],
+ 2: ["Attempt.Exploit"],
+ 3: ["Attempt.Exploit"],
+ 4: ["Attempt.Exploit"],
+ 5: ["Attempt.Exploit"],
+ 6: ["Attempt.Exploit"],
+ 255: ["Attempt.Exploit"]
+ },
+
+ 2: {
+ 1: ["Malware.Worm"],
+ 2: ["Malware.Virus"],
+ 3: ["Malware.Trojan"],
+ 4: ["Intrusion.Botnet"],
+ 5: ["Fraud.Phishing"],
+ 255: ["Malware"]
+ },
+
+ 3: {
+ 1: ["Availability.DDoS"],
+ 2: ["Availability.DDoS"],
+ 3: ["Availability.DDoS"],
+ 255: ["Availability.DDoS"]
+ },
+
+ 4: {
+ 1: ["Other"],
+ 2: ["Other"],
+ 3: ["Other"],
+ 4: ["Other"],
+ 5: ["Other"],
+ 6: ["Attempt.Login"],
+ 7: ["Malware.Spyware"],
+ 255: ["Other"]
+ },
+
+ 5: {
+ 1: ["Recon.Scanning"],
+ 2: ["Attempt.Exploit"],
+ 3: ["Attempt.Exploit"],
+ 4: ["Recon.Scanning", "Attempt.Exploit"],
+ 255: ["Attempt.Exploit"]
+ },
+
+ 6: {
+ 1: ["Anomaly.Protocol"],
+ 2: ["Anomaly.Traffic"],
+ 3: ["Anomaly.Application"],
+ 255: ["Anomaly"]
+ },
+
+ 7: {
+ 1: ["Anomaly.Traffic"],
+ 2: ["Anomaly.Application"],
+ 255: ["Anomaly.Traffic"]
+ },
+
+ 8: {
+ 1: ["Other"],
+ 2: ["Other"],
+ 255: ["Other"]
+ },
+ }
+
+ tp_prot_to_idea = {
+ 1: "appletalk",
+ 2: "auth",
+ 3: "bgp",
+ 4: "cdp",
+ 5: "clns",
+ 6: "dhcp",
+ 7: "domain",
+ 8: "finger",
+ 9: "ftp",
+ 10: "hsrp",
+ 11: "http",
+ 12: "icmp",
+ 13: "igmp",
+ 14: "eigrp",
+ 15: "ipv6",
+ 16: "ipx",
+ 17: "irc",
+ 18: "isis",
+ 19: "isakmp",
+ 20: "ldap",
+ 21: "mpls",
+ 22: "ms-rpc",
+ 23: "ms-sql",
+ 24: "nat",
+ 25: "netbios",
+ 26: "nntp",
+ 27: "ntp",
+ 28: "oracle",
+ 29: "ospf",
+ 30: "pop-imap",
+ 31: "sunrpc",
+ 32: "qos",
+ 33: "rip",
+ 34: "sunrpc",
+ 35: "smb",
+ 36: "smtp",
+ 37: "snmp",
+ 38: "sql",
+ 39: "ssh",
+ 40: "ssl-tls",
+ 41: "tacacs",
+ 42: "tcp",
+ 43: "telnet",
+ 45: "udp",
+ 46: "uucp",
+ 48: "x11",
+ 49: "tftp",
+ 50: "ip",
+ 51: "nfs",
+ 52: "wins",
+ 80: "h323",
+ 81: "megaco",
+ 82: "mgcp",
+ 83: "sip",
+ 84: "rtp",
+ 99: "voip",
+ 100: "aim",
+ 101: "msn",
+ 102: "yahoo",
+ 103: "icq",
+ 119: "im",
+ 120: "musicmatch",
+ 121: "winamp",
+ 122: "shoutcast",
+ 123: "windows",
+ 124: "quicktime",
+ 125: "rtsp",
+ 149: "streaming",
+ 150: "bittorrent",
+ 151: "manolito",
+ 152: "directconnect",
+ 153: "earthstation5",
+ 154: "p2p",
+ 155: "fasttrack",
+ 156: "gnutella",
+ 157: "twister",
+ 158: "winmx",
+ 180: "p2p",
+ 190: "scada-dnp3",
+ 191: "scada-iccp",
+ 192: "scada-iec",
+ 193: "scada-modbus",
+ 194: "scada-opc",
+ 199: "scada",
+ 254: "multi-protocol",
+ }
+
+ def __init__(self, name, test=False, other=False):
+ self.name = name
+ self.test = test
+ self.other = other
+
+ def convert_category_and_protocol(self, category, id_taxonomy):
+ '''
+ converts category from record to IDEA category
+ :param category: TippingPoint category description
+ :param id_taxonomy: TippingPoint taxonomy id
+ :return: if category or incident is empty or is not important for saving it return None, otherwise return
+ converted category
+ '''
+ if not (category and id_taxonomy):
+ return None
+ tp_cat_maj = id_taxonomy >> 24
+ tp_cat_min = id_taxonomy >> 16 & 0b11111111
+ tp_prot = id_taxonomy >> 8 & 0b11111111
+ tp_platf = id_taxonomy & 0b11111111
+ try:
+ category = IdeaGen.tp_tax_to_idea[tp_cat_maj][tp_cat_min]
+ except KeyError:
+ category = ["Other"]
+
+ try:
+ protocol = IdeaGen.tp_prot_to_idea[tp_prot]
+ except KeyError:
+ protocol = None
+
+ return { 'category': category, 'protocol': protocol }
+
+ def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
+ dest_ip, dest_port, conn_count, url, severity, orig_data):
+ '''
+ put every piece of record together into IDEA message
+ :return: new IDEA message
+ '''
+
+ if (category == ["Other"]) and not self.other:
+ return None
+
+ event = {
+ 'Format': "IDEA0",
+ 'ID': str(uuid4()),
+ 'DetectTime': datetime.utcfromtimestamp(timestamp / 1000).isoformat() + 'Z',
+ 'Category': category + (["Test"] if self.test else [])
+ }
+ if cve:
+ event['Ref'] = ['urn:cve:'.format(i) for i in cve]
+ if conn_count and int(conn_count):
+ event['ConnCount'] = int(conn_count)
+ source = {}
+ target = {}
+ if src_ip:
+ # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+ src_ip = src_ip.replace(" ", "")
+ af = "IP4" if not ':' in src_ip else "IP6"
+ source[af] = [src_ip]
+ if src_port and int(src_port):
+ source['Port'] = [int(src_port)]
+ if proto:
+ source['Proto'] = proto
+ if dest_ip and (dest_ip != "0.0.0.0"):
+ # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+ dest_ip = dest_ip.replace(" ", "")
+ af = "IP4" if not ':' in dest_ip else "IP6"
+ target[af] = [dest_ip]
+ if dest_port and int(dest_port):
+ target['Port'] = [int(dest_port)]
+ if proto:
+ target['Proto'] = proto
+ if url:
+ target['URL'] = url
+
+ if source:
+ event['Source'] = [source]
+ if target:
+ if category and category == "Reputation":
+ event['Source'].append(target)
+ else:
+ event['Target'] = [target]
+ if orig_data:
+ event['Attach'] = [{
+ 'Type': ["OrigData"],
+ 'Content': orig_data.strip()
+ }]
+ event['Node'] = [{
+ 'Name': self.name,
+ 'Type': ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"],
+ 'SW': ["TippingPoint_NX_NGIPS"]
+ }]
+ return event
+
+
+def daemonize(
+ work_dir=None, chroot_dir=None,
+ umask=None, uid=None, gid=None,
+ pidfile=None, files_preserve=[], signals={}):
+ # Dirs, limits, users
+ if chroot_dir is not None:
+ os.chdir(chroot_dir)
+ os.chroot(chroot_dir)
+ if umask is not None:
+ os.umask(umask)
+ if work_dir is not None:
+ os.chdir(work_dir)
+ if gid is not None:
+ os.setgid(gid)
+ if uid is not None:
+ os.setuid(uid)
+ # Doublefork, split session
+ if os.fork() > 0:
+ os._exit(0)
+ try:
+ os.setsid()
+ except OSError:
+ pass
+ if os.fork() > 0:
+ os._exit(0)
+ # Setup signal handlers
+ for (signum, handler) in signals.items():
+ signal.signal(signum, handler)
+ # Close descriptors
+ descr_preserve = set(f.fileno() for f in files_preserve)
+ maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+ if maxfd == resource.RLIM_INFINITY:
+ maxfd = 65535
+ for fd in range(maxfd, 3, -1): # 3 means omit stdin, stdout, stderr
+ if fd not in descr_preserve:
+ try:
+ os.close(fd)
+ except Exception:
+ pass
+ # Redirect stdin, stdout, stderr to /dev/null
+ devnull = os.open(os.devnull, os.O_RDWR)
+ for fd in range(3):
+ os.dup2(devnull, fd)
+ # PID file
+ if pidfile is not None:
+ pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
+ os.write(pidd, str(os.getpid())+"\n")
+ os.close(pidd)
+ # Define and setup atexit closure
+
+ @atexit.register
+ def unlink_pid():
+ try:
+ os.unlink(pidfile)
+ except Exception:
+ pass
+
+
+def get_args():
+ optp = optparse.OptionParser(
+ usage="usage: %prog [options] logfile ...",
+ description="Watch TippingPoint logfiles and generate Idea events into directory")
+ optp.add_option(
+ "-n", "--name",
+ default=None,
+ dest="name",
+ type="string",
+ action="store",
+ help="Warden client name")
+ optp.add_option(
+ "--test",
+ default=False,
+ dest="test",
+ action="store_true",
+ help="Add \"Test\" category")
+ optp.add_option(
+ "--other",
+ default=False,
+ dest="other",
+ action="store_true",
+ help="Send events having \"Other\" category (usually nonmalicious)")
+ optp.add_option(
+ "-o", "--oneshot",
+ default=False,
+ dest="oneshot",
+ action="store_true",
+ help="process files and quit (do not daemonize)")
+ optp.add_option(
+ "--poll",
+ default=1,
+ dest="poll",
+ type="int",
+ action="store",
+ help="log file polling interval")
+ optp.add_option(
+ "-d", "--dir",
+ default=None,
+ dest="dir",
+ type="string",
+ action="store",
+ help="Target directory (mandatory)")
+ optp.add_option(
+ "-p", "--pid",
+ default=pth.join("/var/run", pth.splitext(pth.basename(sys.argv[0]))[0] + ".pid"),
+ dest="pid",
+ type="string",
+ action="store",
+ help="create PID file with this name (default: %default)")
+ optp.add_option(
+ "-u", "--uid",
+ default=None,
+ dest="uid",
+ type="int",
+ action="store",
+ help="user id to run under")
+ optp.add_option(
+ "-g", "--gid",
+ default=None,
+ dest="gid",
+ type="int",
+ action="store",
+ help="group id to run under")
+ optp.add_option(
+ "--origdata",
+ default=False,
+ dest="origdata",
+ action="store_true",
+ help="Store original report to IDEA message")
+ return optp
+
+
+def not_empty(test_string):
+ # tests if string is not empty
+ return None if test_string.strip() in ["", "null"] else test_string
+
+
+def save_events(event, filer):
+ f, name = filer.create_unique_file()
+ with f:
+ f.write(json.dumps(event, ensure_ascii=True))
+ filer.publish_file(name)
+
+
+def process_data(line, filer, origdata, idea_gen):
+ '''
+ takes one record, parse it to parameters, give it to Ideagen and writes to file
+ :param line: one record
+ :param idea_file: where output goes
+ :param origdata: if true, write original record to IDEA message
+ '''
+
+ ll_prot_str = [" ip:", " ipv6:", " udp:", " tcp:", "(ip)", "(ipv6)", "(udp)", "(tcp)", " ip ", " ipv6 ", " udp ", " tcp "]
+
+ row = line.split("|")
+
+ catprot = idea_gen.convert_category_and_protocol(category=row[1], id_taxonomy=int(row[2]))
+ proto = []
+ if not_empty(row[5]).lower() in ['ip', 'ip6', 'udp', 'tcp']:
+ proto.append(row[5].lower())
+
+ flt_proto = [pstr for pstr in ll_prot_str if(pstr in not_empty(row[4]).lower())]
+ if flt_proto:
+ proto.append(flt_proto[0].translate(None, " ():v"))
+
+ if catprot['protocol']:
+ proto.append(catprot['protocol'])
+ if set(proto).intersection({"ip", "ip6"}):
+ try:
+ proto.remove("ip")
+ proto.remove("ip6")
+ except ValueError:
+ pass
+ if not "udp" in proto:
+ proto.append("tcp")
+ proto = list(set(proto))
+
+ timestamp = row[0].split(" ")[-1]
+ cve = [i for i in row[3].split(",") if i not in ("null", "")]
+ odata = "|".join([timestamp] + row[1:])
+ if catprot['category'] and not_empty(row[0][-14:-3]):
+ idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=catprot['category'], id_taxonomy=int(row[2]),
+ cve=cve, filter_name=not_empty(row[4]), proto=proto,
+ src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
+ conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
+ orig_data=odata if origdata else False)
+ if idea_event:
+ save_events(idea_event, filer)
+
+
+running_flag = True
+reload_flag = False
+
+
+def terminate_me(signum, frame):
+ global running_flag
+ running_flag = False
+
+
+def reload_me(signum, frame):
+ global reload_flag
+ reload_flag = True
+
+
+def main():
+ global running_flag
+ global reload_flag
+ logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG, filename='tipping_point_log.log', filemode='w')
+ optp = get_args()
+ opts, args = optp.parse_args()
+ if not args or opts.name is None or opts.dir is None:
+ optp.print_help()
+ sys.exit()
+ if opts.oneshot:
+ signal.signal(signal.SIGINT, terminate_me)
+ signal.signal(signal.SIGTERM, terminate_me)
+ files = [open(arg) for arg in args]
+ else:
+ daemonize(
+ pidfile=opts.pid,
+ uid=opts.uid,
+ gid=opts.gid,
+ signals={
+ signal.SIGINT: terminate_me,
+ signal.SIGTERM: terminate_me,
+ signal.SIGHUP: reload_me
+ })
+ files = [FileWatcher(arg) for arg in args]
+ filer = Filer(opts.dir)
+ idea_gen = IdeaGen(opts.name, opts.test, opts.other)
+ while running_flag:
+ for log_file in files:
+ while True:
+ line = log_file.readline()
+ if line is None or not line.strip():
+ logging.info("no line")
+ break
+ logging.info("readline")
+ process_data(line, filer, opts.origdata, idea_gen)
+ if not running_flag:
+ break
+ if reload_flag:
+ for f in files:
+ f.close()
+ f.open()
+ reload_flag = False
+ if opts.oneshot:
+ break
+ else:
+ time.sleep(opts.poll)
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file