diff --git a/censys/censys2warden.cron b/censys/censys2warden.cron
new file mode 100644
index 0000000000000000000000000000000000000000..df55515ba31be5928226212e370ce9d6f6e1dd18
--- /dev/null
+++ b/censys/censys2warden.cron
@@ -0,0 +1,2 @@
+# Run every day at 9:00
+0 9 * * * shodan2warden python3 /data/censys2warden/censys2warden.py $(cat /data/censys2warden/api_params) -a 2852 -n cz.cesnet.ext.censys -d /data/censys2warden/warden_sender --test -v >>/data/censys2warden/censys2warden.log 2>&1
\ No newline at end of file
diff --git a/censys/censys2warden.py b/censys/censys2warden.py
new file mode 100644
index 0000000000000000000000000000000000000000..cab2bc6c826950590ec4fe3e4a0c46693d12c456
--- /dev/null
+++ b/censys/censys2warden.py
@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Author: Pavla Hlu�kov�
+# V�clav Barto� <bartos@cesnet.cz>
+
+import censys.ipv4
+import json
+import os
+from datetime import datetime
+from uuid import uuid4
+import argparse
+
+def vprint(*args, **kwargs):
+ # Verbose print
+ if VERBOSE:
+ print("[{}] ".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")), end="")
+ print(*args, **kwargs)
+
+IPV4_FIELDS = ['ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
+
+MAX_RESULTS_PER_QUERY = 1000
+
+def generateIdeaEvent(detect_time, category, description,ip_string,ports,proto,content_type,content,note):
+ create_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+ # if there's no timezone in detect_time, assume UTC (which Shodan normally uses) and append 'Z'
+ if 'Z' not in detect_time and '+' not in detect_time and '-' not in detect_time:
+ detect_time += 'Z'
+ event = {
+ "Format": "IDEA0",
+ "ID": str(uuid4()),
+ "Category": [category],
+ "CreateTime": create_time,
+ "DetectTime": detect_time,
+ "Description": description,
+ "Ref": ["https://www.censys.io/ipv4/" + ip_string],
+ "Source": [
+ {
+ "IP4": [ip_string],
+ "Port": ports,
+ "Proto": proto
+ }
+ ],
+ "Node": [
+ {
+ "Name": node_name,
+ "SW": ["censys2warden"],
+ "Type": ["External", "Recon"]
+ }
+ ],
+ "Attach": [
+ {
+ "ContentType": content_type,
+ "Content": json.dumps(content) if content_type == 'application/json' else content,
+ "Note": note
+ }
+ ]
+ }
+
+ if test_category:
+ event["Category"].append("Test")
+
+ filename = "{}_{}_{}_{}.json".format(
+ datetime.utcnow().strftime('%Y%m%d%H%M%S%f'),
+ category.replace('.','').lower(),
+ ip_string,
+ event['ID'][:8])
+ tmp_destination = os.path.join(default_directory, 'tmp', filename)
+ inc_destination = os.path.join(default_directory, 'incoming', filename)
+
+ with open(tmp_destination, 'w') as json_file:
+ json.dump(event, json_file)
+
+ os.rename(tmp_destination, inc_destination)
+
+def IPMI():
+ query = asnString + "protocols: \"623/ipmi\" tags: ipmi"
+ category = "Vulnerable.Config"
+ description = "Publicly accessible insecure protocol: IPMI"
+ proto = ["udp", "ipmi"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for IMPI:", query)
+ for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[623],proto,content_type,banner,note)
+
+def SCADA():
+ query = asnString + "scada"
+ category = "Vulnerable.Config"
+ description = "Publicly accessible SCADA (BACnet) system"
+ proto = ["udp", "bacnet"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for SCADA:", query)
+ for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
+ # TODO: find out the port of the scada protocol(s)
+ # (sometimes there are multiple services running on the IP)
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),banner.get('ports'),proto,content_type,banner,note)
+
+def printerIPP():
+ query = asnString + "protocols: \"631/ipp\""
+ category = "Vulnerable.Config"
+ description = "Potentially vulnerable IPP printer"
+ proto = ["tcp", "ipp"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for printer IPP:", query)
+ for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[631],proto,content_type,banner,note)
+
+def mongoDB():
+ query = asnString + "protocols: \"27017/mongodb\""
+ category = "Vulnerable.Config"
+ description = "Potentially vulnerable MongoDB database"
+ proto = ["tcp"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for MongoDB:", query)
+ for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+ # TODO: try to find a field with DB size, but it seems there's none
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[27017],proto,content_type,banner,note)
+
+def PCA():
+ query = asnString + "tags: pca"
+ category = "Vulnerable.Config"
+ description = "An old unsupported service 'PCAnywhere' open to internet"
+ proto = ["tcp","pca"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for PCAnywhere:", query)
+ for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+def elasticSearch():
+ query = asnString + "protocols: \"9200/elasticsearch\""
+ category = "Vulnerable.Config"
+ description = "Possibly vulnerable data displayed - Elastic Search"
+ proto = ["tcp"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for Elastic Indices:", query)
+ for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+
+def hacked():
+ IPV4_FIELDS_HACKED = ['80.http.get.body','443.https.get.body','ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
+ query = asnString + "\"hacked\""
+ category = "Information.UnauthorizedModification"
+ description = "Service probably hacked (\"hacked\" string found in banner)"
+ proto = []
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for \"hacked\" string:", query)
+ for banner in c.search(query, IPV4_FIELDS_HACKED,max_records=MAX_RESULTS_PER_QUERY):
+ if "Test Page" in str(banner.get('80.http.get.body')) or "Test Page" in str(banner.get('443.https.get.body')):
+ continue
+ if "Cyber Security" in str(banner.get('80.http.get.body')) or "Cyber Security" in str(banner.get('443.https.get.body')):
+ # this was added to filter a false positive on a cybersecurity page mentioning hacking, see http://158.196.109.174/
+ continue
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+def unsupportedPHP():
+ query = asnString + "(80.http.get.headers.x_powered_by: PHP\\/5.* OR 8080.http.get.headers.x_powered_by: PHP\\/5.* OR 443.https.get.headers.x_powered_by: PHP\\/5.*)"
+ category = "Vulnerable.Open"
+ description = "Web running on old (unsupported) PHP version"
+ proto = ["tcp", "http"]
+ content_type = "application/json"
+ note = "Original Censys data (subset)"
+ vprint("Querying for unsupported PHP:", query)
+ for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+ vprint("Found problematic IP:", banner.get('ip'))
+ generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+
+def parse_args():
+ # command line argument parser
+ parser = argparse.ArgumentParser(
+ description="Searches Censys for potential problems with open services in given ASN. For each such problem generates an IDEA message into gievn directory (to be sent to Warden by warden_filer). This script is assumed to be run daily by cron.")
+ parser.add_argument('-i', '--apiid', required=True,
+ help="Censys API ID")
+ parser.add_argument('-s', '--apisecret', required=True,
+ help="Censys API secret")
+ parser.add_argument('-a', '--asn', type=int, required=True,
+ help="ASN to query")
+ parser.add_argument('-n', '--node', required=True,
+ help="Node name to fill into IDEA messages")
+ parser.add_argument('-d', '--destdir', dest="path", default=os.getcwd(),
+ help="Path to destination directory (with 'incoming' and 'temp' subdirectories) (default: CWD)")
+ parser.add_argument('-t', '--test', action="store_true",
+ help="Add 'Test' category to IDEA messages.")
+ parser.add_argument('-v', '--verbose', action="store_true",
+ help="Print information about progress and results")
+ return parser.parse_args()
+
+
+def main():
+ IPMI()
+ SCADA()
+ printerIPP()
+ mongoDB()
+ PCA()
+ elasticSearch()
+ hacked()
+ unsupportedPHP()
+
+
+if __name__ == "__main__":
+ # getting arguments from argparse
+ args = parse_args()
+ VERBOSE = args.verbose
+ default_directory = args.path
+ node_name = args.node
+ test_category = args.test
+ asnString = "autonomous_system.asn:" + str(args.asn) + " AND "
+
+ c = censys.ipv4.CensysIPv4(api_id=str(args.apiid),
+ api_secret=str(args.apisecret))
+
+ # incoming directory creation
+ directory = "incoming"
+ path = os.path.join(default_directory, directory)
+ os.makedirs(path, exist_ok=True)
+
+ # tmp directory creation
+ directory = "tmp"
+ path = os.path.join(default_directory, directory)
+ os.makedirs(path, exist_ok=True)
+
+ main()
+
diff --git a/shodan/shodan2warden.cron b/shodan/shodan2warden.cron
new file mode 100644
index 0000000000000000000000000000000000000000..ea09bb7e2b21e1b6ec94404c1df048c3cec24b9c
--- /dev/null
+++ b/shodan/shodan2warden.cron
@@ -0,0 +1,2 @@
+# Run every day at 9:00
+0 9 * * * shodan2warden python3 /data/shodan2warden/shodan2warden.py -k $(cat /data/shodan2warden/shodan_key) -a 2852 -n cz.cesnet.ext.shodan -d /data/shodan2warden/warden_sender --test -v >>/data/shodan2warden/shodan2warden.log 2>&1
\ No newline at end of file