Censys connector added + example cron files for Censys and Shodan

censys/censys2warden.cron

+# Run every day at 9:00
+0 9 * * * shodan2warden python3 /data/censys2warden/censys2warden.py $(cat /data/censys2warden/api_params) -a 2852 -n cz.cesnet.ext.censys -d /data/censys2warden/warden_sender --test -v >>/data/censys2warden/censys2warden.log 2>&1
\ No newline at end of file

censys/censys2warden.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Author: Pavla Hlukov
+#         Vclav Barto <bartos@cesnet.cz>
+
+import censys.ipv4
+import json
+import os
+from datetime import datetime
+from uuid import uuid4
+import argparse
+
+def vprint(*args, **kwargs):
+    # Verbose print
+    if VERBOSE:
+        print("[{}] ".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")), end="")
+        print(*args, **kwargs)
+
+IPV4_FIELDS = ['ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
+
+MAX_RESULTS_PER_QUERY = 1000
+
+def generateIdeaEvent(detect_time, category, description,ip_string,ports,proto,content_type,content,note):
+   create_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+   # if there's no timezone in detect_time, assume UTC (which Shodan normally uses) and append 'Z'
+   if 'Z' not in detect_time and '+' not in detect_time and '-' not in detect_time:
+       detect_time += 'Z'
+   event = {
+       "Format": "IDEA0",
+       "ID": str(uuid4()),
+       "Category": [category],
+       "CreateTime": create_time,
+       "DetectTime": detect_time,
+       "Description": description,
+       "Ref": ["https://www.censys.io/ipv4/" + ip_string],
+       "Source": [
+           {
+               "IP4": [ip_string],
+               "Port": ports,
+               "Proto": proto
+           }
+       ],
+       "Node": [
+           {
+               "Name": node_name,
+               "SW": ["censys2warden"],
+               "Type": ["External", "Recon"]
+           }
+       ],
+       "Attach": [
+           {
+               "ContentType": content_type,
+               "Content": json.dumps(content) if content_type == 'application/json' else content,
+               "Note": note
+           }
+       ]
+   }
+
+   if test_category:
+       event["Category"].append("Test")
+
+   filename = "{}_{}_{}_{}.json".format(
+       datetime.utcnow().strftime('%Y%m%d%H%M%S%f'),
+       category.replace('.','').lower(),
+       ip_string,
+       event['ID'][:8])
+   tmp_destination = os.path.join(default_directory, 'tmp', filename)
+   inc_destination = os.path.join(default_directory, 'incoming', filename)
+
+   with open(tmp_destination, 'w') as json_file:
+       json.dump(event, json_file)
+
+   os.rename(tmp_destination, inc_destination)
+
+def IPMI():
+   query = asnString + "protocols: \"623/ipmi\" tags: ipmi"
+   category = "Vulnerable.Config"
+   description = "Publicly accessible insecure protocol: IPMI"
+   proto = ["udp", "ipmi"]
+   content_type = "application/json"
+   note = "Original Censys data (subset)"
+   vprint("Querying for IMPI:", query)
+   for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
+       vprint("Found problematic IP:", banner.get('ip'))
+       generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[623],proto,content_type,banner,note)
+
+def SCADA():
+    query = asnString + "scada"
+    category = "Vulnerable.Config"
+    description = "Publicly accessible SCADA (BACnet) system"
+    proto = ["udp", "bacnet"]
+    content_type = "application/json"
+    note = "Original Censys data (subset)"
+    vprint("Querying for SCADA:", query)
+    for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY): 
+        # TODO: find out the port of the scada protocol(s)
+        #   (sometimes there are multiple services running on the IP)
+        vprint("Found problematic IP:", banner.get('ip'))
+        generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),banner.get('ports'),proto,content_type,banner,note)
+
+def printerIPP():
+    query = asnString + "protocols: \"631/ipp\""
+    category = "Vulnerable.Config"
+    description = "Potentially vulnerable IPP printer"
+    proto = ["tcp", "ipp"]
+    content_type = "application/json"
+    note = "Original Censys data (subset)"
+    vprint("Querying for printer IPP:", query)
+    for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
+        vprint("Found problematic IP:", banner.get('ip'))
+        generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[631],proto,content_type,banner,note)
+
+def mongoDB():
+    query = asnString + "protocols: \"27017/mongodb\""
+    category = "Vulnerable.Config"
+    description = "Potentially vulnerable MongoDB database"
+    proto = ["tcp"]
+    content_type = "application/json"
+    note = "Original Censys data (subset)"
+    vprint("Querying for MongoDB:", query)
+    for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+        # TODO: try to find a field with DB size, but it seems there's none
+        vprint("Found problematic IP:", banner.get('ip'))
+        generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[27017],proto,content_type,banner,note)
+
+def PCA():
+    query = asnString + "tags: pca"
+    category = "Vulnerable.Config"
+    description = "An old unsupported service 'PCAnywhere' open to internet"
+    proto = ["tcp","pca"]
+    content_type = "application/json"
+    note = "Original Censys data (subset)"
+    vprint("Querying for PCAnywhere:", query)
+    for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+        vprint("Found problematic IP:", banner.get('ip'))
+        generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+def elasticSearch():
+    query = asnString + "protocols: \"9200/elasticsearch\""
+    category = "Vulnerable.Config"
+    description = "Possibly vulnerable data displayed - Elastic Search"
+    proto = ["tcp"]
+    content_type = "application/json"
+    note = "Original Censys data (subset)"
+    vprint("Querying for Elastic Indices:", query)
+    for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+        vprint("Found problematic IP:", banner.get('ip'))
+        generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+
+def hacked():
+    IPV4_FIELDS_HACKED = ['80.http.get.body','443.https.get.body','ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
+    query = asnString + "\"hacked\""
+    category = "Information.UnauthorizedModification"
+    description = "Service probably hacked (\"hacked\" string found in banner)"
+    proto = []
+    content_type = "application/json"
+    note = "Original Censys data (subset)"
+    vprint("Querying for \"hacked\" string:", query)
+    for banner in c.search(query, IPV4_FIELDS_HACKED,max_records=MAX_RESULTS_PER_QUERY):
+        if "Test Page" in str(banner.get('80.http.get.body')) or "Test Page" in str(banner.get('443.https.get.body')):
+            continue
+        if "Cyber Security" in str(banner.get('80.http.get.body')) or "Cyber Security" in str(banner.get('443.https.get.body')):
+            # this was added to filter a false positive on a cybersecurity page mentioning hacking, see http://158.196.109.174/
+            continue
+        vprint("Found problematic IP:", banner.get('ip'))
+        generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+def unsupportedPHP():
+    query = asnString + "(80.http.get.headers.x_powered_by: PHP\\/5.* OR 8080.http.get.headers.x_powered_by: PHP\\/5.* OR 443.https.get.headers.x_powered_by: PHP\\/5.*)"
+    category = "Vulnerable.Open"
+    description = "Web running on old (unsupported) PHP version"
+    proto = ["tcp", "http"]
+    content_type = "application/json"
+    note = "Original Censys data (subset)"
+    vprint("Querying for unsupported PHP:", query)
+    for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
+        vprint("Found problematic IP:", banner.get('ip'))
+        generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
+
+
+def parse_args():
+    # command line argument parser
+    parser = argparse.ArgumentParser(
+        description="Searches Censys for potential problems with open services in given ASN. For each such problem generates an IDEA message into gievn directory (to be sent to Warden by warden_filer). This script is assumed to be run daily by cron.")
+    parser.add_argument('-i', '--apiid', required=True,
+                        help="Censys API ID")
+    parser.add_argument('-s', '--apisecret', required=True,
+                        help="Censys API secret")
+    parser.add_argument('-a', '--asn', type=int, required=True,
+                        help="ASN to query")
+    parser.add_argument('-n', '--node', required=True,
+                        help="Node name to fill into IDEA messages")
+    parser.add_argument('-d', '--destdir', dest="path", default=os.getcwd(),
+                        help="Path to destination directory (with 'incoming' and 'temp' subdirectories) (default: CWD)")
+    parser.add_argument('-t', '--test', action="store_true",
+                        help="Add 'Test' category to IDEA messages.")
+    parser.add_argument('-v', '--verbose', action="store_true",
+                        help="Print information about progress and results")
+    return parser.parse_args()
+
+
+def main():
+        IPMI()
+        SCADA()
+        printerIPP()
+        mongoDB()
+        PCA()
+        elasticSearch()
+        hacked()
+        unsupportedPHP()
+
+
+if __name__ == "__main__":
+       # getting arguments from argparse
+       args = parse_args()
+       VERBOSE = args.verbose
+       default_directory = args.path
+       node_name = args.node
+       test_category = args.test
+       asnString = "autonomous_system.asn:" + str(args.asn) + " AND "
+
+       c = censys.ipv4.CensysIPv4(api_id=str(args.apiid),
+                                  api_secret=str(args.apisecret))
+        
+       # incoming directory creation
+       directory = "incoming"
+       path = os.path.join(default_directory, directory)
+       os.makedirs(path, exist_ok=True)
+
+       # tmp directory creation
+       directory = "tmp"
+       path = os.path.join(default_directory, directory)
+       os.makedirs(path, exist_ok=True)
+
+       main()
+

shodan/shodan2warden.cron

+# Run every day at 9:00
+0 9 * * * shodan2warden python3 /data/shodan2warden/shodan2warden.py -k $(cat /data/shodan2warden/shodan_key) -a 2852 -n cz.cesnet.ext.shodan -d /data/shodan2warden/warden_sender --test -v >>/data/shodan2warden/shodan2warden.log 2>&1
\ No newline at end of file