censys2warden.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Author: Pavla Hlukov
#         Vclav Barto <bartos@cesnet.cz>

import censys.ipv4
import json
import os
from datetime import datetime
from uuid import uuid4
import argparse

def vprint(*args, **kwargs):
    # Verbose print
    if VERBOSE:
        print("[{}] ".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")), end="")
        print(*args, **kwargs)

IPV4_FIELDS = ['ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']

MAX_RESULTS_PER_QUERY = 1000

def generateIdeaEvent(detect_time, category, description,ip_string,ports,proto,content_type,content,note):
   create_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
   # if there's no timezone in detect_time, assume UTC (which Shodan normally uses) and append 'Z'
   if 'Z' not in detect_time and '+' not in detect_time and '-' not in detect_time:
       detect_time += 'Z'
   event = {
       "Format": "IDEA0",
       "ID": str(uuid4()),
       "Category": [category],
       "CreateTime": create_time,
       "DetectTime": detect_time,
       "Description": description,
       "Ref": ["https://www.censys.io/ipv4/" + ip_string],
       "Source": [
           {
               "IP4": [ip_string],
               "Port": ports,
               "Proto": proto
           }
       ],
       "Node": [
           {
               "Name": node_name,
               "SW": ["censys2warden"],
               "Type": ["External", "Recon"]
           }
       ],
       "Attach": [
           {
               "ContentType": content_type,
               "Content": json.dumps(content) if content_type == 'application/json' else content,
               "Note": note
           }
       ]
   }

   if test_category:
       event["Category"].append("Test")

   filename = "{}_{}_{}_{}.json".format(
       datetime.utcnow().strftime('%Y%m%d%H%M%S%f'),
       category.replace('.','').lower(),
       ip_string,
       event['ID'][:8])
   tmp_destination = os.path.join(default_directory, 'tmp', filename)
   inc_destination = os.path.join(default_directory, 'incoming', filename)

   with open(tmp_destination, 'w') as json_file:
       json.dump(event, json_file)

   os.rename(tmp_destination, inc_destination)

def IPMI():
   query = asnString + "protocols: \"623/ipmi\" tags: ipmi"
   category = "Vulnerable.Config"
   description = "Publicly accessible insecure protocol: IPMI"
   proto = ["udp", "ipmi"]
   content_type = "application/json"
   note = "Original Censys data (subset)"
   vprint("Querying for IMPI:", query)
   for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
       vprint("Found problematic IP:", banner.get('ip'))
       generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[623],proto,content_type,banner,note)

def SCADA():
    query = asnString + "scada"
    category = "Vulnerable.Config"
    description = "Publicly accessible SCADA (BACnet) system"
    proto = ["udp", "bacnet"]
    content_type = "application/json"
    note = "Original Censys data (subset)"
    vprint("Querying for SCADA:", query)
    for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY): 
        # TODO: find out the port of the scada protocol(s)
        #   (sometimes there are multiple services running on the IP)
        vprint("Found problematic IP:", banner.get('ip'))
        generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),banner.get('ports'),proto,content_type,banner,note)

def printerIPP():
    query = asnString + "protocols: \"631/ipp\""
    category = "Vulnerable.Config"
    description = "Potentially vulnerable IPP printer"
    proto = ["tcp", "ipp"]
    content_type = "application/json"
    note = "Original Censys data (subset)"
    vprint("Querying for printer IPP:", query)
    for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
        vprint("Found problematic IP:", banner.get('ip'))
        generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[631],proto,content_type,banner,note)

def mongoDB():
    query = asnString + "protocols: \"27017/mongodb\""
    category = "Vulnerable.Config"
    description = "Potentially vulnerable MongoDB database"
    proto = ["tcp"]
    content_type = "application/json"
    note = "Original Censys data (subset)"
    vprint("Querying for MongoDB:", query)
    for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
        # TODO: try to find a field with DB size, but it seems there's none
        vprint("Found problematic IP:", banner.get('ip'))
        generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[27017],proto,content_type,banner,note)

def PCA():
    query = asnString + "tags: pca"
    category = "Vulnerable.Config"
    description = "An old unsupported service 'PCAnywhere' open to internet"
    proto = ["tcp","pca"]
    content_type = "application/json"
    note = "Original Censys data (subset)"
    vprint("Querying for PCAnywhere:", query)
    for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
        vprint("Found problematic IP:", banner.get('ip'))
        generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)

def elasticSearch():
    query = asnString + "protocols: \"9200/elasticsearch\""
    category = "Vulnerable.Config"
    description = "Possibly vulnerable data displayed - Elastic Search"
    proto = ["tcp"]
    content_type = "application/json"
    note = "Original Censys data (subset)"
    vprint("Querying for Elastic Indices:", query)
    for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
        vprint("Found problematic IP:", banner.get('ip'))
        generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)


def hacked():
    IPV4_FIELDS_HACKED = ['80.http.get.body','443.https.get.body','ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
    query = asnString + "\"hacked\""
    category = "Information.UnauthorizedModification"
    description = "Service probably hacked (\"hacked\" string found in banner)"
    proto = []
    content_type = "application/json"
    note = "Original Censys data (subset)"
    vprint("Querying for \"hacked\" string:", query)
    for banner in c.search(query, IPV4_FIELDS_HACKED,max_records=MAX_RESULTS_PER_QUERY):
        if "Test Page" in str(banner.get('80.http.get.body')) or "Test Page" in str(banner.get('443.https.get.body')):
            continue
        if "Cyber Security" in str(banner.get('80.http.get.body')) or "Cyber Security" in str(banner.get('443.https.get.body')):
            # this was added to filter a false positive on a cybersecurity page mentioning hacking, see http://158.196.109.174/
            continue
        vprint("Found problematic IP:", banner.get('ip'))
        generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)

def unsupportedPHP():
    query = asnString + "(80.http.get.headers.x_powered_by: PHP\\/5.* OR 8080.http.get.headers.x_powered_by: PHP\\/5.* OR 443.https.get.headers.x_powered_by: PHP\\/5.*)"
    category = "Vulnerable.Open"
    description = "Web running on old (unsupported) PHP version"
    proto = ["tcp", "http"]
    content_type = "application/json"
    note = "Original Censys data (subset)"
    vprint("Querying for unsupported PHP:", query)
    for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
        vprint("Found problematic IP:", banner.get('ip'))
        generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)


def parse_args():
    # command line argument parser
    parser = argparse.ArgumentParser(
        description="Searches Censys for potential problems with open services in given ASN. For each such problem generates an IDEA message into gievn directory (to be sent to Warden by warden_filer). This script is assumed to be run daily by cron.")
    parser.add_argument('-i', '--apiid', required=True,
                        help="Censys API ID")
    parser.add_argument('-s', '--apisecret', required=True,
                        help="Censys API secret")
    parser.add_argument('-a', '--asn', type=int, required=True,
                        help="ASN to query")
    parser.add_argument('-n', '--node', required=True,
                        help="Node name to fill into IDEA messages")
    parser.add_argument('-d', '--destdir', dest="path", default=os.getcwd(),
                        help="Path to destination directory (with 'incoming' and 'temp' subdirectories) (default: CWD)")
    parser.add_argument('-t', '--test', action="store_true",
                        help="Add 'Test' category to IDEA messages.")
    parser.add_argument('-v', '--verbose', action="store_true",
                        help="Print information about progress and results")
    return parser.parse_args()


def main():
        IPMI()
        SCADA()
        printerIPP()
        mongoDB()
        PCA()
        elasticSearch()
        hacked()
        unsupportedPHP()


if __name__ == "__main__":
       # getting arguments from argparse
       args = parse_args()
       VERBOSE = args.verbose
       default_directory = args.path
       node_name = args.node
       test_category = args.test
       asnString = "autonomous_system.asn:" + str(args.asn) + " AND "

       c = censys.ipv4.CensysIPv4(api_id=str(args.apiid),
                                  api_secret=str(args.apisecret))
        
       # incoming directory creation
       directory = "incoming"
       path = os.path.join(default_directory, directory)
       os.makedirs(path, exist_ok=True)

       # tmp directory creation
       directory = "tmp"
       path = os.path.join(default_directory, directory)
       os.makedirs(path, exist_ok=True)

       main()