Skip to content
Snippets Groups Projects
Commit 5a7bfc37 authored by Václav Bartoš's avatar Václav Bartoš
Browse files

Censys connector added + example cron files for Censys and Shodan

parent 792f9080
No related branches found
No related tags found
No related merge requests found
# Run every day at 9:00
0 9 * * * shodan2warden python3 /data/censys2warden/censys2warden.py $(cat /data/censys2warden/api_params) -a 2852 -n cz.cesnet.ext.censys -d /data/censys2warden/warden_sender --test -v >>/data/censys2warden/censys2warden.log 2>&1
\ No newline at end of file
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Author: Pavla Hlukov
# Vclav Barto <bartos@cesnet.cz>
import censys.ipv4
import json
import os
from datetime import datetime
from uuid import uuid4
import argparse
def vprint(*args, **kwargs):
# Verbose print
if VERBOSE:
print("[{}] ".format(datetime.now().strftime("%Y-%m-%d %H:%M:%S")), end="")
print(*args, **kwargs)
IPV4_FIELDS = ['ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
MAX_RESULTS_PER_QUERY = 1000
def generateIdeaEvent(detect_time, category, description,ip_string,ports,proto,content_type,content,note):
create_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
# if there's no timezone in detect_time, assume UTC (which Shodan normally uses) and append 'Z'
if 'Z' not in detect_time and '+' not in detect_time and '-' not in detect_time:
detect_time += 'Z'
event = {
"Format": "IDEA0",
"ID": str(uuid4()),
"Category": [category],
"CreateTime": create_time,
"DetectTime": detect_time,
"Description": description,
"Ref": ["https://www.censys.io/ipv4/" + ip_string],
"Source": [
{
"IP4": [ip_string],
"Port": ports,
"Proto": proto
}
],
"Node": [
{
"Name": node_name,
"SW": ["censys2warden"],
"Type": ["External", "Recon"]
}
],
"Attach": [
{
"ContentType": content_type,
"Content": json.dumps(content) if content_type == 'application/json' else content,
"Note": note
}
]
}
if test_category:
event["Category"].append("Test")
filename = "{}_{}_{}_{}.json".format(
datetime.utcnow().strftime('%Y%m%d%H%M%S%f'),
category.replace('.','').lower(),
ip_string,
event['ID'][:8])
tmp_destination = os.path.join(default_directory, 'tmp', filename)
inc_destination = os.path.join(default_directory, 'incoming', filename)
with open(tmp_destination, 'w') as json_file:
json.dump(event, json_file)
os.rename(tmp_destination, inc_destination)
def IPMI():
query = asnString + "protocols: \"623/ipmi\" tags: ipmi"
category = "Vulnerable.Config"
description = "Publicly accessible insecure protocol: IPMI"
proto = ["udp", "ipmi"]
content_type = "application/json"
note = "Original Censys data (subset)"
vprint("Querying for IMPI:", query)
for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
vprint("Found problematic IP:", banner.get('ip'))
generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[623],proto,content_type,banner,note)
def SCADA():
query = asnString + "scada"
category = "Vulnerable.Config"
description = "Publicly accessible SCADA (BACnet) system"
proto = ["udp", "bacnet"]
content_type = "application/json"
note = "Original Censys data (subset)"
vprint("Querying for SCADA:", query)
for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
# TODO: find out the port of the scada protocol(s)
# (sometimes there are multiple services running on the IP)
vprint("Found problematic IP:", banner.get('ip'))
generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),banner.get('ports'),proto,content_type,banner,note)
def printerIPP():
query = asnString + "protocols: \"631/ipp\""
category = "Vulnerable.Config"
description = "Potentially vulnerable IPP printer"
proto = ["tcp", "ipp"]
content_type = "application/json"
note = "Original Censys data (subset)"
vprint("Querying for printer IPP:", query)
for banner in c.search(query, IPV4_FIELDS, max_records=MAX_RESULTS_PER_QUERY):
vprint("Found problematic IP:", banner.get('ip'))
generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[631],proto,content_type,banner,note)
def mongoDB():
query = asnString + "protocols: \"27017/mongodb\""
category = "Vulnerable.Config"
description = "Potentially vulnerable MongoDB database"
proto = ["tcp"]
content_type = "application/json"
note = "Original Censys data (subset)"
vprint("Querying for MongoDB:", query)
for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
# TODO: try to find a field with DB size, but it seems there's none
vprint("Found problematic IP:", banner.get('ip'))
generateIdeaEvent(banner.get('updated_at'),category,description,banner.get('ip'),[27017],proto,content_type,banner,note)
def PCA():
query = asnString + "tags: pca"
category = "Vulnerable.Config"
description = "An old unsupported service 'PCAnywhere' open to internet"
proto = ["tcp","pca"]
content_type = "application/json"
note = "Original Censys data (subset)"
vprint("Querying for PCAnywhere:", query)
for banner in c.search(query, IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
vprint("Found problematic IP:", banner.get('ip'))
generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
def elasticSearch():
query = asnString + "protocols: \"9200/elasticsearch\""
category = "Vulnerable.Config"
description = "Possibly vulnerable data displayed - Elastic Search"
proto = ["tcp"]
content_type = "application/json"
note = "Original Censys data (subset)"
vprint("Querying for Elastic Indices:", query)
for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
vprint("Found problematic IP:", banner.get('ip'))
generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
def hacked():
IPV4_FIELDS_HACKED = ['80.http.get.body','443.https.get.body','ip', 'updated_at', 'ports', 'protocols','tags', 'metadata.description','metadata.device_type', 'metadata.manufacturer', 'location.city', 'location.country_code']
query = asnString + "\"hacked\""
category = "Information.UnauthorizedModification"
description = "Service probably hacked (\"hacked\" string found in banner)"
proto = []
content_type = "application/json"
note = "Original Censys data (subset)"
vprint("Querying for \"hacked\" string:", query)
for banner in c.search(query, IPV4_FIELDS_HACKED,max_records=MAX_RESULTS_PER_QUERY):
if "Test Page" in str(banner.get('80.http.get.body')) or "Test Page" in str(banner.get('443.https.get.body')):
continue
if "Cyber Security" in str(banner.get('80.http.get.body')) or "Cyber Security" in str(banner.get('443.https.get.body')):
# this was added to filter a false positive on a cybersecurity page mentioning hacking, see http://158.196.109.174/
continue
vprint("Found problematic IP:", banner.get('ip'))
generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
def unsupportedPHP():
query = asnString + "(80.http.get.headers.x_powered_by: PHP\\/5.* OR 8080.http.get.headers.x_powered_by: PHP\\/5.* OR 443.https.get.headers.x_powered_by: PHP\\/5.*)"
category = "Vulnerable.Open"
description = "Web running on old (unsupported) PHP version"
proto = ["tcp", "http"]
content_type = "application/json"
note = "Original Censys data (subset)"
vprint("Querying for unsupported PHP:", query)
for banner in c.search(query,IPV4_FIELDS,max_records=MAX_RESULTS_PER_QUERY):
vprint("Found problematic IP:", banner.get('ip'))
generateIdeaEvent(banner.get('updated_at'), category, description, banner.get('ip'), banner.get('ports'), proto,content_type, banner, note)
def parse_args():
# command line argument parser
parser = argparse.ArgumentParser(
description="Searches Censys for potential problems with open services in given ASN. For each such problem generates an IDEA message into gievn directory (to be sent to Warden by warden_filer). This script is assumed to be run daily by cron.")
parser.add_argument('-i', '--apiid', required=True,
help="Censys API ID")
parser.add_argument('-s', '--apisecret', required=True,
help="Censys API secret")
parser.add_argument('-a', '--asn', type=int, required=True,
help="ASN to query")
parser.add_argument('-n', '--node', required=True,
help="Node name to fill into IDEA messages")
parser.add_argument('-d', '--destdir', dest="path", default=os.getcwd(),
help="Path to destination directory (with 'incoming' and 'temp' subdirectories) (default: CWD)")
parser.add_argument('-t', '--test', action="store_true",
help="Add 'Test' category to IDEA messages.")
parser.add_argument('-v', '--verbose', action="store_true",
help="Print information about progress and results")
return parser.parse_args()
def main():
IPMI()
SCADA()
printerIPP()
mongoDB()
PCA()
elasticSearch()
hacked()
unsupportedPHP()
if __name__ == "__main__":
# getting arguments from argparse
args = parse_args()
VERBOSE = args.verbose
default_directory = args.path
node_name = args.node
test_category = args.test
asnString = "autonomous_system.asn:" + str(args.asn) + " AND "
c = censys.ipv4.CensysIPv4(api_id=str(args.apiid),
api_secret=str(args.apisecret))
# incoming directory creation
directory = "incoming"
path = os.path.join(default_directory, directory)
os.makedirs(path, exist_ok=True)
# tmp directory creation
directory = "tmp"
path = os.path.join(default_directory, directory)
os.makedirs(path, exist_ok=True)
main()
# Run every day at 9:00
0 9 * * * shodan2warden python3 /data/shodan2warden/shodan2warden.py -k $(cat /data/shodan2warden/shodan_key) -a 2852 -n cz.cesnet.ext.shodan -d /data/shodan2warden/warden_sender --test -v >>/data/shodan2warden/shodan2warden.log 2>&1
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment