Skip to content
Snippets Groups Projects
Commit 7e182e44 authored by Pavel Eis's avatar Pavel Eis
Browse files

TippingPoint connector - remake of id_taxonomy conversion to IDEA. Also... 

TippingPoint connector - remake of id_taxonomy conversion to IDEA. Also corrected some mistakes in IDEA conversion and done some refractoring.
parent 1d673a65
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
hp-tipping-point/tpToIdea.py
+ 149
− 79
View file @ 7e182e44
# python ./tpToIdea.py --origdata --oneshot -d /root/Dokumenty/PycharmProjects/TippingPoint/IdeaLogTest -n cz.cesnet.server.tippingpoint /root/Dokumenty/PycharmProjects/TippingPoint/tp-test-log.txt /root/Dokumenty/PycharmProjects/TippingPoint/tp-log2.txt --test
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017-2018 Cesnet z.s.p.o
# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
import json
from uuid import uuid4
......@@ -13,7 +16,9 @@ import resource
import os.path as pth
import atexit
import time
from datetime import datetime
import logging
from collections import OrderedDict
class FileWatcher(object):
......@@ -134,92 +139,155 @@ class Filer(object):
class IdeaGen(object):
idea_categories = {'Malware.Worm': re.compile("^Exploits: .*?Worm"),
'Attempt.Login': re.compile("SipVicious Brute Force"),
'Attempt.Exploit': re.compile("^Exploits: .*?Attempt"),
'Other': re.compile("P2P"),
'Recon.Scanning': re.compile("^Reconnaissance"),
'Vulnerable.Open': re.compile("(^Vulnerabilities: [0-9]{5}|^Network Equipment: [0-9]{4} SNMP)"),
'Vulnerable.Config': re.compile("^Vulnerabilities: [0-9]{4} ICMP")}
anomaly_re = re.compile("(^Network Equipment: [0-9]{4} IP|^Traffic Normalization: [0-9]{4} (?!Tunneling))")
availability_ddos_re = re.compile("^Traffic Normalization: ")
def __init__(self, name, test=False):
tp_to_idea = {
1: {
1: ["Attempt.Exploit"],
2: ["Attempt.Exploit"],
3: ["Attempt.Exploit"],
4: ["Attempt.Exploit"],
5: ["Attempt.Exploit"],
6: ["Attempt.Exploit"],
255: ["Attempt.Exploit"]
},
2: {
1: ["Malware.Worm"],
2: ["Malware.Virus"],
3: ["Malware.Trojan"],
4: ["Intrusion.Botnet"],
5: ["Fraud.Phishing"],
255: ["Malware"]
},
3: {
1: ["Availability.DDoS"],
2: ["Availability.DDoS"],
3: ["Availability.DDoS"],
255: ["Availability.DDoS"]
},
4: {
1: ["Other"],
2: ["Other"],
3: ["Other"],
4: ["Other"],
5: ["Other"],
6: ["Attempt.Login"],
7: ["Malware.Spyware"],
255: ["Other"]
},
5: {
1: ["Recon.Scanning"],
2: ["Attempt.Exploit"],
3: ["Attempt.Exploit"],
4: ["Recon.Scanning", "Attempt.Exploit"],
255: ["Attempt.Exploit"]
},
6: {
1: ["Anomaly.Protocol"],
2: ["Anomaly.Traffic"],
3: ["Anomaly.Application"],
255: ["Anomaly"]
},
7: {
1: ["Anomaly.Traffic"],
2: ["Anomaly.Application"],
255: ["Anomaly.Traffic"]
},
8: {
1: ["Other"],
2: ["Other"],
255: ["Other"]
},
}
def __init__(self, name, test=False, other=False):
self.name = name
self.test = test
self.other = other
def convert_category(self, category, incident, examples_of_incident):
def convert_category(self, category, id_taxonomy):
'''
converts category from record to IDEA category
:param category: TippingPoint category description
:param incident: TippingPoint incident description
:param examples_of_incident: some TippingPoing incident descriptions has examples
:param id_taxonomy: TippingPoint taxonomy id
:return: if category or incident is empty or is not important for saving it return None, otherwise return
converted category
'''
if not (category and incident):
if not (category and id_taxonomy):
return None
tp_category = category + ": " + incident
if IdeaGen.anomaly_re.search(tp_category) and not examples_of_incident:
return "Anomaly.Traffic"
if IdeaGen.availability_ddos_re.search(tp_category) and examples_of_incident:
return "Availability.DoS"
for category, pattern in IdeaGen.idea_categories.items():
if pattern.search(tp_category):
return category
return None
def gen_event_idea(self, timestamp, category, id_taxonomy, cve, examples_of_incident, proto, src_ip, src_port,
dest_ip, dest_port, conn_count, url, orig_data, incident_desription):
tp_cat_maj = id_taxonomy >> 24
tp_cat_min = id_taxonomy >> 16 & 0b11111111
tp_proto = id_taxonomy >> 8 & 0b11111111
tp_platf = id_taxonomy & 0b11111111
try:
category = IdeaGen.tp_to_idea[tp_cat_maj][tp_cat_min]
except KeyError:
category = ["Other"]
return category
def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
dest_ip, dest_port, conn_count, url, severity, orig_data):
'''
put every piece of record together into IDEA message
:return: new IDEA message
'''
event = {
'Format': "IDEA0",
'ID': str(uuid4()),
'DetectTime': time.strftime("%y-%m-%dT%H:%M:%S", time.localtime(timestamp)),
'Category': [category] + (["Test"] if self.test else []),
'Note': incident_desription,
'Ref': []
}
if (category == ["Other"]) and not self.other:
return None
event = OrderedDict([
("Format","IDEA0"),
("ID", str(uuid4())),
("DetectTime", datetime.fromtimestamp(timestamp / 1000).isoformat() + 'Z'),
("Category", category + (["Test"] if self.test else [])),
])
if cve:
event['Ref'].append("cve:"+cve)
if id_taxonomy:
event['Ref'].append("tipping_point_taxonomy:%d" % int(id_taxonomy))
if conn_count:
event['ConnCount'] = conn_count
source = {}
target = {}
event['Ref'] = ['urn:cve:'.format(i) for i in cve]
if conn_count and int(conn_count):
event['ConnCount'] = int(conn_count)
source = OrderedDict()
target = OrderedDict()
if src_ip:
# TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
src_ip = src_ip.replace(" ", "")
af = "IP4" if not ':' in src_ip else "IP6"
source[af] = [src_ip]
if src_port:
source['Port'] = [src_port]
if src_port and int(src_port):
source['Port'] = [int(src_port)]
if proto:
source['Proto'] = [proto]
if url:
source['url'] = url
if dest_ip:
if dest_ip and (dest_ip != "0.0.0.0"):
# TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
dest_ip = dest_ip.replace(" ", "")
af = "IP4" if not ':' in dest_ip else "IP6"
target[af] = [dest_ip]
if dest_port:
target['Port'] = [dest_port]
if dest_port and int(dest_port):
target['Port'] = [int(dest_port)]
if proto:
target['Proto'] = [proto]
if url:
target['URL'] = url
if source:
event['Source'] = [source]
if target:
event['Target'] = [target]
if examples_of_incident:
event['Note'] += ", examples of event:" + examples_of_incident[6:]
if orig_data:
event['Attach'] = {'Handle': "att1",
'Note': "original data",
'Content': orig_data}
event['Node'] = [{
'Name': self.name,
'Type': ["Connection", "Honeypot", "Recon"],
'SW': ["HP Tipping Point"],
}]
event['Attach'] = [OrderedDict([
('Type', ["OrigData"]),
('Content', orig_data.strip())
])]
event['Node'] = [OrderedDict([
('Name', self.name),
('Type', ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"]),
('SW', ["TippingPoint_NX_NGIPS"])
])]
return event
......@@ -284,7 +352,7 @@ def daemonize(
def get_args():
optp = optparse.OptionParser(
usage="usage: %prog [options] logfile ...",
description="Watch HP-Tipping_Point logfiles and generate Idea events into directory")
description="Watch TippingPoint logfiles and generate Idea events into directory")
optp.add_option(
"-n", "--name",
default=None,
......@@ -297,7 +365,13 @@ def get_args():
default=False,
dest="test",
action="store_true",
help="Add Test category")
help="Add \"Test\" category")
optp.add_option(
"--other",
default=False,
dest="other",
action="store_true",
help="Send events having \"Other\" category (usually nonmalicious)")
optp.add_option(
"-o", "--oneshot",
default=False,
......@@ -367,23 +441,19 @@ def process_data(line, filer, origdata, idea_gen):
:param idea_file: where output goes
:param origdata: if true, write original record to IDEA message
'''
row = line.split(",")
while re.search("^CVE-.*$", row[4]):
row[3] += ", " + row[4]
del row[4]
examples_of_incident = []
while not re.search("(UDP|IP|ICMP|DNS|IP6)", row[5]):
examples_of_incident.append(row[5])
del row[5]
examples_of_incident = " ".join(examples_of_incident)
category = idea_gen.convert_category(category=row[1], incident=row[4], examples_of_incident=examples_of_incident)
row = line.split("|")
category = idea_gen.convert_category(category=row[1], id_taxonomy=int(row[2]))
timestamp = row[0].split(" ")[-1]
cve = [i for i in row[3].split(",") if i not in ("null", "")]
odata = "|".join([timestamp] + row[1:])
if category and not_empty(row[0][-14:-3]):
idea_event = idea_gen.gen_event_idea(timestamp=int(row[0][-14:-3]), category=category, id_taxonomy=not_empty(row[2]), cve=not_empty(row[3]),
examples_of_incident=not_empty(examples_of_incident), proto=not_empty(row[5]),
idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=category, id_taxonomy=int(row[2]),
cve=cve, filter_name=not_empty(row[4]), proto=not_empty(row[5]),
src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
conn_count=not_empty(row[10]), url=not_empty(row[12]),
orig_data=str(line) if origdata else False, incident_desription=row[4])
save_events(idea_event, filer)
conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
orig_data=odata if origdata else False)
if idea_event:
save_events(idea_event, filer)
running_flag = True
......@@ -425,7 +495,7 @@ def main():
})
files = [FileWatcher(arg) for arg in args]
filer = Filer(opts.dir)
idea_gen = IdeaGen(opts.name, opts.test)
idea_gen = IdeaGen(opts.name, opts.test, opts.other)
while running_flag:
for log_file in files:
while True:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment