diff --git a/tippingpoint/tpToIdea.py b/tippingpoint/tpToIdea.py
index 1d723790f1ce7fcbc3db5ea0461d75ea5b7fb034..52b9bacb46e477a368945a52650772d2d15bc961 100644
--- a/tippingpoint/tpToIdea.py
+++ b/tippingpoint/tpToIdea.py
@@ -1,5 +1,8 @@
-# python ./tpToIdea.py --origdata --oneshot -d /root/Dokumenty/PycharmProjects/TippingPoint/IdeaLogTest -n cz.cesnet.server.tippingpoint /root/Dokumenty/PycharmProjects/TippingPoint/tp-test-log.txt /root/Dokumenty/PycharmProjects/TippingPoint/tp-log2.txt --test
-
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2018 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
import json
from uuid import uuid4
@@ -13,6 +16,7 @@ import resource
import os.path as pth
import atexit
import time
+from datetime import datetime
import logging
@@ -134,91 +138,251 @@ class Filer(object):
class IdeaGen(object):
- idea_categories = {'Malware.Worm': re.compile("^Exploits: .*?Worm"),
- 'Attempt.Login': re.compile("SipVicious Brute Force"),
- 'Attempt.Exploit': re.compile("^Exploits: .*?Attempt"),
- 'Other': re.compile("P2P"),
- 'Recon.Scanning': re.compile("^Reconnaissance"),
- 'Vulnerable.Open': re.compile("(^Vulnerabilities: [0-9]{5}|^Network Equipment: [0-9]{4} SNMP)"),
- 'Vulnerable.Config': re.compile("^Vulnerabilities: [0-9]{4} ICMP")}
- anomaly_re = re.compile("(^Network Equipment: [0-9]{4} IP|^Traffic Normalization: [0-9]{4} (?!Tunneling))")
- availability_ddos_re = re.compile("^Traffic Normalization: ")
-
- def __init__(self, name, test=False):
+
+ tp_tax_to_idea = {
+ 1: {
+ 1: ["Attempt.Exploit"],
+ 2: ["Attempt.Exploit"],
+ 3: ["Attempt.Exploit"],
+ 4: ["Attempt.Exploit"],
+ 5: ["Attempt.Exploit"],
+ 6: ["Attempt.Exploit"],
+ 255: ["Attempt.Exploit"]
+ },
+
+ 2: {
+ 1: ["Malware.Worm"],
+ 2: ["Malware.Virus"],
+ 3: ["Malware.Trojan"],
+ 4: ["Intrusion.Botnet"],
+ 5: ["Fraud.Phishing"],
+ 255: ["Malware"]
+ },
+
+ 3: {
+ 1: ["Availability.DDoS"],
+ 2: ["Availability.DDoS"],
+ 3: ["Availability.DDoS"],
+ 255: ["Availability.DDoS"]
+ },
+
+ 4: {
+ 1: ["Other"],
+ 2: ["Other"],
+ 3: ["Other"],
+ 4: ["Other"],
+ 5: ["Other"],
+ 6: ["Attempt.Login"],
+ 7: ["Malware.Spyware"],
+ 255: ["Other"]
+ },
+
+ 5: {
+ 1: ["Recon.Scanning"],
+ 2: ["Attempt.Exploit"],
+ 3: ["Attempt.Exploit"],
+ 4: ["Recon.Scanning", "Attempt.Exploit"],
+ 255: ["Attempt.Exploit"]
+ },
+
+ 6: {
+ 1: ["Anomaly.Protocol"],
+ 2: ["Anomaly.Traffic"],
+ 3: ["Anomaly.Application"],
+ 255: ["Anomaly"]
+ },
+
+ 7: {
+ 1: ["Anomaly.Traffic"],
+ 2: ["Anomaly.Application"],
+ 255: ["Anomaly.Traffic"]
+ },
+
+ 8: {
+ 1: ["Other"],
+ 2: ["Other"],
+ 255: ["Other"]
+ },
+ }
+
+ tp_prot_to_idea = {
+ 1: "appletalk",
+ 2: "auth",
+ 3: "bgp",
+ 4: "cdp",
+ 5: "clns",
+ 6: "dhcp",
+ 7: "domain",
+ 8: "finger",
+ 9: "ftp",
+ 10: "hsrp",
+ 11: "http",
+ 12: "icmp",
+ 13: "igmp",
+ 14: "eigrp",
+ 15: "ipv6",
+ 16: "ipx",
+ 17: "irc",
+ 18: "isis",
+ 19: "isakmp",
+ 20: "ldap",
+ 21: "mpls",
+ 22: "ms-rpc",
+ 23: "ms-sql",
+ 24: "nat",
+ 25: "netbios",
+ 26: "nntp",
+ 27: "ntp",
+ 28: "oracle",
+ 29: "ospf",
+ 30: "pop-imap",
+ 31: "sunrpc",
+ 32: "qos",
+ 33: "rip",
+ 34: "sunrpc",
+ 35: "smb",
+ 36: "smtp",
+ 37: "snmp",
+ 38: "sql",
+ 39: "ssh",
+ 40: "ssl-tls",
+ 41: "tacacs",
+ 42: "tcp",
+ 43: "telnet",
+ 45: "udp",
+ 46: "uucp",
+ 48: "x11",
+ 49: "tftp",
+ 50: "ip",
+ 51: "nfs",
+ 52: "wins",
+ 80: "h323",
+ 81: "megaco",
+ 82: "mgcp",
+ 83: "sip",
+ 84: "rtp",
+ 99: "voip",
+ 100: "aim",
+ 101: "msn",
+ 102: "yahoo",
+ 103: "icq",
+ 119: "im",
+ 120: "musicmatch",
+ 121: "winamp",
+ 122: "shoutcast",
+ 123: "windows",
+ 124: "quicktime",
+ 125: "rtsp",
+ 149: "streaming",
+ 150: "bittorrent",
+ 151: "manolito",
+ 152: "directconnect",
+ 153: "earthstation5",
+ 154: "p2p",
+ 155: "fasttrack",
+ 156: "gnutella",
+ 157: "twister",
+ 158: "winmx",
+ 180: "p2p",
+ 190: "scada-dnp3",
+ 191: "scada-iccp",
+ 192: "scada-iec",
+ 193: "scada-modbus",
+ 194: "scada-opc",
+ 199: "scada",
+ 254: "multi-protocol",
+ }
+
+ def __init__(self, name, test=False, other=False):
self.name = name
self.test = test
+ self.other = other
- def convert_category(self, category, incident, examples_of_incident):
+ def convert_category_and_protocol(self, category, id_taxonomy):
'''
converts category from record to IDEA category
:param category: TippingPoint category description
- :param incident: TippingPoint incident description
- :param examples_of_incident: some TippingPoing incident descriptions has examples
+ :param id_taxonomy: TippingPoint taxonomy id
:return: if category or incident is empty or is not important for saving it return None, otherwise return
converted category
'''
- if not (category and incident):
+ if not (category and id_taxonomy):
return None
- tp_category = category + ": " + incident
- if IdeaGen.anomaly_re.search(tp_category) and not examples_of_incident:
- return "Anomaly.Traffic"
- if IdeaGen.availability_ddos_re.search(tp_category) and examples_of_incident:
- return "Availability.DoS"
- for category, pattern in IdeaGen.idea_categories.items():
- if pattern.search(tp_category):
- return category
- return None
-
- def gen_event_idea(self, timestamp, category, id_taxonomy, cve, examples_of_incident, proto, src_ip, src_port,
- dest_ip, dest_port, conn_count, url, orig_data, incident_desription):
+ tp_cat_maj = id_taxonomy >> 24
+ tp_cat_min = id_taxonomy >> 16 & 0b11111111
+ tp_prot = id_taxonomy >> 8 & 0b11111111
+ tp_platf = id_taxonomy & 0b11111111
+ try:
+ category = IdeaGen.tp_tax_to_idea[tp_cat_maj][tp_cat_min]
+ except KeyError:
+ category = ["Other"]
+
+ try:
+ protocol = IdeaGen.tp_prot_to_idea[tp_prot]
+ except KeyError:
+ protocol = None
+
+ return { 'category': category, 'protocol': protocol }
+
+ def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
+ dest_ip, dest_port, conn_count, url, severity, orig_data):
'''
put every piece of record together into IDEA message
:return: new IDEA message
'''
+
+ if (category == ["Other"]) and not self.other:
+ return None
+
event = {
'Format': "IDEA0",
'ID': str(uuid4()),
- 'DetectTime': time.strftime("%y-%m-%dT%H:%M:%S", time.localtime(timestamp)),
- 'Category': [category] + (["Test"] if self.test else []),
- 'Note': incident_desription,
- 'Ref': []
+ 'DetectTime': datetime.utcfromtimestamp(timestamp / 1000).isoformat() + 'Z',
+ 'Category': category + (["Test"] if self.test else [])
}
if cve:
- event['Ref'].append("cve:"+cve)
- if id_taxonomy:
- event['Ref'].append("tipping_point_taxonomy:%d" % int(id_taxonomy))
- if conn_count:
- event['ConnCount'] = conn_count
+ event['Ref'] = ['urn:cve:'.format(i) for i in cve]
+ if conn_count and int(conn_count):
+ event['ConnCount'] = int(conn_count)
source = {}
target = {}
if src_ip:
+ # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+ src_ip = src_ip.replace(" ", "")
af = "IP4" if not ':' in src_ip else "IP6"
source[af] = [src_ip]
- if src_port:
- source['Port'] = [src_port]
- if proto:
- source['Proto'] = [proto]
- if url:
- source['url'] = url
- if dest_ip:
+ if src_port and int(src_port):
+ source['Port'] = [int(src_port)]
+ if proto:
+ source['Proto'] = proto
+ if dest_ip and (dest_ip != "0.0.0.0"):
+ # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+ dest_ip = dest_ip.replace(" ", "")
af = "IP4" if not ':' in dest_ip else "IP6"
target[af] = [dest_ip]
- if dest_port:
- target['Port'] = [dest_port]
+ if dest_port and int(dest_port):
+ target['Port'] = [int(dest_port)]
+ if proto:
+ target['Proto'] = proto
+ if url:
+ target['URL'] = url
+
if source:
event['Source'] = [source]
if target:
- event['Target'] = [target]
- if examples_of_incident:
- event['Note'] += ", examples of event:" + examples_of_incident[6:]
+ if category and category == "Reputation":
+ event['Source'].append(target)
+ else:
+ event['Target'] = [target]
if orig_data:
- event['Attach'] = {'Handle': "att1",
- 'Note': "original data",
- 'Content': orig_data}
+ event['Attach'] = [{
+ 'Type': ["OrigData"],
+ 'Content': orig_data.strip()
+ }]
event['Node'] = [{
'Name': self.name,
- 'Type': ["Connection", "Honeypot", "Recon"],
- 'SW': ["HP Tipping Point"],
+ 'Type': ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"],
+ 'SW': ["TippingPoint_NX_NGIPS"]
}]
return event
@@ -284,7 +448,7 @@ def daemonize(
def get_args():
optp = optparse.OptionParser(
usage="usage: %prog [options] logfile ...",
- description="Watch HP-Tipping_Point logfiles and generate Idea events into directory")
+ description="Watch TippingPoint logfiles and generate Idea events into directory")
optp.add_option(
"-n", "--name",
default=None,
@@ -297,7 +461,13 @@ def get_args():
default=False,
dest="test",
action="store_true",
- help="Add Test category")
+ help="Add \"Test\" category")
+ optp.add_option(
+ "--other",
+ default=False,
+ dest="other",
+ action="store_true",
+ help="Send events having \"Other\" category (usually nonmalicious)")
optp.add_option(
"-o", "--oneshot",
default=False,
@@ -367,23 +537,43 @@ def process_data(line, filer, origdata, idea_gen):
:param idea_file: where output goes
:param origdata: if true, write original record to IDEA message
'''
- row = line.split(",")
- while re.search("^CVE-.*$", row[4]):
- row[3] += ", " + row[4]
- del row[4]
- examples_of_incident = []
- while not re.search("(UDP|IP|ICMP|DNS|IP6)", row[5]):
- examples_of_incident.append(row[5])
- del row[5]
- examples_of_incident = " ".join(examples_of_incident)
- category = idea_gen.convert_category(category=row[1], incident=row[4], examples_of_incident=examples_of_incident)
- if category and not_empty(row[0][-14:-3]):
- idea_event = idea_gen.gen_event_idea(timestamp=int(row[0][-14:-3]), category=category, id_taxonomy=not_empty(row[2]), cve=not_empty(row[3]),
- examples_of_incident=not_empty(examples_of_incident), proto=not_empty(row[5]),
+
+ ll_prot_str = [" ip:", " ipv6:", " udp:", " tcp:", "(ip)", "(ipv6)", "(udp)", "(tcp)", " ip ", " ipv6 ", " udp ", " tcp "]
+
+ row = line.split("|")
+
+ catprot = idea_gen.convert_category_and_protocol(category=row[1], id_taxonomy=int(row[2]))
+ proto = []
+ if not_empty(row[5]).lower() in ['ip', 'ip6', 'udp', 'tcp']:
+ proto.append(row[5].lower())
+
+ flt_proto = [pstr for pstr in ll_prot_str if(pstr in not_empty(row[4]).lower())]
+ if flt_proto:
+ proto.append(flt_proto[0].translate(None, " ():v"))
+
+ if catprot['protocol']:
+ proto.append(catprot['protocol'])
+ if set(proto).intersection({"ip", "ip6"}):
+ try:
+ proto.remove("ip")
+ proto.remove("ip6")
+ except ValueError:
+ pass
+ if not "udp" in proto:
+ proto.append("tcp")
+ proto = list(set(proto))
+
+ timestamp = row[0].split(" ")[-1]
+ cve = [i for i in row[3].split(",") if i not in ("null", "")]
+ odata = "|".join([timestamp] + row[1:])
+ if catprot['category'] and not_empty(row[0][-14:-3]):
+ idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=catprot['category'], id_taxonomy=int(row[2]),
+ cve=cve, filter_name=not_empty(row[4]), proto=proto,
src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
- conn_count=not_empty(row[10]), url=not_empty(row[12]),
- orig_data=str(line) if origdata else False, incident_desription=row[4])
- save_events(idea_event, filer)
+ conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
+ orig_data=odata if origdata else False)
+ if idea_event:
+ save_events(idea_event, filer)
running_flag = True
@@ -425,7 +615,7 @@ def main():
})
files = [FileWatcher(arg) for arg in args]
filer = Filer(opts.dir)
- idea_gen = IdeaGen(opts.name, opts.test)
+ idea_gen = IdeaGen(opts.name, opts.test, opts.other)
while running_flag:
for log_file in files:
while True: