Remake of TP taxonomy to IDEA category conversion, added new source/target...

Remake of TP taxonomy to IDEA category conversion, added new source/target protocol TP to IDEA (IANA) conversion and some other IDEA generation fixes.

tippingpoint/tpToIdea.py

-# python ./tpToIdea.py --origdata --oneshot -d /root/Dokumenty/PycharmProjects/TippingPoint/IdeaLogTest -n cz.cesnet.server.tippingpoint /root/Dokumenty/PycharmProjects/TippingPoint/tp-test-log.txt /root/Dokumenty/PycharmProjects/TippingPoint/tp-log2.txt --test
-
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2018 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
 import json
 from uuid import uuid4
@@ -13,6 +16,7 @@ import resource
 import os.path as pth
 import atexit
 import time
+from datetime import datetime
 import logging
 
 
@@ -134,91 +138,251 @@ class Filer(object):
 
 
 class IdeaGen(object):
-    idea_categories = {'Malware.Worm': re.compile("^Exploits: .*?Worm"),
-                       'Attempt.Login': re.compile("SipVicious Brute Force"),
-                       'Attempt.Exploit': re.compile("^Exploits: .*?Attempt"),
-                       'Other': re.compile("P2P"),
-                       'Recon.Scanning': re.compile("^Reconnaissance"),
-                       'Vulnerable.Open': re.compile("(^Vulnerabilities: [0-9]{5}|^Network Equipment: [0-9]{4} SNMP)"),
-                       'Vulnerable.Config': re.compile("^Vulnerabilities: [0-9]{4} ICMP")}
-    anomaly_re = re.compile("(^Network Equipment: [0-9]{4} IP|^Traffic Normalization: [0-9]{4} (?!Tunneling))")
-    availability_ddos_re = re.compile("^Traffic Normalization: ")
-
-    def __init__(self, name, test=False):
+
+    tp_tax_to_idea = {
+        1: {
+              1: ["Attempt.Exploit"],
+              2: ["Attempt.Exploit"],
+              3: ["Attempt.Exploit"],
+              4: ["Attempt.Exploit"],
+              5: ["Attempt.Exploit"],
+              6: ["Attempt.Exploit"],
+            255: ["Attempt.Exploit"]
+        },
+
+        2: {
+              1: ["Malware.Worm"],
+              2: ["Malware.Virus"],
+              3: ["Malware.Trojan"],
+              4: ["Intrusion.Botnet"],
+              5: ["Fraud.Phishing"],
+            255: ["Malware"]
+        },
+
+        3: {
+              1: ["Availability.DDoS"],
+              2: ["Availability.DDoS"],
+              3: ["Availability.DDoS"],
+            255: ["Availability.DDoS"]
+        },
+
+        4: {
+              1: ["Other"],
+              2: ["Other"],
+              3: ["Other"],
+              4: ["Other"],
+              5: ["Other"],
+              6: ["Attempt.Login"],
+              7: ["Malware.Spyware"],
+            255: ["Other"]
+        },
+
+        5: {
+              1: ["Recon.Scanning"],
+              2: ["Attempt.Exploit"],
+              3: ["Attempt.Exploit"],
+              4: ["Recon.Scanning", "Attempt.Exploit"],
+            255: ["Attempt.Exploit"]
+        },
+
+        6: {
+              1: ["Anomaly.Protocol"],
+              2: ["Anomaly.Traffic"],
+              3: ["Anomaly.Application"],
+            255: ["Anomaly"]
+        },
+
+        7: {
+              1: ["Anomaly.Traffic"],
+              2: ["Anomaly.Application"],
+            255: ["Anomaly.Traffic"]
+        },
+
+        8: {
+              1: ["Other"],
+              2: ["Other"],
+            255: ["Other"]
+        },
+    }
+
+    tp_prot_to_idea = {
+        1: "appletalk",
+        2: "auth",
+        3: "bgp",
+        4: "cdp",
+        5: "clns",
+        6: "dhcp",
+        7: "domain",
+        8: "finger",
+        9: "ftp",
+        10: "hsrp",
+        11: "http",
+        12: "icmp",
+        13: "igmp",
+        14: "eigrp",
+        15: "ipv6",
+        16: "ipx",
+        17: "irc",
+        18: "isis",
+        19: "isakmp",
+        20: "ldap",
+        21: "mpls",
+        22: "ms-rpc",
+        23: "ms-sql",
+        24: "nat",
+        25: "netbios",
+        26: "nntp",
+        27: "ntp",
+        28: "oracle",
+        29: "ospf",
+        30: "pop-imap",
+        31: "sunrpc",
+        32: "qos",
+        33: "rip",
+        34: "sunrpc",
+        35: "smb",
+        36: "smtp",
+        37: "snmp",
+        38: "sql",
+        39: "ssh",
+        40: "ssl-tls",
+        41: "tacacs",
+        42: "tcp",
+        43: "telnet",
+        45: "udp",
+        46: "uucp",
+        48: "x11",
+        49: "tftp",
+        50: "ip",
+        51: "nfs",
+        52: "wins",
+        80: "h323",
+        81: "megaco",
+        82: "mgcp",
+        83: "sip",
+        84: "rtp",
+        99: "voip",
+        100: "aim",
+        101: "msn",
+        102: "yahoo",
+        103: "icq",
+        119: "im",
+        120: "musicmatch",
+        121: "winamp",
+        122: "shoutcast",
+        123: "windows",
+        124: "quicktime",
+        125: "rtsp",
+        149: "streaming",
+        150: "bittorrent",
+        151: "manolito",
+        152: "directconnect",
+        153: "earthstation5",
+        154: "p2p",
+        155: "fasttrack",
+        156: "gnutella",
+        157: "twister",
+        158: "winmx",
+        180: "p2p",
+        190: "scada-dnp3",
+        191: "scada-iccp",
+        192: "scada-iec",
+        193: "scada-modbus",
+        194: "scada-opc",
+        199: "scada",
+        254: "multi-protocol",
+    }
+
+    def __init__(self, name, test=False, other=False):
         self.name = name
         self.test = test
+        self.other = other
 
-    def convert_category(self, category, incident, examples_of_incident):
+    def convert_category_and_protocol(self, category, id_taxonomy):
         '''
         converts category from record to IDEA category
         :param category: TippingPoint category description
-        :param incident: TippingPoint incident description
-        :param examples_of_incident: some TippingPoing incident descriptions has examples
+        :param id_taxonomy: TippingPoint taxonomy id
         :return: if category or incident is empty or is not important for saving it return None, otherwise return
                  converted category
         '''
-        if not (category and incident):
+        if not (category and id_taxonomy):
             return None
-        tp_category = category + ": " + incident
-        if IdeaGen.anomaly_re.search(tp_category) and not examples_of_incident:
-            return "Anomaly.Traffic"
-        if IdeaGen.availability_ddos_re.search(tp_category) and examples_of_incident:
-            return "Availability.DoS"
-        for category, pattern in IdeaGen.idea_categories.items():
-            if pattern.search(tp_category):
-               return category
-        return None
-
-    def gen_event_idea(self, timestamp, category, id_taxonomy, cve, examples_of_incident, proto, src_ip, src_port,
-                       dest_ip, dest_port, conn_count, url, orig_data, incident_desription):
+        tp_cat_maj = id_taxonomy >> 24
+        tp_cat_min = id_taxonomy >> 16 & 0b11111111
+        tp_prot = id_taxonomy >> 8 & 0b11111111
+        tp_platf = id_taxonomy & 0b11111111
+        try:
+            category = IdeaGen.tp_tax_to_idea[tp_cat_maj][tp_cat_min]
+        except KeyError:
+            category = ["Other"]
+
+        try:
+            protocol = IdeaGen.tp_prot_to_idea[tp_prot]
+        except KeyError:
+            protocol = None
+
+        return { 'category': category, 'protocol': protocol }
+
+    def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
+                       dest_ip, dest_port, conn_count, url, severity, orig_data):
         '''
         put every piece of record together into IDEA message
         :return: new IDEA message
         '''
+
+        if (category == ["Other"]) and not self.other:
+            return None
+
         event = {
             'Format': "IDEA0",
             'ID': str(uuid4()),
-            'DetectTime': time.strftime("%y-%m-%dT%H:%M:%S", time.localtime(timestamp)),
-            'Category': [category] + (["Test"] if self.test else []),
-            'Note': incident_desription,
-            'Ref': []
+            'DetectTime': datetime.utcfromtimestamp(timestamp / 1000).isoformat() + 'Z',
+            'Category': category + (["Test"] if self.test else [])
         }
         if cve:
-            event['Ref'].append("cve:"+cve)
-        if id_taxonomy:
-            event['Ref'].append("tipping_point_taxonomy:%d" % int(id_taxonomy))
-        if conn_count:
-            event['ConnCount'] = conn_count
+            event['Ref'] = ['urn:cve:'.format(i) for i in cve]
+        if conn_count and int(conn_count):
+            event['ConnCount'] = int(conn_count)
         source = {}
         target = {}
         if src_ip:
+            # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+            src_ip = src_ip.replace(" ", "")
             af = "IP4" if not ':' in src_ip else "IP6"
             source[af] = [src_ip]
-        if src_port:
-            source['Port'] = [src_port]
-        if proto:
-            source['Proto'] = [proto]
-        if url:
-            source['url'] = url
-        if dest_ip:
+            if src_port and int(src_port):
+                source['Port'] = [int(src_port)]
+            if proto:
+                source['Proto'] = proto
+        if dest_ip and (dest_ip != "0.0.0.0"):
+            # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+            dest_ip = dest_ip.replace(" ", "")
             af = "IP4" if not ':' in dest_ip else "IP6"
             target[af] = [dest_ip]
-        if dest_port:
-            target['Port'] = [dest_port]
+            if dest_port and int(dest_port):
+                target['Port'] = [int(dest_port)]
+            if proto:
+                target['Proto'] = proto
+            if url:
+                target['URL'] = url
+
         if source:
             event['Source'] = [source]
         if target:
-            event['Target'] = [target]
-        if examples_of_incident:
-            event['Note'] += ", examples of event:" + examples_of_incident[6:]
+            if category and category == "Reputation":
+                event['Source'].append(target)
+            else:
+                event['Target'] = [target]
         if orig_data:
-            event['Attach'] = {'Handle': "att1",
-                               'Note': "original data",
-                               'Content': orig_data}
+            event['Attach'] = [{
+                               'Type': ["OrigData"],
+                               'Content': orig_data.strip()
+            }]
         event['Node'] = [{
                             'Name': self.name,
-                            'Type': ["Connection", "Honeypot", "Recon"],
-                            'SW': ["HP Tipping Point"],
+                            'Type': ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"],
+                            'SW': ["TippingPoint_NX_NGIPS"]
                         }]
         return event
 
@@ -284,7 +448,7 @@ def daemonize(
 def get_args():
     optp = optparse.OptionParser(
         usage="usage: %prog [options] logfile ...",
-        description="Watch HP-Tipping_Point logfiles and generate Idea events into directory")
+        description="Watch TippingPoint logfiles and generate Idea events into directory")
     optp.add_option(
         "-n", "--name",
         default=None,
@@ -297,7 +461,13 @@ def get_args():
         default=False,
         dest="test",
         action="store_true",
-        help="Add Test category")
+        help="Add \"Test\" category")
+    optp.add_option(
+        "--other",
+        default=False,
+        dest="other",
+        action="store_true",
+        help="Send events having \"Other\" category (usually nonmalicious)")
     optp.add_option(
         "-o", "--oneshot",
         default=False,
@@ -367,23 +537,43 @@ def process_data(line, filer, origdata, idea_gen):
     :param idea_file: where output goes
     :param origdata: if true, write original record to IDEA message
     '''
-    row = line.split(",")
-    while re.search("^CVE-.*$", row[4]):
-        row[3] += ", " + row[4]
-        del row[4]
-    examples_of_incident = []
-    while not re.search("(UDP|IP|ICMP|DNS|IP6)", row[5]):
-        examples_of_incident.append(row[5])
-        del row[5]
-    examples_of_incident = " ".join(examples_of_incident)
-    category = idea_gen.convert_category(category=row[1], incident=row[4], examples_of_incident=examples_of_incident)
-    if category and not_empty(row[0][-14:-3]):
-        idea_event = idea_gen.gen_event_idea(timestamp=int(row[0][-14:-3]), category=category, id_taxonomy=not_empty(row[2]), cve=not_empty(row[3]),
-                                             examples_of_incident=not_empty(examples_of_incident), proto=not_empty(row[5]),
+
+    ll_prot_str = [" ip:", " ipv6:", " udp:", " tcp:", "(ip)", "(ipv6)", "(udp)", "(tcp)", " ip ", " ipv6 ", " udp ", " tcp "]
+
+    row = line.split("|")
+
+    catprot = idea_gen.convert_category_and_protocol(category=row[1], id_taxonomy=int(row[2]))
+    proto = []
+    if not_empty(row[5]).lower() in ['ip', 'ip6', 'udp', 'tcp']:
+        proto.append(row[5].lower())
+
+    flt_proto = [pstr for pstr in ll_prot_str if(pstr in not_empty(row[4]).lower())]
+    if flt_proto:
+        proto.append(flt_proto[0].translate(None, " ():v"))
+
+    if catprot['protocol']:
+        proto.append(catprot['protocol'])
+    if set(proto).intersection({"ip", "ip6"}):
+        try:
+            proto.remove("ip")
+            proto.remove("ip6")
+        except ValueError:
+            pass
+        if not "udp" in proto:
+            proto.append("tcp")
+    proto = list(set(proto))
+
+    timestamp = row[0].split(" ")[-1]
+    cve = [i for i in row[3].split(",") if i not in ("null", "")]
+    odata = "|".join([timestamp] + row[1:])
+    if catprot['category'] and not_empty(row[0][-14:-3]):
+        idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=catprot['category'], id_taxonomy=int(row[2]),
+                                             cve=cve, filter_name=not_empty(row[4]), proto=proto,
                                              src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
-                                             conn_count=not_empty(row[10]), url=not_empty(row[12]),
-                                             orig_data=str(line) if origdata else False, incident_desription=row[4])
-        save_events(idea_event, filer)
+                                             conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
+                                             orig_data=odata if origdata else False)
+        if idea_event:
+            save_events(idea_event, filer)
 
 
 running_flag = True
@@ -425,7 +615,7 @@ def main():
             })
         files = [FileWatcher(arg) for arg in args]
     filer = Filer(opts.dir)
-    idea_gen = IdeaGen(opts.name, opts.test)
+    idea_gen = IdeaGen(opts.name, opts.test, opts.other)
     while running_flag:
         for log_file in files:
             while True: