TippingPoint connector - remake of id_taxonomy conversion to IDEA. Also...

TippingPoint connector - remake of id_taxonomy conversion to IDEA. Also corrected some mistakes in IDEA conversion and done some refractoring.

hp-tipping-point/tpToIdea.py

-# python ./tpToIdea.py --origdata --oneshot -d /root/Dokumenty/PycharmProjects/TippingPoint/IdeaLogTest -n cz.cesnet.server.tippingpoint /root/Dokumenty/PycharmProjects/TippingPoint/tp-test-log.txt /root/Dokumenty/PycharmProjects/TippingPoint/tp-log2.txt --test
-
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2018 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
 import json
 from uuid import uuid4
@@ -13,7 +16,9 @@ import resource
 import os.path as pth
 import atexit
 import time
+from datetime import datetime
 import logging
+from collections import OrderedDict
 
 
 class FileWatcher(object):
@@ -134,92 +139,155 @@ class Filer(object):
 
 
 class IdeaGen(object):
-    idea_categories = {'Malware.Worm': re.compile("^Exploits: .*?Worm"),
-                       'Attempt.Login': re.compile("SipVicious Brute Force"),
-                       'Attempt.Exploit': re.compile("^Exploits: .*?Attempt"),
-                       'Other': re.compile("P2P"),
-                       'Recon.Scanning': re.compile("^Reconnaissance"),
-                       'Vulnerable.Open': re.compile("(^Vulnerabilities: [0-9]{5}|^Network Equipment: [0-9]{4} SNMP)"),
-                       'Vulnerable.Config': re.compile("^Vulnerabilities: [0-9]{4} ICMP")}
-    anomaly_re = re.compile("(^Network Equipment: [0-9]{4} IP|^Traffic Normalization: [0-9]{4} (?!Tunneling))")
-    availability_ddos_re = re.compile("^Traffic Normalization: ")
-
-    def __init__(self, name, test=False):
+
+    tp_to_idea = {
+        1: {
+              1: ["Attempt.Exploit"],
+              2: ["Attempt.Exploit"],
+              3: ["Attempt.Exploit"],
+              4: ["Attempt.Exploit"],
+              5: ["Attempt.Exploit"],
+              6: ["Attempt.Exploit"],
+            255: ["Attempt.Exploit"]
+        },
+
+        2: {
+              1: ["Malware.Worm"],
+              2: ["Malware.Virus"],
+              3: ["Malware.Trojan"],
+              4: ["Intrusion.Botnet"],
+              5: ["Fraud.Phishing"],
+            255: ["Malware"]
+        },
+
+        3: {
+              1: ["Availability.DDoS"],
+              2: ["Availability.DDoS"],
+              3: ["Availability.DDoS"],
+            255: ["Availability.DDoS"]
+        },
+
+        4: {
+              1: ["Other"],
+              2: ["Other"],
+              3: ["Other"],
+              4: ["Other"],
+              5: ["Other"],
+              6: ["Attempt.Login"],
+              7: ["Malware.Spyware"],
+            255: ["Other"]
+        },
+
+        5: {
+              1: ["Recon.Scanning"],
+              2: ["Attempt.Exploit"],
+              3: ["Attempt.Exploit"],
+              4: ["Recon.Scanning", "Attempt.Exploit"],
+            255: ["Attempt.Exploit"]
+        },
+
+        6: {
+              1: ["Anomaly.Protocol"],
+              2: ["Anomaly.Traffic"],
+              3: ["Anomaly.Application"],
+            255: ["Anomaly"]
+        },
+
+        7: {
+              1: ["Anomaly.Traffic"],
+              2: ["Anomaly.Application"],
+            255: ["Anomaly.Traffic"]
+        },
+
+        8: {
+              1: ["Other"],
+              2: ["Other"],
+            255: ["Other"]
+        },
+    }
+
+    def __init__(self, name, test=False, other=False):
         self.name = name
         self.test = test
+        self.other = other
 
-    def convert_category(self, category, incident, examples_of_incident):
+    def convert_category(self, category, id_taxonomy):
         '''
         converts category from record to IDEA category
         :param category: TippingPoint category description
-        :param incident: TippingPoint incident description
-        :param examples_of_incident: some TippingPoing incident descriptions has examples
+        :param id_taxonomy: TippingPoint taxonomy id
         :return: if category or incident is empty or is not important for saving it return None, otherwise return
                  converted category
         '''
-        if not (category and incident):
+        if not (category and id_taxonomy):
             return None
-        tp_category = category + ": " + incident
-        if IdeaGen.anomaly_re.search(tp_category) and not examples_of_incident:
-            return "Anomaly.Traffic"
-        if IdeaGen.availability_ddos_re.search(tp_category) and examples_of_incident:
-            return "Availability.DoS"
-        for category, pattern in IdeaGen.idea_categories.items():
-            if pattern.search(tp_category):
-               return category
-        return None
-
-    def gen_event_idea(self, timestamp, category, id_taxonomy, cve, examples_of_incident, proto, src_ip, src_port,
-                       dest_ip, dest_port, conn_count, url, orig_data, incident_desription):
+        tp_cat_maj = id_taxonomy >> 24
+        tp_cat_min = id_taxonomy >> 16 & 0b11111111
+        tp_proto = id_taxonomy >> 8 & 0b11111111
+        tp_platf = id_taxonomy & 0b11111111
+        try:
+            category = IdeaGen.tp_to_idea[tp_cat_maj][tp_cat_min]
+        except KeyError:
+            category = ["Other"]
+
+        return category
+
+    def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
+                       dest_ip, dest_port, conn_count, url, severity, orig_data):
         '''
         put every piece of record together into IDEA message
         :return: new IDEA message
         '''
-        event = {
-            'Format': "IDEA0",
-            'ID': str(uuid4()),
-            'DetectTime': time.strftime("%y-%m-%dT%H:%M:%S", time.localtime(timestamp)),
-            'Category': [category] + (["Test"] if self.test else []),
-            'Note': incident_desription,
-            'Ref': []
-        }
+
+        if (category == ["Other"]) and not self.other:
+            return None
+
+        event = OrderedDict([
+            ("Format","IDEA0"),
+            ("ID", str(uuid4())),
+            ("DetectTime", datetime.fromtimestamp(timestamp / 1000).isoformat() + 'Z'),
+            ("Category", category + (["Test"] if self.test else [])),
+        ])
         if cve:
-            event['Ref'].append("cve:"+cve)
-        if id_taxonomy:
-            event['Ref'].append("tipping_point_taxonomy:%d" % int(id_taxonomy))
-        if conn_count:
-            event['ConnCount'] = conn_count
-        source = {}
-        target = {}
+            event['Ref'] = ['urn:cve:'.format(i) for i in cve]
+        if conn_count and int(conn_count):
+            event['ConnCount'] = int(conn_count)
+        source = OrderedDict()
+        target = OrderedDict()
         if src_ip:
+            # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+            src_ip = src_ip.replace(" ", "")
             af = "IP4" if not ':' in src_ip else "IP6"
             source[af] = [src_ip]
-        if src_port:
-            source['Port'] = [src_port]
+        if src_port and int(src_port):
+            source['Port'] = [int(src_port)]
         if proto:
             source['Proto'] = [proto]
-        if url:
-            source['url'] = url
-        if dest_ip:
+        if dest_ip and (dest_ip != "0.0.0.0"):
+            # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+            dest_ip = dest_ip.replace(" ", "")
             af = "IP4" if not ':' in dest_ip else "IP6"
             target[af] = [dest_ip]
-        if dest_port:
-            target['Port'] = [dest_port]
+            if dest_port and int(dest_port):
+                target['Port'] = [int(dest_port)]
+            if proto:
+                target['Proto'] = [proto]
+            if url:
+                target['URL'] = url
         if source:
             event['Source'] = [source]
         if target:
             event['Target'] = [target]
-        if examples_of_incident:
-            event['Note'] += ", examples of event:" + examples_of_incident[6:]
         if orig_data:
-            event['Attach'] = {'Handle': "att1",
-                               'Note': "original data",
-                               'Content': orig_data}
-        event['Node'] = [{
-                            'Name': self.name,
-                            'Type': ["Connection", "Honeypot", "Recon"],
-                            'SW': ["HP Tipping Point"],
-                        }]
+            event['Attach'] = [OrderedDict([
+                               ('Type', ["OrigData"]),
+                               ('Content', orig_data.strip())
+            ])]
+        event['Node'] = [OrderedDict([
+                            ('Name', self.name),
+                            ('Type', ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"]),
+                            ('SW', ["TippingPoint_NX_NGIPS"])
+                        ])]
         return event
 
 
@@ -284,7 +352,7 @@ def daemonize(
 def get_args():
     optp = optparse.OptionParser(
         usage="usage: %prog [options] logfile ...",
-        description="Watch HP-Tipping_Point logfiles and generate Idea events into directory")
+        description="Watch TippingPoint logfiles and generate Idea events into directory")
     optp.add_option(
         "-n", "--name",
         default=None,
@@ -297,7 +365,13 @@ def get_args():
         default=False,
         dest="test",
         action="store_true",
-        help="Add Test category")
+        help="Add \"Test\" category")
+    optp.add_option(
+        "--other",
+        default=False,
+        dest="other",
+        action="store_true",
+        help="Send events having \"Other\" category (usually nonmalicious)")
     optp.add_option(
         "-o", "--oneshot",
         default=False,
@@ -367,23 +441,19 @@ def process_data(line, filer, origdata, idea_gen):
     :param idea_file: where output goes
     :param origdata: if true, write original record to IDEA message
     '''
-    row = line.split(",")
-    while re.search("^CVE-.*$", row[4]):
-        row[3] += ", " + row[4]
-        del row[4]
-    examples_of_incident = []
-    while not re.search("(UDP|IP|ICMP|DNS|IP6)", row[5]):
-        examples_of_incident.append(row[5])
-        del row[5]
-    examples_of_incident = " ".join(examples_of_incident)
-    category = idea_gen.convert_category(category=row[1], incident=row[4], examples_of_incident=examples_of_incident)
+    row = line.split("|")
+    category = idea_gen.convert_category(category=row[1], id_taxonomy=int(row[2]))
+    timestamp = row[0].split(" ")[-1]
+    cve = [i for i in row[3].split(",") if i not in ("null", "")]
+    odata = "|".join([timestamp] + row[1:])
     if category and not_empty(row[0][-14:-3]):
-        idea_event = idea_gen.gen_event_idea(timestamp=int(row[0][-14:-3]), category=category, id_taxonomy=not_empty(row[2]), cve=not_empty(row[3]),
-                                             examples_of_incident=not_empty(examples_of_incident), proto=not_empty(row[5]),
+        idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=category, id_taxonomy=int(row[2]),
+                                             cve=cve, filter_name=not_empty(row[4]), proto=not_empty(row[5]),
                                              src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
-                                             conn_count=not_empty(row[10]), url=not_empty(row[12]),
-                                             orig_data=str(line) if origdata else False, incident_desription=row[4])
-        save_events(idea_event, filer)
+                                             conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
+                                             orig_data=odata if origdata else False)
+        if idea_event:
+            save_events(idea_event, filer)
 
 
 running_flag = True
@@ -425,7 +495,7 @@ def main():
             })
         files = [FileWatcher(arg) for arg in args]
     filer = Filer(opts.dir)
-    idea_gen = IdeaGen(opts.name, opts.test)
+    idea_gen = IdeaGen(opts.name, opts.test, opts.other)
     while running_flag:
         for log_file in files:
             while True: