diff --git a/hp-tipping-point/tpToIdea.py b/hp-tipping-point/tpToIdea.py
index 1d723790f1ce7fcbc3db5ea0461d75ea5b7fb034..3610e169db4ae5b2f728a496a6453d3740bb076d 100644
--- a/hp-tipping-point/tpToIdea.py
+++ b/hp-tipping-point/tpToIdea.py
@@ -1,5 +1,8 @@
-# python ./tpToIdea.py --origdata --oneshot -d /root/Dokumenty/PycharmProjects/TippingPoint/IdeaLogTest -n cz.cesnet.server.tippingpoint /root/Dokumenty/PycharmProjects/TippingPoint/tp-test-log.txt /root/Dokumenty/PycharmProjects/TippingPoint/tp-log2.txt --test
-
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2018 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
import json
from uuid import uuid4
@@ -13,7 +16,9 @@ import resource
import os.path as pth
import atexit
import time
+from datetime import datetime
import logging
+from collections import OrderedDict
class FileWatcher(object):
@@ -134,92 +139,155 @@ class Filer(object):
class IdeaGen(object):
- idea_categories = {'Malware.Worm': re.compile("^Exploits: .*?Worm"),
- 'Attempt.Login': re.compile("SipVicious Brute Force"),
- 'Attempt.Exploit': re.compile("^Exploits: .*?Attempt"),
- 'Other': re.compile("P2P"),
- 'Recon.Scanning': re.compile("^Reconnaissance"),
- 'Vulnerable.Open': re.compile("(^Vulnerabilities: [0-9]{5}|^Network Equipment: [0-9]{4} SNMP)"),
- 'Vulnerable.Config': re.compile("^Vulnerabilities: [0-9]{4} ICMP")}
- anomaly_re = re.compile("(^Network Equipment: [0-9]{4} IP|^Traffic Normalization: [0-9]{4} (?!Tunneling))")
- availability_ddos_re = re.compile("^Traffic Normalization: ")
-
- def __init__(self, name, test=False):
+
+ tp_to_idea = {
+ 1: {
+ 1: ["Attempt.Exploit"],
+ 2: ["Attempt.Exploit"],
+ 3: ["Attempt.Exploit"],
+ 4: ["Attempt.Exploit"],
+ 5: ["Attempt.Exploit"],
+ 6: ["Attempt.Exploit"],
+ 255: ["Attempt.Exploit"]
+ },
+
+ 2: {
+ 1: ["Malware.Worm"],
+ 2: ["Malware.Virus"],
+ 3: ["Malware.Trojan"],
+ 4: ["Intrusion.Botnet"],
+ 5: ["Fraud.Phishing"],
+ 255: ["Malware"]
+ },
+
+ 3: {
+ 1: ["Availability.DDoS"],
+ 2: ["Availability.DDoS"],
+ 3: ["Availability.DDoS"],
+ 255: ["Availability.DDoS"]
+ },
+
+ 4: {
+ 1: ["Other"],
+ 2: ["Other"],
+ 3: ["Other"],
+ 4: ["Other"],
+ 5: ["Other"],
+ 6: ["Attempt.Login"],
+ 7: ["Malware.Spyware"],
+ 255: ["Other"]
+ },
+
+ 5: {
+ 1: ["Recon.Scanning"],
+ 2: ["Attempt.Exploit"],
+ 3: ["Attempt.Exploit"],
+ 4: ["Recon.Scanning", "Attempt.Exploit"],
+ 255: ["Attempt.Exploit"]
+ },
+
+ 6: {
+ 1: ["Anomaly.Protocol"],
+ 2: ["Anomaly.Traffic"],
+ 3: ["Anomaly.Application"],
+ 255: ["Anomaly"]
+ },
+
+ 7: {
+ 1: ["Anomaly.Traffic"],
+ 2: ["Anomaly.Application"],
+ 255: ["Anomaly.Traffic"]
+ },
+
+ 8: {
+ 1: ["Other"],
+ 2: ["Other"],
+ 255: ["Other"]
+ },
+ }
+
+ def __init__(self, name, test=False, other=False):
self.name = name
self.test = test
+ self.other = other
- def convert_category(self, category, incident, examples_of_incident):
+ def convert_category(self, category, id_taxonomy):
'''
converts category from record to IDEA category
:param category: TippingPoint category description
- :param incident: TippingPoint incident description
- :param examples_of_incident: some TippingPoing incident descriptions has examples
+ :param id_taxonomy: TippingPoint taxonomy id
:return: if category or incident is empty or is not important for saving it return None, otherwise return
converted category
'''
- if not (category and incident):
+ if not (category and id_taxonomy):
return None
- tp_category = category + ": " + incident
- if IdeaGen.anomaly_re.search(tp_category) and not examples_of_incident:
- return "Anomaly.Traffic"
- if IdeaGen.availability_ddos_re.search(tp_category) and examples_of_incident:
- return "Availability.DoS"
- for category, pattern in IdeaGen.idea_categories.items():
- if pattern.search(tp_category):
- return category
- return None
-
- def gen_event_idea(self, timestamp, category, id_taxonomy, cve, examples_of_incident, proto, src_ip, src_port,
- dest_ip, dest_port, conn_count, url, orig_data, incident_desription):
+ tp_cat_maj = id_taxonomy >> 24
+ tp_cat_min = id_taxonomy >> 16 & 0b11111111
+ tp_proto = id_taxonomy >> 8 & 0b11111111
+ tp_platf = id_taxonomy & 0b11111111
+ try:
+ category = IdeaGen.tp_to_idea[tp_cat_maj][tp_cat_min]
+ except KeyError:
+ category = ["Other"]
+
+ return category
+
+ def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
+ dest_ip, dest_port, conn_count, url, severity, orig_data):
'''
put every piece of record together into IDEA message
:return: new IDEA message
'''
- event = {
- 'Format': "IDEA0",
- 'ID': str(uuid4()),
- 'DetectTime': time.strftime("%y-%m-%dT%H:%M:%S", time.localtime(timestamp)),
- 'Category': [category] + (["Test"] if self.test else []),
- 'Note': incident_desription,
- 'Ref': []
- }
+
+ if (category == ["Other"]) and not self.other:
+ return None
+
+ event = OrderedDict([
+ ("Format","IDEA0"),
+ ("ID", str(uuid4())),
+ ("DetectTime", datetime.fromtimestamp(timestamp / 1000).isoformat() + 'Z'),
+ ("Category", category + (["Test"] if self.test else [])),
+ ])
if cve:
- event['Ref'].append("cve:"+cve)
- if id_taxonomy:
- event['Ref'].append("tipping_point_taxonomy:%d" % int(id_taxonomy))
- if conn_count:
- event['ConnCount'] = conn_count
- source = {}
- target = {}
+ event['Ref'] = ['urn:cve:'.format(i) for i in cve]
+ if conn_count and int(conn_count):
+ event['ConnCount'] = int(conn_count)
+ source = OrderedDict()
+ target = OrderedDict()
if src_ip:
+ # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+ src_ip = src_ip.replace(" ", "")
af = "IP4" if not ':' in src_ip else "IP6"
source[af] = [src_ip]
- if src_port:
- source['Port'] = [src_port]
+ if src_port and int(src_port):
+ source['Port'] = [int(src_port)]
if proto:
source['Proto'] = [proto]
- if url:
- source['url'] = url
- if dest_ip:
+ if dest_ip and (dest_ip != "0.0.0.0"):
+ # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+ dest_ip = dest_ip.replace(" ", "")
af = "IP4" if not ':' in dest_ip else "IP6"
target[af] = [dest_ip]
- if dest_port:
- target['Port'] = [dest_port]
+ if dest_port and int(dest_port):
+ target['Port'] = [int(dest_port)]
+ if proto:
+ target['Proto'] = [proto]
+ if url:
+ target['URL'] = url
if source:
event['Source'] = [source]
if target:
event['Target'] = [target]
- if examples_of_incident:
- event['Note'] += ", examples of event:" + examples_of_incident[6:]
if orig_data:
- event['Attach'] = {'Handle': "att1",
- 'Note': "original data",
- 'Content': orig_data}
- event['Node'] = [{
- 'Name': self.name,
- 'Type': ["Connection", "Honeypot", "Recon"],
- 'SW': ["HP Tipping Point"],
- }]
+ event['Attach'] = [OrderedDict([
+ ('Type', ["OrigData"]),
+ ('Content', orig_data.strip())
+ ])]
+ event['Node'] = [OrderedDict([
+ ('Name', self.name),
+ ('Type', ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"]),
+ ('SW', ["TippingPoint_NX_NGIPS"])
+ ])]
return event
@@ -284,7 +352,7 @@ def daemonize(
def get_args():
optp = optparse.OptionParser(
usage="usage: %prog [options] logfile ...",
- description="Watch HP-Tipping_Point logfiles and generate Idea events into directory")
+ description="Watch TippingPoint logfiles and generate Idea events into directory")
optp.add_option(
"-n", "--name",
default=None,
@@ -297,7 +365,13 @@ def get_args():
default=False,
dest="test",
action="store_true",
- help="Add Test category")
+ help="Add \"Test\" category")
+ optp.add_option(
+ "--other",
+ default=False,
+ dest="other",
+ action="store_true",
+ help="Send events having \"Other\" category (usually nonmalicious)")
optp.add_option(
"-o", "--oneshot",
default=False,
@@ -367,23 +441,19 @@ def process_data(line, filer, origdata, idea_gen):
:param idea_file: where output goes
:param origdata: if true, write original record to IDEA message
'''
- row = line.split(",")
- while re.search("^CVE-.*$", row[4]):
- row[3] += ", " + row[4]
- del row[4]
- examples_of_incident = []
- while not re.search("(UDP|IP|ICMP|DNS|IP6)", row[5]):
- examples_of_incident.append(row[5])
- del row[5]
- examples_of_incident = " ".join(examples_of_incident)
- category = idea_gen.convert_category(category=row[1], incident=row[4], examples_of_incident=examples_of_incident)
+ row = line.split("|")
+ category = idea_gen.convert_category(category=row[1], id_taxonomy=int(row[2]))
+ timestamp = row[0].split(" ")[-1]
+ cve = [i for i in row[3].split(",") if i not in ("null", "")]
+ odata = "|".join([timestamp] + row[1:])
if category and not_empty(row[0][-14:-3]):
- idea_event = idea_gen.gen_event_idea(timestamp=int(row[0][-14:-3]), category=category, id_taxonomy=not_empty(row[2]), cve=not_empty(row[3]),
- examples_of_incident=not_empty(examples_of_incident), proto=not_empty(row[5]),
+ idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=category, id_taxonomy=int(row[2]),
+ cve=cve, filter_name=not_empty(row[4]), proto=not_empty(row[5]),
src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
- conn_count=not_empty(row[10]), url=not_empty(row[12]),
- orig_data=str(line) if origdata else False, incident_desription=row[4])
- save_events(idea_event, filer)
+ conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
+ orig_data=odata if origdata else False)
+ if idea_event:
+ save_events(idea_event, filer)
running_flag = True
@@ -425,7 +495,7 @@ def main():
})
files = [FileWatcher(arg) for arg in args]
filer = Filer(opts.dir)
- idea_gen = IdeaGen(opts.name, opts.test)
+ idea_gen = IdeaGen(opts.name, opts.test, opts.other)
while running_flag:
for log_file in files:
while True: