diff --git a/hp-tipping-point/tpToIdea.py b/hp-tipping-point/tpToIdea.py
new file mode 100644
index 0000000000000000000000000000000000000000..3610e169db4ae5b2f728a496a6453d3740bb076d
--- /dev/null
+++ b/hp-tipping-point/tpToIdea.py
@@ -0,0 +1,522 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2017-2018 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+import json
+from uuid import uuid4
+import re
+import socket
+import optparse
+import sys
+import os
+import signal
+import resource
+import os.path as pth
+import atexit
+import time
+from datetime import datetime
+import logging
+from collections import OrderedDict
+
+
+class FileWatcher(object):
+
+ def __init__(self, filename, tail=True):
+ self.filename = filename
+ self.open()
+ self.line_buffer = ""
+ if tail and self.f:
+ self.f.seek(0, os.SEEK_END)
+
+ def open(self):
+ try:
+ self.f = open(self.filename, "r")
+ st = os.fstat(self.f.fileno())
+ self.inode, self.size = st.st_ino, st.st_size
+ except IOError:
+ self.f = None
+ self.inode = -1
+ self.size = -1
+
+ def _check_reopen(self):
+ try:
+ st = os.stat(self.filename)
+ cur_inode, cur_size = st.st_ino, st.st_size
+ except OSError as e:
+ cur_inode = -1
+ cur_size = -1
+ if cur_inode != self.inode or cur_size < self.size:
+ self.close()
+ self.open()
+
+ def readline(self):
+ if not self.f:
+ self.open()
+ if not self.f:
+ return self.line_buffer
+ res = self.f.readline()
+ if not res:
+ self._check_reopen()
+ if not self.f:
+ return self.line_buffer
+ res = self.f.readline()
+ if not res.endswith("\n"):
+ self.line_buffer += res
+ else:
+ res = self.line_buffer + res
+ self.line_buffer = ""
+ return res
+
+ def close(self):
+ try:
+ if self.f:
+ self.f.close()
+ except IOError:
+ pass
+ self.inode = -1
+ self.size = -1
+
+ def __repr__(self):
+ return '%s("%s")' % (type(self).__name__, self.filename)
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.close()
+ return False
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ return self.readline()
+
+
+class Filer(object):
+
+ def __init__(self, directory):
+ self.basedir = self._ensure_path(directory)
+ self.tmp = self._ensure_path(pth.join(self.basedir, "tmp"))
+ self.incoming = self._ensure_path(pth.join(self.basedir, "incoming"))
+ self.hostname = socket.gethostname()
+ self.pid = os.getpid()
+
+ def _ensure_path(self, p):
+ try:
+ os.mkdir(p)
+ except OSError:
+ if not pth.isdir(p):
+ raise
+ return p
+
+ def _get_new_name(self, fd=None):
+ (inode, device) = os.fstat(fd)[1:3] if fd else (0, 0)
+ return "%s.%d.%f.%d.%d" % (
+ self.hostname, self.pid, time.time(), device, inode)
+
+ def create_unique_file(self):
+ # First find and open name unique within tmp
+ tmpname = None
+ while not tmpname:
+ tmpname = self._get_new_name()
+ try:
+ fd = os.open(pth.join(self.tmp, tmpname), os.O_CREAT | os.O_RDWR | os.O_EXCL)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise # other errors than duplicates should get noticed
+ tmpname = None
+ # Now we know the device/inode, rename to make unique within system
+ newname = self._get_new_name(fd)
+ os.rename(pth.join(self.tmp, tmpname), pth.join(self.tmp, newname))
+ nf = os.fdopen(fd, "w")
+ return nf, newname
+
+ def publish_file(self, short_name):
+ os.rename(pth.join(self.tmp, short_name), pth.join(self.incoming, short_name))
+
+
+class IdeaGen(object):
+
+ tp_to_idea = {
+ 1: {
+ 1: ["Attempt.Exploit"],
+ 2: ["Attempt.Exploit"],
+ 3: ["Attempt.Exploit"],
+ 4: ["Attempt.Exploit"],
+ 5: ["Attempt.Exploit"],
+ 6: ["Attempt.Exploit"],
+ 255: ["Attempt.Exploit"]
+ },
+
+ 2: {
+ 1: ["Malware.Worm"],
+ 2: ["Malware.Virus"],
+ 3: ["Malware.Trojan"],
+ 4: ["Intrusion.Botnet"],
+ 5: ["Fraud.Phishing"],
+ 255: ["Malware"]
+ },
+
+ 3: {
+ 1: ["Availability.DDoS"],
+ 2: ["Availability.DDoS"],
+ 3: ["Availability.DDoS"],
+ 255: ["Availability.DDoS"]
+ },
+
+ 4: {
+ 1: ["Other"],
+ 2: ["Other"],
+ 3: ["Other"],
+ 4: ["Other"],
+ 5: ["Other"],
+ 6: ["Attempt.Login"],
+ 7: ["Malware.Spyware"],
+ 255: ["Other"]
+ },
+
+ 5: {
+ 1: ["Recon.Scanning"],
+ 2: ["Attempt.Exploit"],
+ 3: ["Attempt.Exploit"],
+ 4: ["Recon.Scanning", "Attempt.Exploit"],
+ 255: ["Attempt.Exploit"]
+ },
+
+ 6: {
+ 1: ["Anomaly.Protocol"],
+ 2: ["Anomaly.Traffic"],
+ 3: ["Anomaly.Application"],
+ 255: ["Anomaly"]
+ },
+
+ 7: {
+ 1: ["Anomaly.Traffic"],
+ 2: ["Anomaly.Application"],
+ 255: ["Anomaly.Traffic"]
+ },
+
+ 8: {
+ 1: ["Other"],
+ 2: ["Other"],
+ 255: ["Other"]
+ },
+ }
+
+ def __init__(self, name, test=False, other=False):
+ self.name = name
+ self.test = test
+ self.other = other
+
+ def convert_category(self, category, id_taxonomy):
+ '''
+ converts category from record to IDEA category
+ :param category: TippingPoint category description
+ :param id_taxonomy: TippingPoint taxonomy id
+ :return: if category or incident is empty or is not important for saving it return None, otherwise return
+ converted category
+ '''
+ if not (category and id_taxonomy):
+ return None
+ tp_cat_maj = id_taxonomy >> 24
+ tp_cat_min = id_taxonomy >> 16 & 0b11111111
+ tp_proto = id_taxonomy >> 8 & 0b11111111
+ tp_platf = id_taxonomy & 0b11111111
+ try:
+ category = IdeaGen.tp_to_idea[tp_cat_maj][tp_cat_min]
+ except KeyError:
+ category = ["Other"]
+
+ return category
+
+ def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
+ dest_ip, dest_port, conn_count, url, severity, orig_data):
+ '''
+ put every piece of record together into IDEA message
+ :return: new IDEA message
+ '''
+
+ if (category == ["Other"]) and not self.other:
+ return None
+
+ event = OrderedDict([
+ ("Format","IDEA0"),
+ ("ID", str(uuid4())),
+ ("DetectTime", datetime.fromtimestamp(timestamp / 1000).isoformat() + 'Z'),
+ ("Category", category + (["Test"] if self.test else [])),
+ ])
+ if cve:
+ event['Ref'] = ['urn:cve:'.format(i) for i in cve]
+ if conn_count and int(conn_count):
+ event['ConnCount'] = int(conn_count)
+ source = OrderedDict()
+ target = OrderedDict()
+ if src_ip:
+ # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+ src_ip = src_ip.replace(" ", "")
+ af = "IP4" if not ':' in src_ip else "IP6"
+ source[af] = [src_ip]
+ if src_port and int(src_port):
+ source['Port'] = [int(src_port)]
+ if proto:
+ source['Proto'] = [proto]
+ if dest_ip and (dest_ip != "0.0.0.0"):
+ # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
+ dest_ip = dest_ip.replace(" ", "")
+ af = "IP4" if not ':' in dest_ip else "IP6"
+ target[af] = [dest_ip]
+ if dest_port and int(dest_port):
+ target['Port'] = [int(dest_port)]
+ if proto:
+ target['Proto'] = [proto]
+ if url:
+ target['URL'] = url
+ if source:
+ event['Source'] = [source]
+ if target:
+ event['Target'] = [target]
+ if orig_data:
+ event['Attach'] = [OrderedDict([
+ ('Type', ["OrigData"]),
+ ('Content', orig_data.strip())
+ ])]
+ event['Node'] = [OrderedDict([
+ ('Name', self.name),
+ ('Type', ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"]),
+ ('SW', ["TippingPoint_NX_NGIPS"])
+ ])]
+ return event
+
+
+def daemonize(
+ work_dir=None, chroot_dir=None,
+ umask=None, uid=None, gid=None,
+ pidfile=None, files_preserve=[], signals={}):
+ # Dirs, limits, users
+ if chroot_dir is not None:
+ os.chdir(chroot_dir)
+ os.chroot(chroot_dir)
+ if umask is not None:
+ os.umask(umask)
+ if work_dir is not None:
+ os.chdir(work_dir)
+ if gid is not None:
+ os.setgid(gid)
+ if uid is not None:
+ os.setuid(uid)
+ # Doublefork, split session
+ if os.fork() > 0:
+ os._exit(0)
+ try:
+ os.setsid()
+ except OSError:
+ pass
+ if os.fork() > 0:
+ os._exit(0)
+ # Setup signal handlers
+ for (signum, handler) in signals.items():
+ signal.signal(signum, handler)
+ # Close descriptors
+ descr_preserve = set(f.fileno() for f in files_preserve)
+ maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+ if maxfd == resource.RLIM_INFINITY:
+ maxfd = 65535
+ for fd in range(maxfd, 3, -1): # 3 means omit stdin, stdout, stderr
+ if fd not in descr_preserve:
+ try:
+ os.close(fd)
+ except Exception:
+ pass
+ # Redirect stdin, stdout, stderr to /dev/null
+ devnull = os.open(os.devnull, os.O_RDWR)
+ for fd in range(3):
+ os.dup2(devnull, fd)
+ # PID file
+ if pidfile is not None:
+ pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
+ os.write(pidd, str(os.getpid())+"\n")
+ os.close(pidd)
+ # Define and setup atexit closure
+
+ @atexit.register
+ def unlink_pid():
+ try:
+ os.unlink(pidfile)
+ except Exception:
+ pass
+
+
+def get_args():
+ optp = optparse.OptionParser(
+ usage="usage: %prog [options] logfile ...",
+ description="Watch TippingPoint logfiles and generate Idea events into directory")
+ optp.add_option(
+ "-n", "--name",
+ default=None,
+ dest="name",
+ type="string",
+ action="store",
+ help="Warden client name")
+ optp.add_option(
+ "--test",
+ default=False,
+ dest="test",
+ action="store_true",
+ help="Add \"Test\" category")
+ optp.add_option(
+ "--other",
+ default=False,
+ dest="other",
+ action="store_true",
+ help="Send events having \"Other\" category (usually nonmalicious)")
+ optp.add_option(
+ "-o", "--oneshot",
+ default=False,
+ dest="oneshot",
+ action="store_true",
+ help="process files and quit (do not daemonize)")
+ optp.add_option(
+ "--poll",
+ default=1,
+ dest="poll",
+ type="int",
+ action="store",
+ help="log file polling interval")
+ optp.add_option(
+ "-d", "--dir",
+ default=None,
+ dest="dir",
+ type="string",
+ action="store",
+ help="Target directory (mandatory)")
+ optp.add_option(
+ "-p", "--pid",
+ default=pth.join("/var/run", pth.splitext(pth.basename(sys.argv[0]))[0] + ".pid"),
+ dest="pid",
+ type="string",
+ action="store",
+ help="create PID file with this name (default: %default)")
+ optp.add_option(
+ "-u", "--uid",
+ default=None,
+ dest="uid",
+ type="int",
+ action="store",
+ help="user id to run under")
+ optp.add_option(
+ "-g", "--gid",
+ default=None,
+ dest="gid",
+ type="int",
+ action="store",
+ help="group id to run under")
+ optp.add_option(
+ "--origdata",
+ default=False,
+ dest="origdata",
+ action="store_true",
+ help="Store original report to IDEA message")
+ return optp
+
+
+def not_empty(test_string):
+ # tests if string is not empty
+ return None if test_string.strip() in ["", "null"] else test_string
+
+
+def save_events(event, filer):
+ f, name = filer.create_unique_file()
+ with f:
+ f.write(json.dumps(event, ensure_ascii=True))
+ filer.publish_file(name)
+
+
+def process_data(line, filer, origdata, idea_gen):
+ '''
+ takes one record, parse it to parameters, give it to Ideagen and writes to file
+ :param line: one record
+ :param idea_file: where output goes
+ :param origdata: if true, write original record to IDEA message
+ '''
+ row = line.split("|")
+ category = idea_gen.convert_category(category=row[1], id_taxonomy=int(row[2]))
+ timestamp = row[0].split(" ")[-1]
+ cve = [i for i in row[3].split(",") if i not in ("null", "")]
+ odata = "|".join([timestamp] + row[1:])
+ if category and not_empty(row[0][-14:-3]):
+ idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=category, id_taxonomy=int(row[2]),
+ cve=cve, filter_name=not_empty(row[4]), proto=not_empty(row[5]),
+ src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
+ conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
+ orig_data=odata if origdata else False)
+ if idea_event:
+ save_events(idea_event, filer)
+
+
+running_flag = True
+reload_flag = False
+
+
+def terminate_me(signum, frame):
+ global running_flag
+ running_flag = False
+
+
+def reload_me(signum, frame):
+ global reload_flag
+ reload_flag = True
+
+
+def main():
+ global running_flag
+ global reload_flag
+ logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG, filename='tipping_point_log.log', filemode='w')
+ optp = get_args()
+ opts, args = optp.parse_args()
+ if not args or opts.name is None or opts.dir is None:
+ optp.print_help()
+ sys.exit()
+ if opts.oneshot:
+ signal.signal(signal.SIGINT, terminate_me)
+ signal.signal(signal.SIGTERM, terminate_me)
+ files = [open(arg) for arg in args]
+ else:
+ daemonize(
+ pidfile=opts.pid,
+ uid=opts.uid,
+ gid=opts.gid,
+ signals={
+ signal.SIGINT: terminate_me,
+ signal.SIGTERM: terminate_me,
+ signal.SIGHUP: reload_me
+ })
+ files = [FileWatcher(arg) for arg in args]
+ filer = Filer(opts.dir)
+ idea_gen = IdeaGen(opts.name, opts.test, opts.other)
+ while running_flag:
+ for log_file in files:
+ while True:
+ line = log_file.readline()
+ if line is None or not line.strip():
+ logging.info("no line")
+ break
+ logging.info("readline")
+ process_data(line, filer, opts.origdata, idea_gen)
+ if not running_flag:
+ break
+ if reload_flag:
+ for f in files:
+ f.close()
+ f.open()
+ reload_flag = False
+ if opts.oneshot:
+ break
+ else:
+ time.sleep(opts.poll)
+
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file