diff --git a/hp-tipping-point/tpToIdea.py b/hp-tipping-point/tpToIdea.py
deleted file mode 100644
index 3610e169db4ae5b2f728a496a6453d3740bb076d..0000000000000000000000000000000000000000
--- a/hp-tipping-point/tpToIdea.py
+++ /dev/null
@@ -1,522 +0,0 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2017-2018 Cesnet z.s.p.o
-# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
-
-import json
-from uuid import uuid4
-import re
-import socket
-import optparse
-import sys
-import os
-import signal
-import resource
-import os.path as pth
-import atexit
-import time
-from datetime import datetime
-import logging
-from collections import OrderedDict
-
-
-class FileWatcher(object):
-
- def __init__(self, filename, tail=True):
- self.filename = filename
- self.open()
- self.line_buffer = ""
- if tail and self.f:
- self.f.seek(0, os.SEEK_END)
-
- def open(self):
- try:
- self.f = open(self.filename, "r")
- st = os.fstat(self.f.fileno())
- self.inode, self.size = st.st_ino, st.st_size
- except IOError:
- self.f = None
- self.inode = -1
- self.size = -1
-
- def _check_reopen(self):
- try:
- st = os.stat(self.filename)
- cur_inode, cur_size = st.st_ino, st.st_size
- except OSError as e:
- cur_inode = -1
- cur_size = -1
- if cur_inode != self.inode or cur_size < self.size:
- self.close()
- self.open()
-
- def readline(self):
- if not self.f:
- self.open()
- if not self.f:
- return self.line_buffer
- res = self.f.readline()
- if not res:
- self._check_reopen()
- if not self.f:
- return self.line_buffer
- res = self.f.readline()
- if not res.endswith("\n"):
- self.line_buffer += res
- else:
- res = self.line_buffer + res
- self.line_buffer = ""
- return res
-
- def close(self):
- try:
- if self.f:
- self.f.close()
- except IOError:
- pass
- self.inode = -1
- self.size = -1
-
- def __repr__(self):
- return '%s("%s")' % (type(self).__name__, self.filename)
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.close()
- return False
-
- def __iter__(self):
- return self
-
- def next(self):
- return self.readline()
-
-
-class Filer(object):
-
- def __init__(self, directory):
- self.basedir = self._ensure_path(directory)
- self.tmp = self._ensure_path(pth.join(self.basedir, "tmp"))
- self.incoming = self._ensure_path(pth.join(self.basedir, "incoming"))
- self.hostname = socket.gethostname()
- self.pid = os.getpid()
-
- def _ensure_path(self, p):
- try:
- os.mkdir(p)
- except OSError:
- if not pth.isdir(p):
- raise
- return p
-
- def _get_new_name(self, fd=None):
- (inode, device) = os.fstat(fd)[1:3] if fd else (0, 0)
- return "%s.%d.%f.%d.%d" % (
- self.hostname, self.pid, time.time(), device, inode)
-
- def create_unique_file(self):
- # First find and open name unique within tmp
- tmpname = None
- while not tmpname:
- tmpname = self._get_new_name()
- try:
- fd = os.open(pth.join(self.tmp, tmpname), os.O_CREAT | os.O_RDWR | os.O_EXCL)
- except OSError as e:
- if e.errno != errno.EEXIST:
- raise # other errors than duplicates should get noticed
- tmpname = None
- # Now we know the device/inode, rename to make unique within system
- newname = self._get_new_name(fd)
- os.rename(pth.join(self.tmp, tmpname), pth.join(self.tmp, newname))
- nf = os.fdopen(fd, "w")
- return nf, newname
-
- def publish_file(self, short_name):
- os.rename(pth.join(self.tmp, short_name), pth.join(self.incoming, short_name))
-
-
-class IdeaGen(object):
-
- tp_to_idea = {
- 1: {
- 1: ["Attempt.Exploit"],
- 2: ["Attempt.Exploit"],
- 3: ["Attempt.Exploit"],
- 4: ["Attempt.Exploit"],
- 5: ["Attempt.Exploit"],
- 6: ["Attempt.Exploit"],
- 255: ["Attempt.Exploit"]
- },
-
- 2: {
- 1: ["Malware.Worm"],
- 2: ["Malware.Virus"],
- 3: ["Malware.Trojan"],
- 4: ["Intrusion.Botnet"],
- 5: ["Fraud.Phishing"],
- 255: ["Malware"]
- },
-
- 3: {
- 1: ["Availability.DDoS"],
- 2: ["Availability.DDoS"],
- 3: ["Availability.DDoS"],
- 255: ["Availability.DDoS"]
- },
-
- 4: {
- 1: ["Other"],
- 2: ["Other"],
- 3: ["Other"],
- 4: ["Other"],
- 5: ["Other"],
- 6: ["Attempt.Login"],
- 7: ["Malware.Spyware"],
- 255: ["Other"]
- },
-
- 5: {
- 1: ["Recon.Scanning"],
- 2: ["Attempt.Exploit"],
- 3: ["Attempt.Exploit"],
- 4: ["Recon.Scanning", "Attempt.Exploit"],
- 255: ["Attempt.Exploit"]
- },
-
- 6: {
- 1: ["Anomaly.Protocol"],
- 2: ["Anomaly.Traffic"],
- 3: ["Anomaly.Application"],
- 255: ["Anomaly"]
- },
-
- 7: {
- 1: ["Anomaly.Traffic"],
- 2: ["Anomaly.Application"],
- 255: ["Anomaly.Traffic"]
- },
-
- 8: {
- 1: ["Other"],
- 2: ["Other"],
- 255: ["Other"]
- },
- }
-
- def __init__(self, name, test=False, other=False):
- self.name = name
- self.test = test
- self.other = other
-
- def convert_category(self, category, id_taxonomy):
- '''
- converts category from record to IDEA category
- :param category: TippingPoint category description
- :param id_taxonomy: TippingPoint taxonomy id
- :return: if category or incident is empty or is not important for saving it return None, otherwise return
- converted category
- '''
- if not (category and id_taxonomy):
- return None
- tp_cat_maj = id_taxonomy >> 24
- tp_cat_min = id_taxonomy >> 16 & 0b11111111
- tp_proto = id_taxonomy >> 8 & 0b11111111
- tp_platf = id_taxonomy & 0b11111111
- try:
- category = IdeaGen.tp_to_idea[tp_cat_maj][tp_cat_min]
- except KeyError:
- category = ["Other"]
-
- return category
-
- def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
- dest_ip, dest_port, conn_count, url, severity, orig_data):
- '''
- put every piece of record together into IDEA message
- :return: new IDEA message
- '''
-
- if (category == ["Other"]) and not self.other:
- return None
-
- event = OrderedDict([
- ("Format","IDEA0"),
- ("ID", str(uuid4())),
- ("DetectTime", datetime.fromtimestamp(timestamp / 1000).isoformat() + 'Z'),
- ("Category", category + (["Test"] if self.test else [])),
- ])
- if cve:
- event['Ref'] = ['urn:cve:'.format(i) for i in cve]
- if conn_count and int(conn_count):
- event['ConnCount'] = int(conn_count)
- source = OrderedDict()
- target = OrderedDict()
- if src_ip:
- # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
- src_ip = src_ip.replace(" ", "")
- af = "IP4" if not ':' in src_ip else "IP6"
- source[af] = [src_ip]
- if src_port and int(src_port):
- source['Port'] = [int(src_port)]
- if proto:
- source['Proto'] = [proto]
- if dest_ip and (dest_ip != "0.0.0.0"):
- # TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
- dest_ip = dest_ip.replace(" ", "")
- af = "IP4" if not ':' in dest_ip else "IP6"
- target[af] = [dest_ip]
- if dest_port and int(dest_port):
- target['Port'] = [int(dest_port)]
- if proto:
- target['Proto'] = [proto]
- if url:
- target['URL'] = url
- if source:
- event['Source'] = [source]
- if target:
- event['Target'] = [target]
- if orig_data:
- event['Attach'] = [OrderedDict([
- ('Type', ["OrigData"]),
- ('Content', orig_data.strip())
- ])]
- event['Node'] = [OrderedDict([
- ('Name', self.name),
- ('Type', ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"]),
- ('SW', ["TippingPoint_NX_NGIPS"])
- ])]
- return event
-
-
-def daemonize(
- work_dir=None, chroot_dir=None,
- umask=None, uid=None, gid=None,
- pidfile=None, files_preserve=[], signals={}):
- # Dirs, limits, users
- if chroot_dir is not None:
- os.chdir(chroot_dir)
- os.chroot(chroot_dir)
- if umask is not None:
- os.umask(umask)
- if work_dir is not None:
- os.chdir(work_dir)
- if gid is not None:
- os.setgid(gid)
- if uid is not None:
- os.setuid(uid)
- # Doublefork, split session
- if os.fork() > 0:
- os._exit(0)
- try:
- os.setsid()
- except OSError:
- pass
- if os.fork() > 0:
- os._exit(0)
- # Setup signal handlers
- for (signum, handler) in signals.items():
- signal.signal(signum, handler)
- # Close descriptors
- descr_preserve = set(f.fileno() for f in files_preserve)
- maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
- if maxfd == resource.RLIM_INFINITY:
- maxfd = 65535
- for fd in range(maxfd, 3, -1): # 3 means omit stdin, stdout, stderr
- if fd not in descr_preserve:
- try:
- os.close(fd)
- except Exception:
- pass
- # Redirect stdin, stdout, stderr to /dev/null
- devnull = os.open(os.devnull, os.O_RDWR)
- for fd in range(3):
- os.dup2(devnull, fd)
- # PID file
- if pidfile is not None:
- pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
- os.write(pidd, str(os.getpid())+"\n")
- os.close(pidd)
- # Define and setup atexit closure
-
- @atexit.register
- def unlink_pid():
- try:
- os.unlink(pidfile)
- except Exception:
- pass
-
-
-def get_args():
- optp = optparse.OptionParser(
- usage="usage: %prog [options] logfile ...",
- description="Watch TippingPoint logfiles and generate Idea events into directory")
- optp.add_option(
- "-n", "--name",
- default=None,
- dest="name",
- type="string",
- action="store",
- help="Warden client name")
- optp.add_option(
- "--test",
- default=False,
- dest="test",
- action="store_true",
- help="Add \"Test\" category")
- optp.add_option(
- "--other",
- default=False,
- dest="other",
- action="store_true",
- help="Send events having \"Other\" category (usually nonmalicious)")
- optp.add_option(
- "-o", "--oneshot",
- default=False,
- dest="oneshot",
- action="store_true",
- help="process files and quit (do not daemonize)")
- optp.add_option(
- "--poll",
- default=1,
- dest="poll",
- type="int",
- action="store",
- help="log file polling interval")
- optp.add_option(
- "-d", "--dir",
- default=None,
- dest="dir",
- type="string",
- action="store",
- help="Target directory (mandatory)")
- optp.add_option(
- "-p", "--pid",
- default=pth.join("/var/run", pth.splitext(pth.basename(sys.argv[0]))[0] + ".pid"),
- dest="pid",
- type="string",
- action="store",
- help="create PID file with this name (default: %default)")
- optp.add_option(
- "-u", "--uid",
- default=None,
- dest="uid",
- type="int",
- action="store",
- help="user id to run under")
- optp.add_option(
- "-g", "--gid",
- default=None,
- dest="gid",
- type="int",
- action="store",
- help="group id to run under")
- optp.add_option(
- "--origdata",
- default=False,
- dest="origdata",
- action="store_true",
- help="Store original report to IDEA message")
- return optp
-
-
-def not_empty(test_string):
- # tests if string is not empty
- return None if test_string.strip() in ["", "null"] else test_string
-
-
-def save_events(event, filer):
- f, name = filer.create_unique_file()
- with f:
- f.write(json.dumps(event, ensure_ascii=True))
- filer.publish_file(name)
-
-
-def process_data(line, filer, origdata, idea_gen):
- '''
- takes one record, parse it to parameters, give it to Ideagen and writes to file
- :param line: one record
- :param idea_file: where output goes
- :param origdata: if true, write original record to IDEA message
- '''
- row = line.split("|")
- category = idea_gen.convert_category(category=row[1], id_taxonomy=int(row[2]))
- timestamp = row[0].split(" ")[-1]
- cve = [i for i in row[3].split(",") if i not in ("null", "")]
- odata = "|".join([timestamp] + row[1:])
- if category and not_empty(row[0][-14:-3]):
- idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=category, id_taxonomy=int(row[2]),
- cve=cve, filter_name=not_empty(row[4]), proto=not_empty(row[5]),
- src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
- conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
- orig_data=odata if origdata else False)
- if idea_event:
- save_events(idea_event, filer)
-
-
-running_flag = True
-reload_flag = False
-
-
-def terminate_me(signum, frame):
- global running_flag
- running_flag = False
-
-
-def reload_me(signum, frame):
- global reload_flag
- reload_flag = True
-
-
-def main():
- global running_flag
- global reload_flag
- logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG, filename='tipping_point_log.log', filemode='w')
- optp = get_args()
- opts, args = optp.parse_args()
- if not args or opts.name is None or opts.dir is None:
- optp.print_help()
- sys.exit()
- if opts.oneshot:
- signal.signal(signal.SIGINT, terminate_me)
- signal.signal(signal.SIGTERM, terminate_me)
- files = [open(arg) for arg in args]
- else:
- daemonize(
- pidfile=opts.pid,
- uid=opts.uid,
- gid=opts.gid,
- signals={
- signal.SIGINT: terminate_me,
- signal.SIGTERM: terminate_me,
- signal.SIGHUP: reload_me
- })
- files = [FileWatcher(arg) for arg in args]
- filer = Filer(opts.dir)
- idea_gen = IdeaGen(opts.name, opts.test, opts.other)
- while running_flag:
- for log_file in files:
- while True:
- line = log_file.readline()
- if line is None or not line.strip():
- logging.info("no line")
- break
- logging.info("readline")
- process_data(line, filer, opts.origdata, idea_gen)
- if not running_flag:
- break
- if reload_flag:
- for f in files:
- f.close()
- f.open()
- reload_flag = False
- if opts.oneshot:
- break
- else:
- time.sleep(opts.poll)
-
-
-if __name__ == "__main__":
- main()
\ No newline at end of file