Copied Pavel Eis version of connector from separate branch.

tippingpoint/tpToIdea.py

+# python ./tpToIdea.py --origdata --oneshot -d /root/Dokumenty/PycharmProjects/TippingPoint/IdeaLogTest -n cz.cesnet.server.tippingpoint /root/Dokumenty/PycharmProjects/TippingPoint/tp-test-log.txt /root/Dokumenty/PycharmProjects/TippingPoint/tp-log2.txt --test
+
+
+import json
+from uuid import uuid4
+import re
+import socket
+import optparse
+import sys
+import os
+import signal
+import resource
+import os.path as pth
+import atexit
+import time
+import logging
+
+
+class FileWatcher(object):
+
+    def __init__(self, filename, tail=True):
+        self.filename = filename
+        self.open()
+        self.line_buffer = ""
+        if tail and self.f:
+            self.f.seek(0, os.SEEK_END)
+
+    def open(self):
+        try:
+            self.f = open(self.filename, "r")
+            st = os.fstat(self.f.fileno())
+            self.inode, self.size = st.st_ino, st.st_size
+        except IOError:
+            self.f = None
+            self.inode = -1
+            self.size = -1
+
+    def _check_reopen(self):
+        try:
+            st = os.stat(self.filename)
+            cur_inode, cur_size = st.st_ino, st.st_size
+        except OSError as e:
+            cur_inode = -1
+            cur_size = -1
+        if cur_inode != self.inode or cur_size < self.size:
+            self.close()
+            self.open()
+
+    def readline(self):
+        if not self.f:
+            self.open()
+            if not self.f:
+                return self.line_buffer
+        res = self.f.readline()
+        if not res:
+            self._check_reopen()
+            if not self.f:
+                return self.line_buffer
+            res = self.f.readline()
+        if not res.endswith("\n"):
+            self.line_buffer += res
+        else:
+            res = self.line_buffer + res
+            self.line_buffer = ""
+            return res
+
+    def close(self):
+        try:
+            if self.f:
+                self.f.close()
+        except IOError:
+            pass
+        self.inode = -1
+        self.size = -1
+
+    def __repr__(self):
+        return '%s("%s")' % (type(self).__name__, self.filename)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+        return False
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        return self.readline()
+
+
+class Filer(object):
+
+    def __init__(self, directory):
+        self.basedir = self._ensure_path(directory)
+        self.tmp = self._ensure_path(pth.join(self.basedir, "tmp"))
+        self.incoming = self._ensure_path(pth.join(self.basedir, "incoming"))
+        self.hostname = socket.gethostname()
+        self.pid = os.getpid()
+
+    def _ensure_path(self, p):
+        try:
+            os.mkdir(p)
+        except OSError:
+            if not pth.isdir(p):
+                raise
+        return p
+
+    def _get_new_name(self, fd=None):
+        (inode, device) = os.fstat(fd)[1:3] if fd else (0, 0)
+        return "%s.%d.%f.%d.%d" % (
+            self.hostname, self.pid, time.time(), device, inode)
+
+    def create_unique_file(self):
+        # First find and open name unique within tmp
+        tmpname = None
+        while not tmpname:
+            tmpname = self._get_new_name()
+            try:
+                fd = os.open(pth.join(self.tmp, tmpname), os.O_CREAT | os.O_RDWR | os.O_EXCL)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise   # other errors than duplicates should get noticed
+                tmpname = None
+        # Now we know the device/inode, rename to make unique within system
+        newname = self._get_new_name(fd)
+        os.rename(pth.join(self.tmp, tmpname), pth.join(self.tmp, newname))
+        nf = os.fdopen(fd, "w")
+        return nf, newname
+
+    def publish_file(self, short_name):
+        os.rename(pth.join(self.tmp, short_name), pth.join(self.incoming, short_name))
+
+
+class IdeaGen(object):
+    idea_categories = {'Malware.Worm': re.compile("^Exploits: .*?Worm"),
+                       'Attempt.Login': re.compile("SipVicious Brute Force"),
+                       'Attempt.Exploit': re.compile("^Exploits: .*?Attempt"),
+                       'Other': re.compile("P2P"),
+                       'Recon.Scanning': re.compile("^Reconnaissance"),
+                       'Vulnerable.Open': re.compile("(^Vulnerabilities: [0-9]{5}|^Network Equipment: [0-9]{4} SNMP)"),
+                       'Vulnerable.Config': re.compile("^Vulnerabilities: [0-9]{4} ICMP")}
+    anomaly_re = re.compile("(^Network Equipment: [0-9]{4} IP|^Traffic Normalization: [0-9]{4} (?!Tunneling))")
+    availability_ddos_re = re.compile("^Traffic Normalization: ")
+
+    def __init__(self, name, test=False):
+        self.name = name
+        self.test = test
+
+    def convert_category(self, category, incident, examples_of_incident):
+        '''
+        converts category from record to IDEA category
+        :param category: TippingPoint category description
+        :param incident: TippingPoint incident description
+        :param examples_of_incident: some TippingPoing incident descriptions has examples
+        :return: if category or incident is empty or is not important for saving it return None, otherwise return
+                 converted category
+        '''
+        if not (category and incident):
+            return None
+        tp_category = category + ": " + incident
+        if IdeaGen.anomaly_re.search(tp_category) and not examples_of_incident:
+            return "Anomaly.Traffic"
+        if IdeaGen.availability_ddos_re.search(tp_category) and examples_of_incident:
+            return "Availability.DoS"
+        for category, pattern in IdeaGen.idea_categories.items():
+            if pattern.search(tp_category):
+               return category
+        return None
+
+    def gen_event_idea(self, timestamp, category, id_taxonomy, cve, examples_of_incident, proto, src_ip, src_port,
+                       dest_ip, dest_port, conn_count, url, orig_data, incident_desription):
+        '''
+        put every piece of record together into IDEA message
+        :return: new IDEA message
+        '''
+        event = {
+            'Format': "IDEA0",
+            'ID': str(uuid4()),
+            'DetectTime': time.strftime("%y-%m-%dT%H:%M:%S", time.localtime(timestamp)),
+            'Category': [category] + (["Test"] if self.test else []),
+            'Note': incident_desription,
+            'Ref': []
+        }
+        if cve:
+            event['Ref'].append("cve:"+cve)
+        if id_taxonomy:
+            event['Ref'].append("tipping_point_taxonomy:%d" % int(id_taxonomy))
+        if conn_count:
+            event['ConnCount'] = conn_count
+        source = {}
+        target = {}
+        if src_ip:
+            af = "IP4" if not ':' in src_ip else "IP6"
+            source[af] = [src_ip]
+        if src_port:
+            source['Port'] = [src_port]
+        if proto:
+            source['Proto'] = [proto]
+        if url:
+            source['url'] = url
+        if dest_ip:
+            af = "IP4" if not ':' in dest_ip else "IP6"
+            target[af] = [dest_ip]
+        if dest_port:
+            target['Port'] = [dest_port]
+        if source:
+            event['Source'] = [source]
+        if target:
+            event['Target'] = [target]
+        if examples_of_incident:
+            event['Note'] += ", examples of event:" + examples_of_incident[6:]
+        if orig_data:
+            event['Attach'] = {'Handle': "att1",
+                               'Note': "original data",
+                               'Content': orig_data}
+        event['Node'] = [{
+                            'Name': self.name,
+                            'Type': ["Connection", "Honeypot", "Recon"],
+                            'SW': ["HP Tipping Point"],
+                        }]
+        return event
+
+
+def daemonize(
+        work_dir=None, chroot_dir=None,
+        umask=None, uid=None, gid=None,
+        pidfile=None, files_preserve=[], signals={}):
+    # Dirs, limits, users
+    if chroot_dir is not None:
+        os.chdir(chroot_dir)
+        os.chroot(chroot_dir)
+    if umask is not None:
+        os.umask(umask)
+    if work_dir is not None:
+        os.chdir(work_dir)
+    if gid is not None:
+        os.setgid(gid)
+    if uid is not None:
+        os.setuid(uid)
+    # Doublefork, split session
+    if os.fork() > 0:
+        os._exit(0)
+    try:
+        os.setsid()
+    except OSError:
+        pass
+    if os.fork() > 0:
+        os._exit(0)
+    # Setup signal handlers
+    for (signum, handler) in signals.items():
+        signal.signal(signum, handler)
+    # Close descriptors
+    descr_preserve = set(f.fileno() for f in files_preserve)
+    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    if maxfd == resource.RLIM_INFINITY:
+        maxfd = 65535
+    for fd in range(maxfd, 3, -1):  # 3 means omit stdin, stdout, stderr
+        if fd not in descr_preserve:
+            try:
+                os.close(fd)
+            except Exception:
+                pass
+    # Redirect stdin, stdout, stderr to /dev/null
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for fd in range(3):
+        os.dup2(devnull, fd)
+    # PID file
+    if pidfile is not None:
+        pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
+        os.write(pidd, str(os.getpid())+"\n")
+        os.close(pidd)
+        # Define and setup atexit closure
+
+        @atexit.register
+        def unlink_pid():
+            try:
+                os.unlink(pidfile)
+            except Exception:
+                pass
+
+
+def get_args():
+    optp = optparse.OptionParser(
+        usage="usage: %prog [options] logfile ...",
+        description="Watch HP-Tipping_Point logfiles and generate Idea events into directory")
+    optp.add_option(
+        "-n", "--name",
+        default=None,
+        dest="name",
+        type="string",
+        action="store",
+        help="Warden client name")
+    optp.add_option(
+        "--test",
+        default=False,
+        dest="test",
+        action="store_true",
+        help="Add Test category")
+    optp.add_option(
+        "-o", "--oneshot",
+        default=False,
+        dest="oneshot",
+        action="store_true",
+        help="process files and quit (do not daemonize)")
+    optp.add_option(
+        "--poll",
+        default=1,
+        dest="poll",
+        type="int",
+        action="store",
+        help="log file polling interval")
+    optp.add_option(
+        "-d", "--dir",
+        default=None,
+        dest="dir",
+        type="string",
+        action="store",
+        help="Target directory (mandatory)")
+    optp.add_option(
+        "-p", "--pid",
+        default=pth.join("/var/run", pth.splitext(pth.basename(sys.argv[0]))[0] + ".pid"),
+        dest="pid",
+        type="string",
+        action="store",
+        help="create PID file with this name (default: %default)")
+    optp.add_option(
+        "-u", "--uid",
+        default=None,
+        dest="uid",
+        type="int",
+        action="store",
+        help="user id to run under")
+    optp.add_option(
+        "-g", "--gid",
+        default=None,
+        dest="gid",
+        type="int",
+        action="store",
+        help="group id to run under")
+    optp.add_option(
+        "--origdata",
+        default=False,
+        dest="origdata",
+        action="store_true",
+        help="Store original report to IDEA message")
+    return optp
+
+
+def not_empty(test_string):
+    # tests if string is not empty
+    return None if test_string.strip() in ["", "null"] else test_string
+
+
+def save_events(event, filer):
+    f, name = filer.create_unique_file()
+    with f:
+        f.write(json.dumps(event, ensure_ascii=True))
+    filer.publish_file(name)
+    
+
+def process_data(line, filer, origdata, idea_gen):
+    '''
+    takes one record, parse it to parameters, give it to Ideagen and writes to file
+    :param line: one record
+    :param idea_file: where output goes
+    :param origdata: if true, write original record to IDEA message
+    '''
+    row = line.split(",")
+    while re.search("^CVE-.*$", row[4]):
+        row[3] += ", " + row[4]
+        del row[4]
+    examples_of_incident = []
+    while not re.search("(UDP|IP|ICMP|DNS|IP6)", row[5]):
+        examples_of_incident.append(row[5])
+        del row[5]
+    examples_of_incident = " ".join(examples_of_incident)
+    category = idea_gen.convert_category(category=row[1], incident=row[4], examples_of_incident=examples_of_incident)
+    if category and not_empty(row[0][-14:-3]):
+        idea_event = idea_gen.gen_event_idea(timestamp=int(row[0][-14:-3]), category=category, id_taxonomy=not_empty(row[2]), cve=not_empty(row[3]),
+                                             examples_of_incident=not_empty(examples_of_incident), proto=not_empty(row[5]),
+                                             src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
+                                             conn_count=not_empty(row[10]), url=not_empty(row[12]),
+                                             orig_data=str(line) if origdata else False, incident_desription=row[4])
+        save_events(idea_event, filer)
+
+
+running_flag = True
+reload_flag = False
+
+
+def terminate_me(signum, frame):
+    global running_flag
+    running_flag = False
+
+
+def reload_me(signum, frame):
+    global reload_flag
+    reload_flag = True
+
+
+def main():
+    global running_flag
+    global reload_flag
+    logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG, filename='tipping_point_log.log', filemode='w')
+    optp = get_args()
+    opts, args = optp.parse_args()
+    if not args or opts.name is None or opts.dir is None:
+        optp.print_help()
+        sys.exit()
+    if opts.oneshot:
+        signal.signal(signal.SIGINT, terminate_me)
+        signal.signal(signal.SIGTERM, terminate_me)
+        files = [open(arg) for arg in args]
+    else:
+        daemonize(
+            pidfile=opts.pid,
+            uid=opts.uid,
+            gid=opts.gid,
+            signals={
+                signal.SIGINT: terminate_me,
+                signal.SIGTERM: terminate_me,
+                signal.SIGHUP: reload_me
+            })
+        files = [FileWatcher(arg) for arg in args]
+    filer = Filer(opts.dir)
+    idea_gen = IdeaGen(opts.name, opts.test)
+    while running_flag:
+        for log_file in files:
+            while True:
+                line = log_file.readline()
+                if line is None or not line.strip():
+                    logging.info("no line")
+                    break
+                logging.info("readline")
+                process_data(line, filer, opts.origdata, idea_gen)
+            if not running_flag:
+                break
+            if reload_flag:
+                for f in files:
+                    f.close()
+                    f.open()
+                reload_flag = False
+        if opts.oneshot:
+            break
+        else:
+            time.sleep(opts.poll)
+            
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file