tpToIdea.py

# python ./tpToIdea.py --origdata --oneshot -d /root/Dokumenty/PycharmProjects/TippingPoint/IdeaLogTest -n cz.cesnet.server.tippingpoint /root/Dokumenty/PycharmProjects/TippingPoint/tp-test-log.txt /root/Dokumenty/PycharmProjects/TippingPoint/tp-log2.txt --test


import json
from uuid import uuid4
import re
import socket
import optparse
import sys
import os
import signal
import resource
import os.path as pth
import atexit
import time
import logging


class FileWatcher(object):

    def __init__(self, filename, tail=True):
        self.filename = filename
        self.open()
        self.line_buffer = ""
        if tail and self.f:
            self.f.seek(0, os.SEEK_END)

    def open(self):
        try:
            self.f = open(self.filename, "r")
            st = os.fstat(self.f.fileno())
            self.inode, self.size = st.st_ino, st.st_size
        except IOError:
            self.f = None
            self.inode = -1
            self.size = -1

    def _check_reopen(self):
        try:
            st = os.stat(self.filename)
            cur_inode, cur_size = st.st_ino, st.st_size
        except OSError as e:
            cur_inode = -1
            cur_size = -1
        if cur_inode != self.inode or cur_size < self.size:
            self.close()
            self.open()

    def readline(self):
        if not self.f:
            self.open()
            if not self.f:
                return self.line_buffer
        res = self.f.readline()
        if not res:
            self._check_reopen()
            if not self.f:
                return self.line_buffer
            res = self.f.readline()
        if not res.endswith("\n"):
            self.line_buffer += res
        else:
            res = self.line_buffer + res
            self.line_buffer = ""
            return res

    def close(self):
        try:
            if self.f:
                self.f.close()
        except IOError:
            pass
        self.inode = -1
        self.size = -1

    def __repr__(self):
        return '%s("%s")' % (type(self).__name__, self.filename)

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
        return False

    def __iter__(self):
        return self

    def next(self):
        return self.readline()


class Filer(object):

    def __init__(self, directory):
        self.basedir = self._ensure_path(directory)
        self.tmp = self._ensure_path(pth.join(self.basedir, "tmp"))
        self.incoming = self._ensure_path(pth.join(self.basedir, "incoming"))
        self.hostname = socket.gethostname()
        self.pid = os.getpid()

    def _ensure_path(self, p):
        try:
            os.mkdir(p)
        except OSError:
            if not pth.isdir(p):
                raise
        return p

    def _get_new_name(self, fd=None):
        (inode, device) = os.fstat(fd)[1:3] if fd else (0, 0)
        return "%s.%d.%f.%d.%d" % (
            self.hostname, self.pid, time.time(), device, inode)

    def create_unique_file(self):
        # First find and open name unique within tmp
        tmpname = None
        while not tmpname:
            tmpname = self._get_new_name()
            try:
                fd = os.open(pth.join(self.tmp, tmpname), os.O_CREAT | os.O_RDWR | os.O_EXCL)
            except OSError as e:
                if e.errno != errno.EEXIST:
                    raise   # other errors than duplicates should get noticed
                tmpname = None
        # Now we know the device/inode, rename to make unique within system
        newname = self._get_new_name(fd)
        os.rename(pth.join(self.tmp, tmpname), pth.join(self.tmp, newname))
        nf = os.fdopen(fd, "w")
        return nf, newname

    def publish_file(self, short_name):
        os.rename(pth.join(self.tmp, short_name), pth.join(self.incoming, short_name))


class IdeaGen(object):
    idea_categories = {'Malware.Worm': re.compile("^Exploits: .*?Worm"),
                       'Attempt.Login': re.compile("SipVicious Brute Force"),
                       'Attempt.Exploit': re.compile("^Exploits: .*?Attempt"),
                       'Other': re.compile("P2P"),
                       'Recon.Scanning': re.compile("^Reconnaissance"),
                       'Vulnerable.Open': re.compile("(^Vulnerabilities: [0-9]{5}|^Network Equipment: [0-9]{4} SNMP)"),
                       'Vulnerable.Config': re.compile("^Vulnerabilities: [0-9]{4} ICMP")}
    anomaly_re = re.compile("(^Network Equipment: [0-9]{4} IP|^Traffic Normalization: [0-9]{4} (?!Tunneling))")
    availability_ddos_re = re.compile("^Traffic Normalization: ")

    def __init__(self, name, test=False):
        self.name = name
        self.test = test

    def convert_category(self, category, incident, examples_of_incident):
        '''
        converts category from record to IDEA category
        :param category: TippingPoint category description
        :param incident: TippingPoint incident description
        :param examples_of_incident: some TippingPoing incident descriptions has examples
        :return: if category or incident is empty or is not important for saving it return None, otherwise return
                 converted category
        '''
        if not (category and incident):
            return None
        tp_category = category + ": " + incident
        if IdeaGen.anomaly_re.search(tp_category) and not examples_of_incident:
            return "Anomaly.Traffic"
        if IdeaGen.availability_ddos_re.search(tp_category) and examples_of_incident:
            return "Availability.DoS"
        for category, pattern in IdeaGen.idea_categories.items():
            if pattern.search(tp_category):
               return category
        return None

    def gen_event_idea(self, timestamp, category, id_taxonomy, cve, examples_of_incident, proto, src_ip, src_port,
                       dest_ip, dest_port, conn_count, url, orig_data, incident_desription):
        '''
        put every piece of record together into IDEA message
        :return: new IDEA message
        '''
        event = {
            'Format': "IDEA0",
            'ID': str(uuid4()),
            'DetectTime': time.strftime("%y-%m-%dT%H:%M:%S", time.localtime(timestamp)),
            'Category': [category] + (["Test"] if self.test else []),
            'Note': incident_desription,
            'Ref': []
        }
        if cve:
            event['Ref'].append("cve:"+cve)
        if id_taxonomy:
            event['Ref'].append("tipping_point_taxonomy:%d" % int(id_taxonomy))
        if conn_count:
            event['ConnCount'] = conn_count
        source = {}
        target = {}
        if src_ip:
            af = "IP4" if not ':' in src_ip else "IP6"
            source[af] = [src_ip]
        if src_port:
            source['Port'] = [src_port]
        if proto:
            source['Proto'] = [proto]
        if url:
            source['url'] = url
        if dest_ip:
            af = "IP4" if not ':' in dest_ip else "IP6"
            target[af] = [dest_ip]
        if dest_port:
            target['Port'] = [dest_port]
        if source:
            event['Source'] = [source]
        if target:
            event['Target'] = [target]
        if examples_of_incident:
            event['Note'] += ", examples of event:" + examples_of_incident[6:]
        if orig_data:
            event['Attach'] = {'Handle': "att1",
                               'Note': "original data",
                               'Content': orig_data}
        event['Node'] = [{
                            'Name': self.name,
                            'Type': ["Connection", "Honeypot", "Recon"],
                            'SW': ["HP Tipping Point"],
                        }]
        return event


def daemonize(
        work_dir=None, chroot_dir=None,
        umask=None, uid=None, gid=None,
        pidfile=None, files_preserve=[], signals={}):
    # Dirs, limits, users
    if chroot_dir is not None:
        os.chdir(chroot_dir)
        os.chroot(chroot_dir)
    if umask is not None:
        os.umask(umask)
    if work_dir is not None:
        os.chdir(work_dir)
    if gid is not None:
        os.setgid(gid)
    if uid is not None:
        os.setuid(uid)
    # Doublefork, split session
    if os.fork() > 0:
        os._exit(0)
    try:
        os.setsid()
    except OSError:
        pass
    if os.fork() > 0:
        os._exit(0)
    # Setup signal handlers
    for (signum, handler) in signals.items():
        signal.signal(signum, handler)
    # Close descriptors
    descr_preserve = set(f.fileno() for f in files_preserve)
    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
    if maxfd == resource.RLIM_INFINITY:
        maxfd = 65535
    for fd in range(maxfd, 3, -1):  # 3 means omit stdin, stdout, stderr
        if fd not in descr_preserve:
            try:
                os.close(fd)
            except Exception:
                pass
    # Redirect stdin, stdout, stderr to /dev/null
    devnull = os.open(os.devnull, os.O_RDWR)
    for fd in range(3):
        os.dup2(devnull, fd)
    # PID file
    if pidfile is not None:
        pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
        os.write(pidd, str(os.getpid())+"\n")
        os.close(pidd)
        # Define and setup atexit closure

        @atexit.register
        def unlink_pid():
            try:
                os.unlink(pidfile)
            except Exception:
                pass


def get_args():
    optp = optparse.OptionParser(
        usage="usage: %prog [options] logfile ...",
        description="Watch HP-Tipping_Point logfiles and generate Idea events into directory")
    optp.add_option(
        "-n", "--name",
        default=None,
        dest="name",
        type="string",
        action="store",
        help="Warden client name")
    optp.add_option(
        "--test",
        default=False,
        dest="test",
        action="store_true",
        help="Add Test category")
    optp.add_option(
        "-o", "--oneshot",
        default=False,
        dest="oneshot",
        action="store_true",
        help="process files and quit (do not daemonize)")
    optp.add_option(
        "--poll",
        default=1,
        dest="poll",
        type="int",
        action="store",
        help="log file polling interval")
    optp.add_option(
        "-d", "--dir",
        default=None,
        dest="dir",
        type="string",
        action="store",
        help="Target directory (mandatory)")
    optp.add_option(
        "-p", "--pid",
        default=pth.join("/var/run", pth.splitext(pth.basename(sys.argv[0]))[0] + ".pid"),
        dest="pid",
        type="string",
        action="store",
        help="create PID file with this name (default: %default)")
    optp.add_option(
        "-u", "--uid",
        default=None,
        dest="uid",
        type="int",
        action="store",
        help="user id to run under")
    optp.add_option(
        "-g", "--gid",
        default=None,
        dest="gid",
        type="int",
        action="store",
        help="group id to run under")
    optp.add_option(
        "--origdata",
        default=False,
        dest="origdata",
        action="store_true",
        help="Store original report to IDEA message")
    return optp


def not_empty(test_string):
    # tests if string is not empty
    return None if test_string.strip() in ["", "null"] else test_string


def save_events(event, filer):
    f, name = filer.create_unique_file()
    with f:
        f.write(json.dumps(event, ensure_ascii=True))
    filer.publish_file(name)
    

def process_data(line, filer, origdata, idea_gen):
    '''
    takes one record, parse it to parameters, give it to Ideagen and writes to file
    :param line: one record
    :param idea_file: where output goes
    :param origdata: if true, write original record to IDEA message
    '''
    row = line.split(",")
    while re.search("^CVE-.*$", row[4]):
        row[3] += ", " + row[4]
        del row[4]
    examples_of_incident = []
    while not re.search("(UDP|IP|ICMP|DNS|IP6)", row[5]):
        examples_of_incident.append(row[5])
        del row[5]
    examples_of_incident = " ".join(examples_of_incident)
    category = idea_gen.convert_category(category=row[1], incident=row[4], examples_of_incident=examples_of_incident)
    if category and not_empty(row[0][-14:-3]):
        idea_event = idea_gen.gen_event_idea(timestamp=int(row[0][-14:-3]), category=category, id_taxonomy=not_empty(row[2]), cve=not_empty(row[3]),
                                             examples_of_incident=not_empty(examples_of_incident), proto=not_empty(row[5]),
                                             src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
                                             conn_count=not_empty(row[10]), url=not_empty(row[12]),
                                             orig_data=str(line) if origdata else False, incident_desription=row[4])
        save_events(idea_event, filer)


running_flag = True
reload_flag = False


def terminate_me(signum, frame):
    global running_flag
    running_flag = False


def reload_me(signum, frame):
    global reload_flag
    reload_flag = True


def main():
    global running_flag
    global reload_flag
    logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG, filename='tipping_point_log.log', filemode='w')
    optp = get_args()
    opts, args = optp.parse_args()
    if not args or opts.name is None or opts.dir is None:
        optp.print_help()
        sys.exit()
    if opts.oneshot:
        signal.signal(signal.SIGINT, terminate_me)
        signal.signal(signal.SIGTERM, terminate_me)
        files = [open(arg) for arg in args]
    else:
        daemonize(
            pidfile=opts.pid,
            uid=opts.uid,
            gid=opts.gid,
            signals={
                signal.SIGINT: terminate_me,
                signal.SIGTERM: terminate_me,
                signal.SIGHUP: reload_me
            })
        files = [FileWatcher(arg) for arg in args]
    filer = Filer(opts.dir)
    idea_gen = IdeaGen(opts.name, opts.test)
    while running_flag:
        for log_file in files:
            while True:
                line = log_file.readline()
                if line is None or not line.strip():
                    logging.info("no line")
                    break
                logging.info("readline")
                process_data(line, filer, opts.origdata, idea_gen)
            if not running_flag:
                break
            if reload_flag:
                for f in files:
                    f.close()
                    f.open()
                reload_flag = False
        if opts.oneshot:
            break
        else:
            time.sleep(opts.poll)
            

if __name__ == "__main__":
    main()