Skip to content
Snippets Groups Projects
Commit 3c69d477 authored by Pavel Kácha's avatar Pavel Kácha
Browse files

Remove hp-tipping-point tree as development continued in tippingpoint tree

parent 558fc26f
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
hp-tipping-point/tpToIdea.py deleted 100644 → 0
+ 0
− 522
View file @ 558fc26f
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017-2018 Cesnet z.s.p.o
# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
import json
from uuid import uuid4
import re
import socket
import optparse
import sys
import os
import signal
import resource
import os.path as pth
import atexit
import time
from datetime import datetime
import logging
from collections import OrderedDict
class FileWatcher(object):
def __init__(self, filename, tail=True):
self.filename = filename
self.open()
self.line_buffer = ""
if tail and self.f:
self.f.seek(0, os.SEEK_END)
def open(self):
try:
self.f = open(self.filename, "r")
st = os.fstat(self.f.fileno())
self.inode, self.size = st.st_ino, st.st_size
except IOError:
self.f = None
self.inode = -1
self.size = -1
def _check_reopen(self):
try:
st = os.stat(self.filename)
cur_inode, cur_size = st.st_ino, st.st_size
except OSError as e:
cur_inode = -1
cur_size = -1
if cur_inode != self.inode or cur_size < self.size:
self.close()
self.open()
def readline(self):
if not self.f:
self.open()
if not self.f:
return self.line_buffer
res = self.f.readline()
if not res:
self._check_reopen()
if not self.f:
return self.line_buffer
res = self.f.readline()
if not res.endswith("\n"):
self.line_buffer += res
else:
res = self.line_buffer + res
self.line_buffer = ""
return res
def close(self):
try:
if self.f:
self.f.close()
except IOError:
pass
self.inode = -1
self.size = -1
def __repr__(self):
return '%s("%s")' % (type(self).__name__, self.filename)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
return False
def __iter__(self):
return self
def next(self):
return self.readline()
class Filer(object):
def __init__(self, directory):
self.basedir = self._ensure_path(directory)
self.tmp = self._ensure_path(pth.join(self.basedir, "tmp"))
self.incoming = self._ensure_path(pth.join(self.basedir, "incoming"))
self.hostname = socket.gethostname()
self.pid = os.getpid()
def _ensure_path(self, p):
try:
os.mkdir(p)
except OSError:
if not pth.isdir(p):
raise
return p
def _get_new_name(self, fd=None):
(inode, device) = os.fstat(fd)[1:3] if fd else (0, 0)
return "%s.%d.%f.%d.%d" % (
self.hostname, self.pid, time.time(), device, inode)
def create_unique_file(self):
# First find and open name unique within tmp
tmpname = None
while not tmpname:
tmpname = self._get_new_name()
try:
fd = os.open(pth.join(self.tmp, tmpname), os.O_CREAT | os.O_RDWR | os.O_EXCL)
except OSError as e:
if e.errno != errno.EEXIST:
raise # other errors than duplicates should get noticed
tmpname = None
# Now we know the device/inode, rename to make unique within system
newname = self._get_new_name(fd)
os.rename(pth.join(self.tmp, tmpname), pth.join(self.tmp, newname))
nf = os.fdopen(fd, "w")
return nf, newname
def publish_file(self, short_name):
os.rename(pth.join(self.tmp, short_name), pth.join(self.incoming, short_name))
class IdeaGen(object):
tp_to_idea = {
1: {
1: ["Attempt.Exploit"],
2: ["Attempt.Exploit"],
3: ["Attempt.Exploit"],
4: ["Attempt.Exploit"],
5: ["Attempt.Exploit"],
6: ["Attempt.Exploit"],
255: ["Attempt.Exploit"]
},
2: {
1: ["Malware.Worm"],
2: ["Malware.Virus"],
3: ["Malware.Trojan"],
4: ["Intrusion.Botnet"],
5: ["Fraud.Phishing"],
255: ["Malware"]
},
3: {
1: ["Availability.DDoS"],
2: ["Availability.DDoS"],
3: ["Availability.DDoS"],
255: ["Availability.DDoS"]
},
4: {
1: ["Other"],
2: ["Other"],
3: ["Other"],
4: ["Other"],
5: ["Other"],
6: ["Attempt.Login"],
7: ["Malware.Spyware"],
255: ["Other"]
},
5: {
1: ["Recon.Scanning"],
2: ["Attempt.Exploit"],
3: ["Attempt.Exploit"],
4: ["Recon.Scanning", "Attempt.Exploit"],
255: ["Attempt.Exploit"]
},
6: {
1: ["Anomaly.Protocol"],
2: ["Anomaly.Traffic"],
3: ["Anomaly.Application"],
255: ["Anomaly"]
},
7: {
1: ["Anomaly.Traffic"],
2: ["Anomaly.Application"],
255: ["Anomaly.Traffic"]
},
8: {
1: ["Other"],
2: ["Other"],
255: ["Other"]
},
}
def __init__(self, name, test=False, other=False):
self.name = name
self.test = test
self.other = other
def convert_category(self, category, id_taxonomy):
'''
converts category from record to IDEA category
:param category: TippingPoint category description
:param id_taxonomy: TippingPoint taxonomy id
:return: if category or incident is empty or is not important for saving it return None, otherwise return
converted category
'''
if not (category and id_taxonomy):
return None
tp_cat_maj = id_taxonomy >> 24
tp_cat_min = id_taxonomy >> 16 & 0b11111111
tp_proto = id_taxonomy >> 8 & 0b11111111
tp_platf = id_taxonomy & 0b11111111
try:
category = IdeaGen.tp_to_idea[tp_cat_maj][tp_cat_min]
except KeyError:
category = ["Other"]
return category
def gen_event_idea(self, timestamp, category, id_taxonomy, cve, filter_name, proto, src_ip, src_port,
dest_ip, dest_port, conn_count, url, severity, orig_data):
'''
put every piece of record together into IDEA message
:return: new IDEA message
'''
if (category == ["Other"]) and not self.other:
return None
event = OrderedDict([
("Format","IDEA0"),
("ID", str(uuid4())),
("DetectTime", datetime.fromtimestamp(timestamp / 1000).isoformat() + 'Z'),
("Category", category + (["Test"] if self.test else [])),
])
if cve:
event['Ref'] = ['urn:cve:'.format(i) for i in cve]
if conn_count and int(conn_count):
event['ConnCount'] = int(conn_count)
source = OrderedDict()
target = OrderedDict()
if src_ip:
# TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
src_ip = src_ip.replace(" ", "")
af = "IP4" if not ':' in src_ip else "IP6"
source[af] = [src_ip]
if src_port and int(src_port):
source['Port'] = [int(src_port)]
if proto:
source['Proto'] = [proto]
if dest_ip and (dest_ip != "0.0.0.0"):
# TippingPoint vSMS bugfix: Remove excessive spaces occasionally included inside the IPv6 address
dest_ip = dest_ip.replace(" ", "")
af = "IP4" if not ':' in dest_ip else "IP6"
target[af] = [dest_ip]
if dest_port and int(dest_port):
target['Port'] = [int(dest_port)]
if proto:
target['Proto'] = [proto]
if url:
target['URL'] = url
if source:
event['Source'] = [source]
if target:
event['Target'] = [target]
if orig_data:
event['Attach'] = [OrderedDict([
('Type', ["OrigData"]),
('Content', orig_data.strip())
])]
event['Node'] = [OrderedDict([
('Name', self.name),
('Type', ["Datagram", "Content", "Protocol", "Signature", "Policy", "Heuristic"]),
('SW', ["TippingPoint_NX_NGIPS"])
])]
return event
def daemonize(
work_dir=None, chroot_dir=None,
umask=None, uid=None, gid=None,
pidfile=None, files_preserve=[], signals={}):
# Dirs, limits, users
if chroot_dir is not None:
os.chdir(chroot_dir)
os.chroot(chroot_dir)
if umask is not None:
os.umask(umask)
if work_dir is not None:
os.chdir(work_dir)
if gid is not None:
os.setgid(gid)
if uid is not None:
os.setuid(uid)
# Doublefork, split session
if os.fork() > 0:
os._exit(0)
try:
os.setsid()
except OSError:
pass
if os.fork() > 0:
os._exit(0)
# Setup signal handlers
for (signum, handler) in signals.items():
signal.signal(signum, handler)
# Close descriptors
descr_preserve = set(f.fileno() for f in files_preserve)
maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
if maxfd == resource.RLIM_INFINITY:
maxfd = 65535
for fd in range(maxfd, 3, -1): # 3 means omit stdin, stdout, stderr
if fd not in descr_preserve:
try:
os.close(fd)
except Exception:
pass
# Redirect stdin, stdout, stderr to /dev/null
devnull = os.open(os.devnull, os.O_RDWR)
for fd in range(3):
os.dup2(devnull, fd)
# PID file
if pidfile is not None:
pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
os.write(pidd, str(os.getpid())+"\n")
os.close(pidd)
# Define and setup atexit closure
@atexit.register
def unlink_pid():
try:
os.unlink(pidfile)
except Exception:
pass
def get_args():
optp = optparse.OptionParser(
usage="usage: %prog [options] logfile ...",
description="Watch TippingPoint logfiles and generate Idea events into directory")
optp.add_option(
"-n", "--name",
default=None,
dest="name",
type="string",
action="store",
help="Warden client name")
optp.add_option(
"--test",
default=False,
dest="test",
action="store_true",
help="Add \"Test\" category")
optp.add_option(
"--other",
default=False,
dest="other",
action="store_true",
help="Send events having \"Other\" category (usually nonmalicious)")
optp.add_option(
"-o", "--oneshot",
default=False,
dest="oneshot",
action="store_true",
help="process files and quit (do not daemonize)")
optp.add_option(
"--poll",
default=1,
dest="poll",
type="int",
action="store",
help="log file polling interval")
optp.add_option(
"-d", "--dir",
default=None,
dest="dir",
type="string",
action="store",
help="Target directory (mandatory)")
optp.add_option(
"-p", "--pid",
default=pth.join("/var/run", pth.splitext(pth.basename(sys.argv[0]))[0] + ".pid"),
dest="pid",
type="string",
action="store",
help="create PID file with this name (default: %default)")
optp.add_option(
"-u", "--uid",
default=None,
dest="uid",
type="int",
action="store",
help="user id to run under")
optp.add_option(
"-g", "--gid",
default=None,
dest="gid",
type="int",
action="store",
help="group id to run under")
optp.add_option(
"--origdata",
default=False,
dest="origdata",
action="store_true",
help="Store original report to IDEA message")
return optp
def not_empty(test_string):
# tests if string is not empty
return None if test_string.strip() in ["", "null"] else test_string
def save_events(event, filer):
f, name = filer.create_unique_file()
with f:
f.write(json.dumps(event, ensure_ascii=True))
filer.publish_file(name)
def process_data(line, filer, origdata, idea_gen):
'''
takes one record, parse it to parameters, give it to Ideagen and writes to file
:param line: one record
:param idea_file: where output goes
:param origdata: if true, write original record to IDEA message
'''
row = line.split("|")
category = idea_gen.convert_category(category=row[1], id_taxonomy=int(row[2]))
timestamp = row[0].split(" ")[-1]
cve = [i for i in row[3].split(",") if i not in ("null", "")]
odata = "|".join([timestamp] + row[1:])
if category and not_empty(row[0][-14:-3]):
idea_event = idea_gen.gen_event_idea(timestamp=int(timestamp), category=category, id_taxonomy=int(row[2]),
cve=cve, filter_name=not_empty(row[4]), proto=not_empty(row[5]),
src_ip=not_empty(row[6]), src_port=not_empty(row[7]), dest_ip=not_empty(row[8]), dest_port=not_empty(row[9]),
conn_count=not_empty(row[10]), severity=not_empty(row[11]), url=not_empty(row[12]),
orig_data=odata if origdata else False)
if idea_event:
save_events(idea_event, filer)
running_flag = True
reload_flag = False
def terminate_me(signum, frame):
global running_flag
running_flag = False
def reload_me(signum, frame):
global reload_flag
reload_flag = True
def main():
global running_flag
global reload_flag
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG, filename='tipping_point_log.log', filemode='w')
optp = get_args()
opts, args = optp.parse_args()
if not args or opts.name is None or opts.dir is None:
optp.print_help()
sys.exit()
if opts.oneshot:
signal.signal(signal.SIGINT, terminate_me)
signal.signal(signal.SIGTERM, terminate_me)
files = [open(arg) for arg in args]
else:
daemonize(
pidfile=opts.pid,
uid=opts.uid,
gid=opts.gid,
signals={
signal.SIGINT: terminate_me,
signal.SIGTERM: terminate_me,
signal.SIGHUP: reload_me
})
files = [FileWatcher(arg) for arg in args]
filer = Filer(opts.dir)
idea_gen = IdeaGen(opts.name, opts.test, opts.other)
while running_flag:
for log_file in files:
while True:
line = log_file.readline()
if line is None or not line.strip():
logging.info("no line")
break
logging.info("readline")
process_data(line, filer, opts.origdata, idea_gen)
if not running_flag:
break
if reload_flag:
for f in files:
f.close()
f.open()
reload_flag = False
if opts.oneshot:
break
else:
time.sleep(opts.poll)
if __name__ == "__main__":
main()
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment