diff --git a/warden3/contrib/connectors/hp-kippo/LICENSE b/warden3/contrib/connectors/hp-kippo/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..b1a5f2252f95e456260aca3e0cab6c244a981891 --- /dev/null +++ b/warden3/contrib/connectors/hp-kippo/LICENSE @@ -0,0 +1,27 @@ +BSD License + +Copyright © 2011-2015 Cesnet z.s.p.o +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the Cesnet z.s.p.o nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE Cesnet z.s.p.o BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/warden3/contrib/connectors/hp-kippo/README b/warden3/contrib/connectors/hp-kippo/README new file mode 100644 index 0000000000000000000000000000000000000000..b2ccf31cbb6f8d12d49fd888852d6d76238cc611 --- /dev/null +++ b/warden3/contrib/connectors/hp-kippo/README @@ -0,0 +1,56 @@ ++-------------------------------------------+ +| Warden Kippo connector 0.1 for Warden 3.X | ++-------------------------------------------+ + +Content + + A. Introduction + B. Dependencies + C. Usage + D. Configuration + +------------------------------------------------------------------------------ +A. Introduction + + Warden Kippo connector (executable warden3-kippo-sender.py) is a one-shot + script to send events from Kippo honeypot toward the Warden server. + +------------------------------------------------------------------------------ +B. Dependencies + + 1. Platform + + Python 2.7+ + + 2. Python packages + + warden_client 3.0+ + +------------------------------------------------------------------------------ +C. Usage + + warden3-kippo-sender.py + + This script does not run as a daemon, for regularly run use job scheduler cron. + +------------------------------------------------------------------------------ +D. Configuration + + warden_client-kippo.cfg + name - sensor's source id used as a source of events, e.g. 'cz.cesnet.server.kippo' + + dbhost - hostname/IP of MySQL DB server + dbuser - username + dbpass - password + dbname - database + dbport - db port + awin - aggregation window, e.g. 5 for events in the last 5 minutes + + cron + SCRIPT_PATH=/opt/warden_client/ + */5 * * * * root cd $SCRIPT_PATH; warden3-kippo-sender.py >> kippo-sender.log + + Note: Repeat interval must be the same as value of 'awin'. + +------------------------------------------------------------------------------ +Copyright (C) 2011-2015 Cesnet z.s.p.o diff --git a/warden3/contrib/connectors/hp-kippo/warden3-kippo-sender.py b/warden3/contrib/connectors/hp-kippo/warden3-kippo-sender.py index cf105f68fb069556e9ae38985388099d342b7751..998e8c73b3b7789d1d95c3dcf6da5bed006eaebe 100644 --- a/warden3/contrib/connectors/hp-kippo/warden3-kippo-sender.py +++ b/warden3/contrib/connectors/hp-kippo/warden3-kippo-sender.py @@ -7,13 +7,10 @@ from warden_client import Client, Error, read_cfg import json import string -from time import time, gmtime +from time import time, gmtime, strftime from math import trunc from uuid import uuid4 -# from pprint import pprint from os import path -# from random import randint, randrange, choice, random; -# from base64 import b64encode; import MySQLdb as my import MySQLdb.cursors as mycursors @@ -25,63 +22,35 @@ def get_precise_timestamp(epoch=None): iso = '%04d-%02d-%02dT%02d:%02d:%02d.%0dZ' % (g[0:6]+(us,)) return iso -def gen_event_idea(client_name, detect_time, win_start_time, win_end_time, conn_count, src_ip4, dst_ip4): +def gen_event_idea(client_name, detect_time, win_start_time, win_end_time, conn_count, src_ip4, dst_ip4, aggr_win): event = { "Format": "IDEA0", "ID": str(uuid4()), - # "CreateTime": get_precise_timestamp(), "DetectTime": detect_time, "WinStartTime": win_start_time, "WinEndTime": win_end_time, - # "EventTime": get_precise_timestamp(), - # "CeaseTime": get_precise_timestamp(), "Category": ["Attempt.Login", "Test"], - # "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()], - # "Confidence": random(), "Note": "SSH login attempt", "ConnCount": conn_count, -# "ConnCount": choice([randint(0, 65535), "asdf"]), # Send wrong event sometimes "Source": [ - { - # "Type": ["Phishing"], - "IP4": [src_ip4], - # "IP6": [randip6() for i in range(randrange(1, 5))], - # "Hostname": ["example.com"], - # "Port": [src_ip4_port], - # "AttachHand": ["att1"], - # "Netname": ["arin:TEST-NET-1"] + { + "IP4": [src_ip4], } ], "Target": [ { "IP4": [dst_ip4], - # "IP6": [randip6() for i in range(randrange(1, 5))], - # "URL": ["http://example.com/%s" % randstr()], "Proto": ["tcp", "ssh"], "Port" : [22] - # "Netname": ["arin:TEST-NET-1"] } ], - # "Attach": [ - # { - # "Handle": "att1", - # "FileName": [randstr()], - # "Type": ["Malware"], - # "ContentType": "application/octet-stream", - # "Hash": ["sha1:%s" % randstr(string.hexdigits, 24)], - # "Size": 46, - # "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())], - # "ContentEncoding": "base64", - # "Content": b64encode(randstr()) - # } - # ], "Node": [ { "Name": client_name, "Tags": ["Connection","Honeypot","Recon"], "SW": ["Kippo"], - "AggrWin": "00:05:00" + "AggrWin": strftime("%H:%M:%S", gmtime(aggr_win)) } ] } @@ -89,42 +58,41 @@ def gen_event_idea(client_name, detect_time, win_start_time, win_end_time, conn_ return event def main(): - wclient = Client(**read_cfg("warden_client.cfg")) - appconf = read_cfg("warden_client-kippo.cfg") + wconfig = read_cfg("warden_client.cfg") + aconfig = read_cfg("warden_client-kippo.cfg") + wconfig['name'] = aconfig['name'] + + wclient = Client(**wconfig) - con = my.connect( host=appconf['dbhost'], user=appconf['dbuser'], passwd=appconf['dbpass'], - db=appconf['dbname'], port=appconf['dbport'], cursorclass=mycursors.DictCursor) + con = my.connect( host=aconfig['dbhost'], user=aconfig['dbuser'], passwd=aconfig['dbpass'], + db=aconfig['dbname'], port=aconfig['dbport'], cursorclass=mycursors.DictCursor) crs = con.cursor() events = [] - query = ["SELECT UNIX_TIMESTAMP(s.starttime) as starttime, s.ip, COUNT(s.id) as attack_scale \ + query = ["SELECT UNIX_TIMESTAMP(s.starttime) as starttime, s.ip, COUNT(s.id) as attack_scale, sn.ip as sensor \ FROM sessions s \ - LEFT JOIN input i ON s.id = i.session \ + LEFT JOIN sensors sn ON s.sensor=sn.id \ WHERE s.starttime > DATE_SUB(UTC_TIMESTAMP(), INTERVAL + %s MINUTE) \ GROUP BY s.ip ORDER BY s.starttime ASC;"] - - # crs.execute("".join(query), [appconf['a_win']]) - crs.execute("".join(query), [5]) + crs.execute("".join(query), aconfig['awin']) rows = crs.fetchall() for row in rows: dtime = get_precise_timestamp(row['starttime']) etime = get_precise_timestamp(time()) - stime = get_precise_timestamp(time() - appconf['a_win'] * 60) - events.append(gen_event_idea(client_name=appconf['name'], detect_time = dtime, win_start_time = stime, win_end_time = etime, conn_count = row['attack_scale'], src_ip4 = row['ip'], dst_ip4 = appconf['sensor_ip4'])) + stime = get_precise_timestamp(time() - aconfig['awin'] * 60) + events.append(gen_event_idea(client_name = aconfig['name'], detect_time = dtime, win_start_time = stime, win_end_time = etime, conn_count = row['attack_scale'], src_ip4 = row['ip'], dst_ip4 = row['sensor'], aggr_win = aconfig['awin'])) print "=== Sending ===" start = time() ret = wclient.sendEvents(events) - if not ret: - print "%d event(s) successfully delivered." % len(rows) - else: - print ret + + if ret: + wclient.logger.info("%d event(s) successfully delivered." % len(rows)) print "Time: %f" % (time()-start) - if __name__ == "__main__": main() diff --git a/warden3/contrib/connectors/hp-kippo/warden_client-kippo.cfg b/warden3/contrib/connectors/hp-kippo/warden_client-kippo.cfg index 5d9c65892ec42750de88d8d5db8343a4cd0a247c..e529c83faaad3f29f1f5088956b2d2faf2b0e067 100644 --- a/warden3/contrib/connectors/hp-kippo/warden_client-kippo.cfg +++ b/warden3/contrib/connectors/hp-kippo/warden_client-kippo.cfg @@ -2,7 +2,6 @@ "name": "cz.cesnet.server.kippo", "sensor_ip4": "195.113.x.x", - "idstore": "warden_client.id", "dbhost": "localhost", "dbuser": "kippo", "dbpass": "kippopass", diff --git a/warden3/contrib/connectors/hp-kippo/warden_client.cfg b/warden3/contrib/connectors/hp-kippo/warden_client.cfg deleted file mode 100644 index 9d5daa6d98557fc8d5e665dc07b716cff58f8023..0000000000000000000000000000000000000000 --- a/warden3/contrib/connectors/hp-kippo/warden_client.cfg +++ /dev/null @@ -1,18 +0,0 @@ -{ - "url": "https://warden-dev.cesnet.cz/warden3", - - "certfile": "/etc/ssl/certs/server.cesnet.cz.pem", - "keyfile": "/etc/ssl/private/server.cesnet.cz.key", - "cafile": "/etc/ssl/certs/tcs-ca-bundle.pem", - - "timeout": 60, - "recv_events_limit": 6000, - - "errlog": {"level": "debug"}, - "filelog": {"file": "warden_client.log", "level": "warning"}, - #"syslog": {"socket": "/dev/log", "facility": "local7", "level": "warning"}, - - #"name": "xxx" - "secret": "xxxxxxxxxxxxxxxxxxxx" -} - diff --git a/warden3/contrib/connectors/hp-kippo/warden_client.py b/warden3/contrib/connectors/hp-kippo/warden_client.py deleted file mode 100644 index 6a807fca16533ec96adcd7156123f6d395b33eaa..0000000000000000000000000000000000000000 --- a/warden3/contrib/connectors/hp-kippo/warden_client.py +++ /dev/null @@ -1,425 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2011-2013 Cesnet z.s.p.o -# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file. - -import json, httplib, ssl, socket, logging, logging.handlers -from urlparse import urlparse -from urllib import urlencode -from sys import stderr, exc_info -from pprint import pformat -from traceback import format_tb -from os import path - - -class HTTPSConnection(httplib.HTTPSConnection): - ''' - Overridden to allow peer certificate validation, configuration - of SSL/ TLS version and cipher selection. See: - http://hg.python.org/cpython/file/c1c45755397b/Lib/httplib.py#l1144 - and `ssl.wrap_socket()` - ''' - def __init__(self, host, **kwargs): - self.ciphers = kwargs.pop('ciphers',None) - self.ca_certs = kwargs.pop('ca_certs',None) - self.ssl_version = kwargs.pop('ssl_version',ssl.PROTOCOL_SSLv23) - - httplib.HTTPSConnection.__init__(self,host,**kwargs) - - def connect(self): - sock = socket.create_connection( (self.host, self.port), self.timeout) - - if self._tunnel_host: - self.sock = sock - self._tunnel() - - self.sock = ssl.wrap_socket( - sock, - keyfile = self.key_file, - certfile = self.cert_file, - ca_certs = self.ca_certs, - cert_reqs = ssl.CERT_REQUIRED if self.ca_certs else ssl.CERT_NONE, - ssl_version = self.ssl_version) - - - -class Error(Exception): - """ Object for returning error messages to calling application. - Caller can test whether it received data or error by checking - isinstance(res, Error). - However if he does not want to deal with errors altogether, - this error object also returns False value if used in Bool - context (e.g. in "if res: print res" print is not evaluated), - and also acts as empty iterator (e.g. in "for e in res: print e" - print is also not evaluated). - Also, it can be raised as an exception. - """ - - def __init__(self, message, logger=None, error=None, prio="error", method=None, - req_id=None, detail=None, exc=None): - - self.message = message - self.error = error - self.method = method - self.req_id = req_id - self.detail = detail - (self.exctype, self.excval, self.exctb) = exc or (None, None, None) - self.cause = self.excval # compatibility with other exceptions - if logger: - getattr(logger, prio, "error")(str(self)) - info = self.info_str() - if info: - logger.info(info) - debug = self.debug_str() - if debug: - logger.debug(debug) - - - def __len__ (self): - """ In list or iterable context we're empty """ - return 0 - - - def __iter__(self): - """ We are the iterator """ - return self - - - def next(self): - """ In list or iterable context we're empty """ - raise StopIteration - - - def __bool__(self): - """ In boolean context we're never True """ - return False - - - def __str__(self): - out = [] - out.append("(%s)" % (self.error or "local")) - if self.method is not None: - out.append(" in %s" % self.method) - if self.req_id is not None: - out.append("(%8x)" % self.req_id) - if self.message is not None: - out.append(": %s" % self.message) - if self.excval is not None: - out.append(" - cause was %s: %s" % (type(self.excval).__name__, str(self.excval))) - return "".join(out) - - - def info_str(self): - return ("Detail: %s" % pformat(self.detail)) or "" - - - def debug_str(self): - out = [] - if self.excval is not None: - out.append("Exception %s: %s\n" % (type(self.excval).__name__, str(self.excval))) - if self.exctb is not None: - out.append("Traceback:\n%s" % "".join(format_tb(self.exctb))) - return "".join(out) - - - -class Client(object): - - def __init__(self, - url, - certfile=None, - keyfile=None, - cafile=None, - timeout=60, - recv_events_limit=6000, - errlog={"level": "debug"}, - syslog=None, - filelog=None, - idstore=None, - name="warden_client", - secret=None): - - self.name = name - self.secret = secret - # Init logging as soon as possible and make sure we don't - # spit out exceptions but just log or return Error objects - self.init_log(errlog, syslog, filelog) - - self.url = urlparse(url, allow_fragments=False) - - self.conn = None - - base = path.join(path.dirname(__file__)) - self.certfile = path.join(base, certfile or "cert.pem") - self.keyfile = path.join(base, keyfile or "key.pem") - self.cafile = path.join(base, cafile or "ca.pem") - self.timeout = int(timeout) - self.recv_events_limit = int(recv_events_limit) - self.idstore = path.join(base, idstore) if idstore is not None else None - - self.ciphers = 'TLS_RSA_WITH_AES_256_CBC_SHA' - self.sslversion = ssl.PROTOCOL_TLSv1 - - - def init_log(self, errlog, syslog, filelog): - - def loglevel(lev): - try: - return int(getattr(logging, lev.upper())) - except (AttributeError, ValueError): - self.logger.warning("Unknown loglevel \"%s\", using \"debug\"" % lev) - return logging.DEBUG - - def facility(fac): - try: - return int(getattr(logging.handlers.SysLogHandler, "LOG_" + fac.upper())) - except (AttributeError, ValueError): - self.logger.warning("Unknown syslog facility \"%s\", using \"local7\"" % fac) - return logging.handlers.SysLogHandler.LOG_LOCAL7 - - form = "%(filename)s[%(process)d]: (%(levelname)s) %(name)s %(message)s" - format_notime = logging.Formatter(form) - format_time = logging.Formatter('%(asctime)s ' + form) - - self.logger = logging.getLogger(self.name) - self.logger.propagate = False # Don't bubble up to root logger - self.logger.setLevel(logging.DEBUG) - - if errlog is not None: - el = logging.StreamHandler(stderr) - el.setFormatter(format_time) - el.setLevel(loglevel(errlog.get("level", "debug"))) - self.logger.addHandler(el) - - if filelog is not None: - try: - fl = logging.FileHandler( - filename=path.join( - path.dirname(__file__), - filelog.get("file", "%s.log" % self.name))) - fl.setLevel(loglevel(filelog.get("level", "warning"))) - fl.setFormatter(format_time) - self.logger.addHandler(fl) - except Exception as e: - Error("Unable to setup file logging", self.logger, exc=exc_info()) - - if syslog is not None: - try: - sl = logging.handlers.SysLogHandler( - address=syslog.get("socket", "/dev/log"), - facility=facility(syslog.get("facility", "local7"))) - sl.setLevel(loglevel(syslog.get("level", "warning"))) - sl.setFormatter(format_notime) - self.logger.addHandler(sl) - except Exception as e: - Error("Unable to setup syslog logging", self.logger, exc=exc_info()) - - if not (errlog or filelog or syslog): - # User wants explicitly no logging, so let him shoot his socks off. - # This silences complaining of logging module about no suitable - # handler. - self.logger.addHandler(logging.NullHandler()) - - - def connect(self): - - try: - if self.url.scheme=="https": - self.conn = HTTPSConnection( - self.url.netloc, - key_file = self.keyfile, - cert_file = self.certfile, - timeout = self.timeout, - ciphers = self.ciphers, - ca_certs = self.cafile, - ssl_version = self.sslversion) - elif self.url.scheme=="http": - self.conn = httplib.HTTPConnection( - self.url.netloc, - timeout = self.timeout) - else: - return Error("Don't know how to connect to \"%s\"" % self.url.scheme, self.logger, - detail={"url": self.url.geturl()}) - except Exception: - return Error("HTTPS connection failed", self.logger, exc=exc_info(), - detail={ - "url": self.url.geturl(), - "timeout": self.timeout, - "key_file": self.keyfile, - "cert_file": self.certfile, - "cafile": self.cafile, - "ciphers": self.ciphers, - "ssl_version": self.sslversion}) - - return True - - - def sendRequest(self, func="", payload=None, **kwargs): - - if self.secret is None: - kwargs["client"] = self.name - else: - kwargs["secret"] = self.secret - - if kwargs: - for k in kwargs.keys(): - if kwargs[k] is None: - del kwargs[k] - argurl = "?" + urlencode(kwargs, doseq=True) - else: - argurl = "" - - try: - if payload is None: - data = "" - else: - data = json.dumps(payload) - except: - return Error("Serialization to JSON failed", self.logger, - exc=exc_info(), method=func, detail=payload) - - self.headers = { - "Content-Type": "application/json", - "Accept": "application/json", - "Content-Length": str(len(data)) - } - - # We are connecting here at first use instead of in - # constructor, because constructor cannot return data/errors - # and we don't want to spit exceptions into user's face - # And maaaybee sometime we will implement reconnection on errors - if self.conn is None: - err = self.connect() - if not err: - return err # either False of Error instance - - loc = '%s/%s%s' % (self.url.path, func, argurl) - try: - self.conn.request("POST", loc, data, self.headers) - except: - return Error("Sending of request to server failed", self.logger, - exc=exc_info(), method=func, detail={ - "loc": loc, - "headers": self.headers, - "data": data}) - - try: - res = self.conn.getresponse() - except: - return Error("HTTP reply failed", self.logger, method=func, exc=exc_info()) - - try: - response_data = res.read() - except: - return Error("Fetching HTTP data from server failed", self.logger, method=func, exc=exc_info()) - - if res.status==httplib.OK: - try: - data = json.loads(response_data) - except: - data = Error("JSON message parsing failed", self.logger, - exc=exc_info(), method=func, detail={"response": response_data}) - else: - try: - data = json.loads(response_data) - data["error"] # trigger exception if not dict or no error key - except: - data = Error("Generic server HTTP error", self.logger, - method=func, - error=res.status, - exc=exc_info(), - detail={"response": response_data}) - else: - data = Error(data.get("message", None), self.logger, - method=data.get("method", None), - error=res.status, - req_id=data.get("req_id", None), - detail=data.get("detail", None)) - - return data - - - def _saveID(self, id, idstore=None): - idf = idstore or self.idstore - if not idf: - return False - try: - with open(idf, "w+") as f: - f.write(str(id)) - except (ValueError, IOError) as e: - # Use Error instance just for proper logging - Error("Writing id file \"%s\" failed" % idf, self.logger, - prio="info", exc=exc_info(), detail={"idstore": idf}) - return id - - - def _loadID(self, idstore=None): - idf = idstore or self.idstore - if not idf: - return None - try: - with open(idf, "r") as f: - id = int(f.read()) - except (ValueError, IOError) as e: - Error("Reading id file \"%s\" failed, relying on server" % idf, - self.logger, prio="info", exc=exc_info(), detail={"idstore": idf}) - id = None - return id - - - def getDebug(self): - return self.sendRequest("getDebug") - - - def getInfo(self): - return self.sendRequest("getInfo") - - - def sendEvents(self, events=[]): - res = self.sendRequest( - "sendEvents", payload=events) - return res - - - def getEvents(self, id=None, idstore=None, count=1, - cat=None, nocat=None, - tag=None, notag=None, - group=None, nogroup=None): - - if not id: - id = self._loadID(idstore) - - res = self.sendRequest( - "getEvents", id=id, count=count or self.recv_events_limit, cat=cat, - nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup) - if not res: - return res # Should be Error instance - - try: - events = res["events"] - newid = res["lastid"] - except KeyError: - return Error("Server returned bogus reply", self.logger, - method="getEvents", exc=exc_info(), detail={"response": res}) - - self._saveID(newid) - - return events - - - def close(self): - - if hasattr(self, "conn") and hasattr(self.conn, "close"): - self.conn.close() - - - __del__ = close - - - -def read_cfg(cfgfile): - abspath = path.join(path.dirname(__file__), cfgfile) - with open(abspath, "r") as f: - stripcomments = "\n".join((l for l in f if not l.lstrip().startswith("#"))) - return json.loads(stripcomments)