Kippo connector - initial commit

warden3/contrib/connectors/hp-kippo/warden3-kippo-sender.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2015 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+from warden_client import Client, Error, read_cfg
+import json
+import string
+from time import time, gmtime
+from math import trunc
+from uuid import uuid4
+# from pprint import pprint
+from os import path
+# from random import randint, randrange, choice, random;
+# from base64 import b64encode;
+
+import MySQLdb as my
+import MySQLdb.cursors as mycursors
+
+def get_precise_timestamp(epoch=None):
+    t = epoch if epoch else time()
+    us = trunc((t-trunc(t))*1000000)
+    g = gmtime(t)
+    iso = '%04d-%02d-%02dT%02d:%02d:%02d.%0dZ' % (g[0:6]+(us,))
+    return iso
+
+def gen_event_idea(client_name, detect_time, win_start_time, win_end_time, conn_count, src_ip4, dst_ip4):
+
+  event = {
+     "Format": "IDEA0",
+     "ID": str(uuid4()),
+     # "CreateTime": get_precise_timestamp(),
+     "DetectTime": detect_time,
+     "WinStartTime": win_start_time,
+     "WinEndTime": win_end_time,
+     # "EventTime": get_precise_timestamp(),
+     # "CeaseTime": get_precise_timestamp(),
+     "Category": ["Attempt.Login", "Test"],
+     # "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()],
+     # "Confidence": random(),
+     "Note": "SSH login attempt",
+     "ConnCount": conn_count,
+#       "ConnCount": choice([randint(0, 65535), "asdf"]),    # Send wrong event sometimes
+     "Source": [
+        {
+           # "Type": ["Phishing"],
+           "IP4": [src_ip4],
+           # "IP6": [randip6() for i in range(randrange(1, 5))],
+           # "Hostname": ["example.com"],
+           # "Port": [src_ip4_port],
+           # "AttachHand": ["att1"],
+           # "Netname": ["arin:TEST-NET-1"]
+        }
+     ],
+     "Target": [
+        {
+           "IP4": [dst_ip4],
+           # "IP6": [randip6() for i in range(randrange(1, 5))],
+           # "URL": ["http://example.com/%s" % randstr()],
+           "Proto": ["tcp", "ssh"],
+           "Port" : [22]
+           # "Netname": ["arin:TEST-NET-1"]
+        }
+     ],
+     # "Attach": [
+     #    {
+     #       "Handle": "att1",
+     #       "FileName": [randstr()],
+     #       "Type": ["Malware"],
+     #       "ContentType": "application/octet-stream",
+     #       "Hash": ["sha1:%s" % randstr(string.hexdigits, 24)],
+     #       "Size": 46,
+     #       "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())],
+     #       "ContentEncoding": "base64",
+     #       "Content": b64encode(randstr())
+     #    }
+     # ],
+     "Node": [
+        {
+           "Name": client_name,
+           "Tags": ["Connection","Honeypot","Recon"],
+           "SW": ["Kippo"],
+           "AggrWin": "00:05:00"
+        }
+     ]
+  }
+
+  return event
+
+def main():
+    wclient = Client(**read_cfg("warden_client.cfg"))
+    appconf = read_cfg("warden_client-kippo.cfg")
+
+    con = my.connect( host=appconf['dbhost'], user=appconf['dbuser'], passwd=appconf['dbpass'],
+                      db=appconf['dbname'], port=appconf['dbport'], cursorclass=mycursors.DictCursor)
+    
+    crs = con.cursor()
+
+    events = []
+    query = ["SELECT UNIX_TIMESTAMP(s.starttime) as starttime, s.ip, COUNT(s.id) as attack_scale \
+              FROM sessions s \
+              LEFT JOIN input i ON s.id = i.session \
+              WHERE s.starttime > DATE_SUB(UTC_TIMESTAMP(), INTERVAL + %s MINUTE) \
+              GROUP BY s.ip ORDER BY s.starttime ASC;"]
+
+
+    # crs.execute("".join(query), [appconf['a_win']])
+    crs.execute("".join(query), [5])
+    rows = crs.fetchall()
+    for row in rows:
+      dtime = get_precise_timestamp(row['starttime'])
+      etime = get_precise_timestamp(time())
+      stime = get_precise_timestamp(time() - appconf['a_win'] * 60)
+      events.append(gen_event_idea(client_name=appconf['name'], detect_time = dtime, win_start_time = stime, win_end_time = etime, conn_count = row['attack_scale'], src_ip4 = row['ip'], dst_ip4 = appconf['sensor_ip4']))
+        
+    print "=== Sending ==="
+    start = time()
+    ret = wclient.sendEvents(events)
+    if not ret:
+        print "%d event(s) successfully delivered." % len(rows)
+    else:
+        print ret
+
+    print "Time: %f" % (time()-start)
+
+
+
+if __name__ == "__main__":
+    main()

warden3/contrib/connectors/hp-kippo/warden_client-kippo.cfg

+{
+    "name": "cz.cesnet.server.kippo",
+    "sensor_ip4": "195.113.x.x",
+
+    "idstore": "warden_client.id",
+    "dbhost": "localhost",
+    "dbuser": "kippo",
+    "dbpass": "kippopass",
+    "dbname": "kippo",
+    "dbport": 3306,
+    "a_win": 5
+}

warden3/contrib/connectors/hp-kippo/warden_client.cfg

+{
+    "url": "https://warden-dev.cesnet.cz/warden3",
+
+    "certfile": "/etc/ssl/certs/server.cesnet.cz.pem",
+    "keyfile": "/etc/ssl/private/server.cesnet.cz.key",
+    "cafile": "/etc/ssl/certs/tcs-ca-bundle.pem",
+
+    "timeout": 60,
+    "recv_events_limit": 6000,
+
+    "errlog": {"level": "debug"},
+    "filelog": {"file": "warden_client.log", "level": "warning"},
+    #"syslog": {"socket": "/dev/log", "facility": "local7", "level": "warning"},
+
+    #"name": "xxx"
+    "secret": "xxxxxxxxxxxxxxxxxxxx"
+}
+

warden3/contrib/connectors/hp-kippo/warden_client.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2013 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+import json, httplib, ssl, socket, logging, logging.handlers
+from urlparse import urlparse
+from urllib import urlencode
+from sys import stderr, exc_info
+from pprint import pformat
+from traceback import format_tb
+from os import path
+
+
+class HTTPSConnection(httplib.HTTPSConnection):
+    '''
+    Overridden to allow peer certificate validation, configuration
+    of SSL/ TLS version and cipher selection.  See:
+    http://hg.python.org/cpython/file/c1c45755397b/Lib/httplib.py#l1144
+    and `ssl.wrap_socket()`
+    '''
+    def __init__(self, host, **kwargs):
+        self.ciphers = kwargs.pop('ciphers',None)
+        self.ca_certs = kwargs.pop('ca_certs',None)
+        self.ssl_version = kwargs.pop('ssl_version',ssl.PROTOCOL_SSLv23)
+
+        httplib.HTTPSConnection.__init__(self,host,**kwargs)
+
+    def connect(self):
+        sock = socket.create_connection( (self.host, self.port), self.timeout)
+
+        if self._tunnel_host:
+            self.sock = sock
+            self._tunnel()
+
+        self.sock = ssl.wrap_socket(
+            sock,
+            keyfile = self.key_file,
+            certfile = self.cert_file,
+            ca_certs = self.ca_certs,
+            cert_reqs = ssl.CERT_REQUIRED if self.ca_certs else ssl.CERT_NONE,
+            ssl_version = self.ssl_version)
+
+
+
+class Error(Exception):
+    """ Object for returning error messages to calling application.
+        Caller can test whether it received data or error by checking
+        isinstance(res, Error).
+        However if he does not want to deal with errors altogether,
+        this error object also returns False value if used in Bool
+        context (e.g. in "if res: print res" print is not evaluated),
+        and also acts as empty iterator (e.g. in "for e in res: print e"
+        print is also not evaluated).
+        Also, it can be raised as an exception.
+    """
+
+    def __init__(self, message, logger=None, error=None, prio="error", method=None,
+            req_id=None, detail=None, exc=None):
+
+        self.message = message
+        self.error = error
+        self.method = method
+        self.req_id = req_id
+        self.detail = detail
+        (self.exctype, self.excval, self.exctb) = exc or (None, None, None)
+        self.cause = self.excval # compatibility with other exceptions
+        if logger:
+            getattr(logger, prio, "error")(str(self))
+            info = self.info_str()
+            if info:
+                logger.info(info)
+            debug = self.debug_str()
+            if debug:
+                logger.debug(debug)
+
+
+    def __len__ (self):
+        """ In list or iterable context we're empty """
+        return 0
+
+
+    def __iter__(self):
+        """ We are the iterator """
+        return self
+
+
+    def next(self):
+        """ In list or iterable context we're empty """
+        raise StopIteration
+
+
+    def __bool__(self):
+        """ In boolean context we're never True """
+        return False
+
+
+    def __str__(self):
+        out = []
+        out.append("(%s)" % (self.error or "local"))
+        if self.method is not None:
+            out.append(" in %s" % self.method)
+        if self.req_id is not None:
+            out.append("(%8x)" % self.req_id)
+        if self.message is not None:
+            out.append(": %s" % self.message)
+        if self.excval is not None:
+            out.append(" - cause was %s: %s" % (type(self.excval).__name__, str(self.excval)))
+        return "".join(out)
+
+
+    def info_str(self):
+        return ("Detail: %s" % pformat(self.detail)) or ""
+
+
+    def debug_str(self):
+        out = []
+        if self.excval is not None:
+            out.append("Exception %s: %s\n" % (type(self.excval).__name__, str(self.excval)))
+        if self.exctb is not None:
+            out.append("Traceback:\n%s" % "".join(format_tb(self.exctb)))
+        return "".join(out)
+
+
+
+class Client(object):
+
+    def __init__(self,
+            url,
+            certfile=None,
+            keyfile=None,
+            cafile=None,
+            timeout=60,
+            recv_events_limit=6000,
+            errlog={"level": "debug"},
+            syslog=None,
+            filelog=None,
+            idstore=None,
+            name="warden_client",
+            secret=None):
+
+        self.name = name
+        self.secret = secret
+        # Init logging as soon as possible and make sure we don't
+        # spit out exceptions but just log or return Error objects
+        self.init_log(errlog, syslog, filelog)
+
+        self.url = urlparse(url, allow_fragments=False)
+
+        self.conn = None
+
+        base = path.join(path.dirname(__file__))
+        self.certfile = path.join(base, certfile or "cert.pem")
+        self.keyfile  = path.join(base, keyfile or "key.pem")
+        self.cafile = path.join(base, cafile or "ca.pem")
+        self.timeout = int(timeout)
+        self.recv_events_limit = int(recv_events_limit)
+        self.idstore = path.join(base, idstore) if idstore is not None else None
+
+        self.ciphers = 'TLS_RSA_WITH_AES_256_CBC_SHA'
+        self.sslversion = ssl.PROTOCOL_TLSv1
+
+
+    def init_log(self, errlog, syslog, filelog):
+
+        def loglevel(lev):
+            try:
+                return int(getattr(logging, lev.upper()))
+            except (AttributeError, ValueError):
+                self.logger.warning("Unknown loglevel \"%s\", using \"debug\"" % lev)
+                return logging.DEBUG
+
+        def facility(fac):
+            try:
+                return int(getattr(logging.handlers.SysLogHandler, "LOG_" + fac.upper()))
+            except (AttributeError, ValueError):
+                self.logger.warning("Unknown syslog facility \"%s\", using \"local7\"" % fac)
+                return logging.handlers.SysLogHandler.LOG_LOCAL7
+
+        form = "%(filename)s[%(process)d]: (%(levelname)s) %(name)s %(message)s"
+        format_notime = logging.Formatter(form)
+        format_time = logging.Formatter('%(asctime)s ' + form)
+
+        self.logger = logging.getLogger(self.name)
+        self.logger.propagate = False   # Don't bubble up to root logger
+        self.logger.setLevel(logging.DEBUG)
+
+        if errlog is not None:
+            el = logging.StreamHandler(stderr)
+            el.setFormatter(format_time)
+            el.setLevel(loglevel(errlog.get("level", "debug")))
+            self.logger.addHandler(el)
+
+        if filelog is not None:
+            try:
+                fl = logging.FileHandler(
+                    filename=path.join(
+                        path.dirname(__file__),
+                        filelog.get("file", "%s.log" % self.name)))
+                fl.setLevel(loglevel(filelog.get("level", "warning")))
+                fl.setFormatter(format_time)
+                self.logger.addHandler(fl)
+            except Exception as e:
+                Error("Unable to setup file logging", self.logger, exc=exc_info())
+
+        if syslog is not None:
+            try:
+                sl = logging.handlers.SysLogHandler(
+                    address=syslog.get("socket", "/dev/log"),
+                    facility=facility(syslog.get("facility", "local7")))
+                sl.setLevel(loglevel(syslog.get("level", "warning")))
+                sl.setFormatter(format_notime)
+                self.logger.addHandler(sl)
+            except Exception as e:
+                Error("Unable to setup syslog logging", self.logger, exc=exc_info())
+
+        if not (errlog or filelog or syslog):
+            # User wants explicitly no logging, so let him shoot his socks off.
+            # This silences complaining of logging module about no suitable
+            # handler.
+            self.logger.addHandler(logging.NullHandler())
+
+
+    def connect(self):
+
+        try:
+            if self.url.scheme=="https":
+                self.conn = HTTPSConnection(
+                    self.url.netloc,
+                    key_file = self.keyfile,
+                    cert_file = self.certfile,
+                    timeout = self.timeout,
+                    ciphers = self.ciphers,
+                    ca_certs = self.cafile,
+                    ssl_version = self.sslversion)
+            elif self.url.scheme=="http":
+                self.conn = httplib.HTTPConnection(
+                    self.url.netloc,
+                    timeout = self.timeout)
+            else:
+                return Error("Don't know how to connect to \"%s\"" % self.url.scheme, self.logger,
+                        detail={"url": self.url.geturl()})
+        except Exception:
+            return Error("HTTPS connection failed", self.logger, exc=exc_info(),
+                detail={
+                    "url": self.url.geturl(),
+                    "timeout": self.timeout,
+                    "key_file": self.keyfile,
+                    "cert_file": self.certfile,
+                    "cafile": self.cafile,
+                    "ciphers": self.ciphers,
+                    "ssl_version": self.sslversion})
+
+        return True
+
+
+    def sendRequest(self, func="", payload=None, **kwargs):
+
+        if self.secret is None:
+            kwargs["client"] = self.name
+        else:
+            kwargs["secret"] = self.secret
+
+        if kwargs:
+            for k in kwargs.keys():
+                if kwargs[k] is None:
+                    del kwargs[k]
+            argurl = "?" + urlencode(kwargs, doseq=True)
+        else:
+            argurl = ""
+
+        try:
+            if payload is None:
+                data = ""
+            else:
+                data = json.dumps(payload)
+        except:
+            return Error("Serialization to JSON failed", self.logger,
+                exc=exc_info(), method=func, detail=payload)
+
+        self.headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "Content-Length": str(len(data))
+        }
+
+        # We are connecting here at first use instead of in
+        # constructor, because constructor cannot return data/errors
+        # and we don't want to spit exceptions into user's face
+        # And maaaybee sometime we will implement reconnection on errors
+        if self.conn is None:
+            err = self.connect()
+            if not err:
+                return err  # either False of Error instance
+
+        loc = '%s/%s%s' % (self.url.path, func, argurl)
+        try:
+            self.conn.request("POST", loc, data, self.headers)
+        except:
+            return Error("Sending of request to server failed", self.logger,
+                exc=exc_info(), method=func, detail={
+                    "loc": loc,
+                    "headers": self.headers,
+                    "data": data})
+
+        try:
+            res = self.conn.getresponse()
+        except:
+            return Error("HTTP reply failed", self.logger, method=func, exc=exc_info())
+
+        try:
+            response_data = res.read()
+        except:
+            return Error("Fetching HTTP data from server failed", self.logger, method=func, exc=exc_info())
+
+        if res.status==httplib.OK:
+            try:
+                data = json.loads(response_data)
+            except:
+                data = Error("JSON message parsing failed", self.logger,
+                    exc=exc_info(), method=func, detail={"response": response_data})
+        else:
+            try:
+                data = json.loads(response_data)
+                data["error"]   # trigger exception if not dict or no error key
+            except:
+                data = Error("Generic server HTTP error", self.logger,
+                    method=func,
+                    error=res.status,
+                    exc=exc_info(),
+                    detail={"response": response_data})
+            else:
+                data = Error(data.get("message", None), self.logger,
+                    method=data.get("method", None),
+                    error=res.status,
+                    req_id=data.get("req_id", None),
+                    detail=data.get("detail", None))
+
+        return data
+
+
+    def _saveID(self, id, idstore=None):
+        idf = idstore or self.idstore
+        if not idf:
+            return False
+        try:
+            with open(idf, "w+") as f:
+                f.write(str(id))
+        except (ValueError, IOError) as e:
+            # Use Error instance just for proper logging
+            Error("Writing id file \"%s\" failed" % idf, self.logger,
+                prio="info", exc=exc_info(), detail={"idstore": idf})
+        return id
+
+
+    def _loadID(self, idstore=None):
+        idf = idstore or self.idstore
+        if not idf:
+            return None
+        try:
+            with open(idf, "r") as f:
+                id = int(f.read())
+        except (ValueError, IOError) as e:
+            Error("Reading id file \"%s\" failed, relying on server" % idf,
+                self.logger, prio="info", exc=exc_info(), detail={"idstore": idf})
+            id = None
+        return id
+
+
+    def getDebug(self):
+        return self.sendRequest("getDebug")
+
+
+    def getInfo(self):
+        return self.sendRequest("getInfo")
+
+
+    def sendEvents(self, events=[]):
+        res = self.sendRequest(
+            "sendEvents", payload=events)
+        return res
+
+
+    def getEvents(self, id=None, idstore=None, count=1,
+            cat=None, nocat=None,
+            tag=None, notag=None,
+            group=None, nogroup=None):
+
+        if not id:
+            id = self._loadID(idstore)
+
+        res = self.sendRequest(
+            "getEvents", id=id, count=count or self.recv_events_limit, cat=cat,
+            nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
+        if not res:
+            return res  # Should be Error instance
+
+        try:
+            events = res["events"]
+            newid = res["lastid"]
+        except KeyError:
+            return Error("Server returned bogus reply", self.logger,
+                method="getEvents", exc=exc_info(), detail={"response": res})
+
+        self._saveID(newid)
+
+        return events
+
+
+    def close(self):
+
+        if hasattr(self, "conn") and hasattr(self.conn, "close"):
+            self.conn.close()
+
+
+    __del__ = close
+
+
+
+def read_cfg(cfgfile):
+    abspath = path.join(path.dirname(__file__), cfgfile)
+    with open(abspath, "r") as f:
+        stripcomments = "\n".join((l for l in f if not l.lstrip().startswith("#")))
+        return json.loads(stripcomments)