README added

Minor update

warden3/contrib/connectors/hp-kippo/LICENSE

+BSD License
+
+Copyright © 2011-2015 Cesnet z.s.p.o
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the Cesnet z.s.p.o nor the names of its
+      contributors may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE Cesnet z.s.p.o BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

warden3/contrib/connectors/hp-kippo/README

++-------------------------------------------+
+| Warden Kippo connector 0.1 for Warden 3.X |
++-------------------------------------------+
+
+Content
+
+  A. Introduction
+  B. Dependencies
+  C. Usage
+  D. Configuration
+
+------------------------------------------------------------------------------
+A. Introduction
+
+   Warden Kippo connector (executable warden3-kippo-sender.py) is a one-shot 
+   script to send events from Kippo honeypot toward the Warden server.
+
+------------------------------------------------------------------------------
+B. Dependencies
+
+ 1. Platform
+
+    Python 2.7+
+
+ 2. Python packages
+
+    warden_client 3.0+
+
+------------------------------------------------------------------------------
+C. Usage
+
+   warden3-kippo-sender.py 
+
+   This script does not run as a daemon, for regularly run use job scheduler cron.
+
+------------------------------------------------------------------------------
+D. Configuration
+   
+   warden_client-kippo.cfg
+    name - sensor's source id used as a source of events, e.g. 'cz.cesnet.server.kippo'
+
+    dbhost - hostname/IP of MySQL DB server 
+    dbuser - username 
+    dbpass - password
+    dbname - database
+    dbport - db port
+    awin   - aggregation window, e.g. 5 for events in the last 5 minutes
+    
+   cron
+    SCRIPT_PATH=/opt/warden_client/
+    */5  *   * * *  root cd $SCRIPT_PATH; warden3-kippo-sender.py >> kippo-sender.log
+
+    Note: Repeat interval must be the same as value of 'awin'.
+
+------------------------------------------------------------------------------
+Copyright (C) 2011-2015 Cesnet z.s.p.o

warden3/contrib/connectors/hp-kippo/warden3-kippo-sender.py

@@ -7,13 +7,10 @@
 from warden_client import Client, Error, read_cfg
 import json
 import string
-from time import time, gmtime
+from time import time, gmtime, strftime
 from math import trunc
 from uuid import uuid4
-# from pprint import pprint
 from os import path
-# from random import randint, randrange, choice, random;
-# from base64 import b64encode;
 
 import MySQLdb as my
 import MySQLdb.cursors as mycursors
@@ -25,63 +22,35 @@ def get_precise_timestamp(epoch=None):
     iso = '%04d-%02d-%02dT%02d:%02d:%02d.%0dZ' % (g[0:6]+(us,))
     return iso
 
-def gen_event_idea(client_name, detect_time, win_start_time, win_end_time, conn_count, src_ip4, dst_ip4):
+def gen_event_idea(client_name, detect_time, win_start_time, win_end_time, conn_count, src_ip4, dst_ip4, aggr_win):
 
   event = {
      "Format": "IDEA0",
      "ID": str(uuid4()),
-     # "CreateTime": get_precise_timestamp(),
      "DetectTime": detect_time,
      "WinStartTime": win_start_time,
      "WinEndTime": win_end_time,
-     # "EventTime": get_precise_timestamp(),
-     # "CeaseTime": get_precise_timestamp(),
      "Category": ["Attempt.Login", "Test"],
-     # "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()],
-     # "Confidence": random(),
      "Note": "SSH login attempt",
      "ConnCount": conn_count,
-#       "ConnCount": choice([randint(0, 65535), "asdf"]),    # Send wrong event sometimes
      "Source": [
-        {
-           # "Type": ["Phishing"],
-           "IP4": [src_ip4],
-           # "IP6": [randip6() for i in range(randrange(1, 5))],
-           # "Hostname": ["example.com"],
-           # "Port": [src_ip4_port],
-           # "AttachHand": ["att1"],
-           # "Netname": ["arin:TEST-NET-1"]
+        { 
+          "IP4": [src_ip4],
         }
      ],
      "Target": [
         {
            "IP4": [dst_ip4],
-           # "IP6": [randip6() for i in range(randrange(1, 5))],
-           # "URL": ["http://example.com/%s" % randstr()],
            "Proto": ["tcp", "ssh"],
            "Port" : [22]
-           # "Netname": ["arin:TEST-NET-1"]
         }
      ],
-     # "Attach": [
-     #    {
-     #       "Handle": "att1",
-     #       "FileName": [randstr()],
-     #       "Type": ["Malware"],
-     #       "ContentType": "application/octet-stream",
-     #       "Hash": ["sha1:%s" % randstr(string.hexdigits, 24)],
-     #       "Size": 46,
-     #       "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())],
-     #       "ContentEncoding": "base64",
-     #       "Content": b64encode(randstr())
-     #    }
-     # ],
      "Node": [
         {
            "Name": client_name,
            "Tags": ["Connection","Honeypot","Recon"],
            "SW": ["Kippo"],
-           "AggrWin": "00:05:00"
+           "AggrWin": strftime("%H:%M:%S", gmtime(aggr_win))
         }
      ]
   }
@@ -89,42 +58,41 @@ def gen_event_idea(client_name, detect_time, win_start_time, win_end_time, conn_
   return event
 
 def main():
-    wclient = Client(**read_cfg("warden_client.cfg"))
-    appconf = read_cfg("warden_client-kippo.cfg")
+    wconfig = read_cfg("warden_client.cfg")
+    aconfig = read_cfg("warden_client-kippo.cfg")
+    wconfig['name'] = aconfig['name']
+
+    wclient = Client(**wconfig)   
 
-    con = my.connect( host=appconf['dbhost'], user=appconf['dbuser'], passwd=appconf['dbpass'],
-                      db=appconf['dbname'], port=appconf['dbport'], cursorclass=mycursors.DictCursor)
+    con = my.connect( host=aconfig['dbhost'], user=aconfig['dbuser'], passwd=aconfig['dbpass'],
+                      db=aconfig['dbname'], port=aconfig['dbport'], cursorclass=mycursors.DictCursor)
     
     crs = con.cursor()
 
     events = []
-    query = ["SELECT UNIX_TIMESTAMP(s.starttime) as starttime, s.ip, COUNT(s.id) as attack_scale \
+    query = ["SELECT UNIX_TIMESTAMP(s.starttime) as starttime, s.ip, COUNT(s.id) as attack_scale, sn.ip as sensor \
               FROM sessions s \
-              LEFT JOIN input i ON s.id = i.session \
+              LEFT JOIN sensors sn ON s.sensor=sn.id \
               WHERE s.starttime > DATE_SUB(UTC_TIMESTAMP(), INTERVAL + %s MINUTE) \
               GROUP BY s.ip ORDER BY s.starttime ASC;"]
 
-
-    # crs.execute("".join(query), [appconf['a_win']])
-    crs.execute("".join(query), [5])
+    crs.execute("".join(query), aconfig['awin'])
     rows = crs.fetchall()
     for row in rows:
       dtime = get_precise_timestamp(row['starttime'])
       etime = get_precise_timestamp(time())
-      stime = get_precise_timestamp(time() - appconf['a_win'] * 60)
-      events.append(gen_event_idea(client_name=appconf['name'], detect_time = dtime, win_start_time = stime, win_end_time = etime, conn_count = row['attack_scale'], src_ip4 = row['ip'], dst_ip4 = appconf['sensor_ip4']))
+      stime = get_precise_timestamp(time() - aconfig['awin'] * 60)
+      events.append(gen_event_idea(client_name = aconfig['name'], detect_time = dtime, win_start_time = stime, win_end_time = etime, conn_count = row['attack_scale'], src_ip4 = row['ip'], dst_ip4 = row['sensor'], aggr_win = aconfig['awin']))
         
     print "=== Sending ==="
     start = time()
     ret = wclient.sendEvents(events)
-    if not ret:
-        print "%d event(s) successfully delivered." % len(rows)
-    else:
-        print ret
+    
+    if ret:
+      wclient.logger.info("%d event(s) successfully delivered." % len(rows))
 
     print "Time: %f" % (time()-start)
 
 
-
 if __name__ == "__main__":
     main()

warden3/contrib/connectors/hp-kippo/warden_client-kippo.cfg

@@ -2,7 +2,6 @@
     "name": "cz.cesnet.server.kippo",
     "sensor_ip4": "195.113.x.x",
 
-    "idstore": "warden_client.id",
     "dbhost": "localhost",
     "dbuser": "kippo",
     "dbpass": "kippopass",

warden3/contrib/connectors/hp-kippo/warden_client.cfg

-{
-    "url": "https://warden-dev.cesnet.cz/warden3",
-
-    "certfile": "/etc/ssl/certs/server.cesnet.cz.pem",
-    "keyfile": "/etc/ssl/private/server.cesnet.cz.key",
-    "cafile": "/etc/ssl/certs/tcs-ca-bundle.pem",
-
-    "timeout": 60,
-    "recv_events_limit": 6000,
-
-    "errlog": {"level": "debug"},
-    "filelog": {"file": "warden_client.log", "level": "warning"},
-    #"syslog": {"socket": "/dev/log", "facility": "local7", "level": "warning"},
-
-    #"name": "xxx"
-    "secret": "xxxxxxxxxxxxxxxxxxxx"
-}
-

warden3/contrib/connectors/hp-kippo/warden_client.py

-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2011-2013 Cesnet z.s.p.o
-# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
-
-import json, httplib, ssl, socket, logging, logging.handlers
-from urlparse import urlparse
-from urllib import urlencode
-from sys import stderr, exc_info
-from pprint import pformat
-from traceback import format_tb
-from os import path
-
-
-class HTTPSConnection(httplib.HTTPSConnection):
-    '''
-    Overridden to allow peer certificate validation, configuration
-    of SSL/ TLS version and cipher selection.  See:
-    http://hg.python.org/cpython/file/c1c45755397b/Lib/httplib.py#l1144
-    and `ssl.wrap_socket()`
-    '''
-    def __init__(self, host, **kwargs):
-        self.ciphers = kwargs.pop('ciphers',None)
-        self.ca_certs = kwargs.pop('ca_certs',None)
-        self.ssl_version = kwargs.pop('ssl_version',ssl.PROTOCOL_SSLv23)
-
-        httplib.HTTPSConnection.__init__(self,host,**kwargs)
-
-    def connect(self):
-        sock = socket.create_connection( (self.host, self.port), self.timeout)
-
-        if self._tunnel_host:
-            self.sock = sock
-            self._tunnel()
-
-        self.sock = ssl.wrap_socket(
-            sock,
-            keyfile = self.key_file,
-            certfile = self.cert_file,
-            ca_certs = self.ca_certs,
-            cert_reqs = ssl.CERT_REQUIRED if self.ca_certs else ssl.CERT_NONE,
-            ssl_version = self.ssl_version)
-
-
-
-class Error(Exception):
-    """ Object for returning error messages to calling application.
-        Caller can test whether it received data or error by checking
-        isinstance(res, Error).
-        However if he does not want to deal with errors altogether,
-        this error object also returns False value if used in Bool
-        context (e.g. in "if res: print res" print is not evaluated),
-        and also acts as empty iterator (e.g. in "for e in res: print e"
-        print is also not evaluated).
-        Also, it can be raised as an exception.
-    """
-
-    def __init__(self, message, logger=None, error=None, prio="error", method=None,
-            req_id=None, detail=None, exc=None):
-
-        self.message = message
-        self.error = error
-        self.method = method
-        self.req_id = req_id
-        self.detail = detail
-        (self.exctype, self.excval, self.exctb) = exc or (None, None, None)
-        self.cause = self.excval # compatibility with other exceptions
-        if logger:
-            getattr(logger, prio, "error")(str(self))
-            info = self.info_str()
-            if info:
-                logger.info(info)
-            debug = self.debug_str()
-            if debug:
-                logger.debug(debug)
-
-
-    def __len__ (self):
-        """ In list or iterable context we're empty """
-        return 0
-
-
-    def __iter__(self):
-        """ We are the iterator """
-        return self
-
-
-    def next(self):
-        """ In list or iterable context we're empty """
-        raise StopIteration
-
-
-    def __bool__(self):
-        """ In boolean context we're never True """
-        return False
-
-
-    def __str__(self):
-        out = []
-        out.append("(%s)" % (self.error or "local"))
-        if self.method is not None:
-            out.append(" in %s" % self.method)
-        if self.req_id is not None:
-            out.append("(%8x)" % self.req_id)
-        if self.message is not None:
-            out.append(": %s" % self.message)
-        if self.excval is not None:
-            out.append(" - cause was %s: %s" % (type(self.excval).__name__, str(self.excval)))
-        return "".join(out)
-
-
-    def info_str(self):
-        return ("Detail: %s" % pformat(self.detail)) or ""
-
-
-    def debug_str(self):
-        out = []
-        if self.excval is not None:
-            out.append("Exception %s: %s\n" % (type(self.excval).__name__, str(self.excval)))
-        if self.exctb is not None:
-            out.append("Traceback:\n%s" % "".join(format_tb(self.exctb)))
-        return "".join(out)
-
-
-
-class Client(object):
-
-    def __init__(self,
-            url,
-            certfile=None,
-            keyfile=None,
-            cafile=None,
-            timeout=60,
-            recv_events_limit=6000,
-            errlog={"level": "debug"},
-            syslog=None,
-            filelog=None,
-            idstore=None,
-            name="warden_client",
-            secret=None):
-
-        self.name = name
-        self.secret = secret
-        # Init logging as soon as possible and make sure we don't
-        # spit out exceptions but just log or return Error objects
-        self.init_log(errlog, syslog, filelog)
-
-        self.url = urlparse(url, allow_fragments=False)
-
-        self.conn = None
-
-        base = path.join(path.dirname(__file__))
-        self.certfile = path.join(base, certfile or "cert.pem")
-        self.keyfile  = path.join(base, keyfile or "key.pem")
-        self.cafile = path.join(base, cafile or "ca.pem")
-        self.timeout = int(timeout)
-        self.recv_events_limit = int(recv_events_limit)
-        self.idstore = path.join(base, idstore) if idstore is not None else None
-
-        self.ciphers = 'TLS_RSA_WITH_AES_256_CBC_SHA'
-        self.sslversion = ssl.PROTOCOL_TLSv1
-
-
-    def init_log(self, errlog, syslog, filelog):
-
-        def loglevel(lev):
-            try:
-                return int(getattr(logging, lev.upper()))
-            except (AttributeError, ValueError):
-                self.logger.warning("Unknown loglevel \"%s\", using \"debug\"" % lev)
-                return logging.DEBUG
-
-        def facility(fac):
-            try:
-                return int(getattr(logging.handlers.SysLogHandler, "LOG_" + fac.upper()))
-            except (AttributeError, ValueError):
-                self.logger.warning("Unknown syslog facility \"%s\", using \"local7\"" % fac)
-                return logging.handlers.SysLogHandler.LOG_LOCAL7
-
-        form = "%(filename)s[%(process)d]: (%(levelname)s) %(name)s %(message)s"
-        format_notime = logging.Formatter(form)
-        format_time = logging.Formatter('%(asctime)s ' + form)
-
-        self.logger = logging.getLogger(self.name)
-        self.logger.propagate = False   # Don't bubble up to root logger
-        self.logger.setLevel(logging.DEBUG)
-
-        if errlog is not None:
-            el = logging.StreamHandler(stderr)
-            el.setFormatter(format_time)
-            el.setLevel(loglevel(errlog.get("level", "debug")))
-            self.logger.addHandler(el)
-
-        if filelog is not None:
-            try:
-                fl = logging.FileHandler(
-                    filename=path.join(
-                        path.dirname(__file__),
-                        filelog.get("file", "%s.log" % self.name)))
-                fl.setLevel(loglevel(filelog.get("level", "warning")))
-                fl.setFormatter(format_time)
-                self.logger.addHandler(fl)
-            except Exception as e:
-                Error("Unable to setup file logging", self.logger, exc=exc_info())
-
-        if syslog is not None:
-            try:
-                sl = logging.handlers.SysLogHandler(
-                    address=syslog.get("socket", "/dev/log"),
-                    facility=facility(syslog.get("facility", "local7")))
-                sl.setLevel(loglevel(syslog.get("level", "warning")))
-                sl.setFormatter(format_notime)
-                self.logger.addHandler(sl)
-            except Exception as e:
-                Error("Unable to setup syslog logging", self.logger, exc=exc_info())
-
-        if not (errlog or filelog or syslog):
-            # User wants explicitly no logging, so let him shoot his socks off.
-            # This silences complaining of logging module about no suitable
-            # handler.
-            self.logger.addHandler(logging.NullHandler())
-
-
-    def connect(self):
-
-        try:
-            if self.url.scheme=="https":
-                self.conn = HTTPSConnection(
-                    self.url.netloc,
-                    key_file = self.keyfile,
-                    cert_file = self.certfile,
-                    timeout = self.timeout,
-                    ciphers = self.ciphers,
-                    ca_certs = self.cafile,
-                    ssl_version = self.sslversion)
-            elif self.url.scheme=="http":
-                self.conn = httplib.HTTPConnection(
-                    self.url.netloc,
-                    timeout = self.timeout)
-            else:
-                return Error("Don't know how to connect to \"%s\"" % self.url.scheme, self.logger,
-                        detail={"url": self.url.geturl()})
-        except Exception:
-            return Error("HTTPS connection failed", self.logger, exc=exc_info(),
-                detail={
-                    "url": self.url.geturl(),
-                    "timeout": self.timeout,
-                    "key_file": self.keyfile,
-                    "cert_file": self.certfile,
-                    "cafile": self.cafile,
-                    "ciphers": self.ciphers,
-                    "ssl_version": self.sslversion})
-
-        return True
-
-
-    def sendRequest(self, func="", payload=None, **kwargs):
-
-        if self.secret is None:
-            kwargs["client"] = self.name
-        else:
-            kwargs["secret"] = self.secret
-
-        if kwargs:
-            for k in kwargs.keys():
-                if kwargs[k] is None:
-                    del kwargs[k]
-            argurl = "?" + urlencode(kwargs, doseq=True)
-        else:
-            argurl = ""
-
-        try:
-            if payload is None:
-                data = ""
-            else:
-                data = json.dumps(payload)
-        except:
-            return Error("Serialization to JSON failed", self.logger,
-                exc=exc_info(), method=func, detail=payload)
-
-        self.headers = {
-            "Content-Type": "application/json",
-            "Accept": "application/json",
-            "Content-Length": str(len(data))
-        }
-
-        # We are connecting here at first use instead of in
-        # constructor, because constructor cannot return data/errors
-        # and we don't want to spit exceptions into user's face
-        # And maaaybee sometime we will implement reconnection on errors
-        if self.conn is None:
-            err = self.connect()
-            if not err:
-                return err  # either False of Error instance
-
-        loc = '%s/%s%s' % (self.url.path, func, argurl)
-        try:
-            self.conn.request("POST", loc, data, self.headers)
-        except:
-            return Error("Sending of request to server failed", self.logger,
-                exc=exc_info(), method=func, detail={
-                    "loc": loc,
-                    "headers": self.headers,
-                    "data": data})
-
-        try:
-            res = self.conn.getresponse()
-        except:
-            return Error("HTTP reply failed", self.logger, method=func, exc=exc_info())
-
-        try:
-            response_data = res.read()
-        except:
-            return Error("Fetching HTTP data from server failed", self.logger, method=func, exc=exc_info())
-
-        if res.status==httplib.OK:
-            try:
-                data = json.loads(response_data)
-            except:
-                data = Error("JSON message parsing failed", self.logger,
-                    exc=exc_info(), method=func, detail={"response": response_data})
-        else:
-            try:
-                data = json.loads(response_data)
-                data["error"]   # trigger exception if not dict or no error key
-            except:
-                data = Error("Generic server HTTP error", self.logger,
-                    method=func,
-                    error=res.status,
-                    exc=exc_info(),
-                    detail={"response": response_data})
-            else:
-                data = Error(data.get("message", None), self.logger,
-                    method=data.get("method", None),
-                    error=res.status,
-                    req_id=data.get("req_id", None),
-                    detail=data.get("detail", None))
-
-        return data
-
-
-    def _saveID(self, id, idstore=None):
-        idf = idstore or self.idstore
-        if not idf:
-            return False
-        try:
-            with open(idf, "w+") as f:
-                f.write(str(id))
-        except (ValueError, IOError) as e:
-            # Use Error instance just for proper logging
-            Error("Writing id file \"%s\" failed" % idf, self.logger,
-                prio="info", exc=exc_info(), detail={"idstore": idf})
-        return id
-
-
-    def _loadID(self, idstore=None):
-        idf = idstore or self.idstore
-        if not idf:
-            return None
-        try:
-            with open(idf, "r") as f:
-                id = int(f.read())
-        except (ValueError, IOError) as e:
-            Error("Reading id file \"%s\" failed, relying on server" % idf,
-                self.logger, prio="info", exc=exc_info(), detail={"idstore": idf})
-            id = None
-        return id
-
-
-    def getDebug(self):
-        return self.sendRequest("getDebug")
-
-
-    def getInfo(self):
-        return self.sendRequest("getInfo")
-
-
-    def sendEvents(self, events=[]):
-        res = self.sendRequest(
-            "sendEvents", payload=events)
-        return res
-
-
-    def getEvents(self, id=None, idstore=None, count=1,
-            cat=None, nocat=None,
-            tag=None, notag=None,
-            group=None, nogroup=None):
-
-        if not id:
-            id = self._loadID(idstore)
-
-        res = self.sendRequest(
-            "getEvents", id=id, count=count or self.recv_events_limit, cat=cat,
-            nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
-        if not res:
-            return res  # Should be Error instance
-
-        try:
-            events = res["events"]
-            newid = res["lastid"]
-        except KeyError:
-            return Error("Server returned bogus reply", self.logger,
-                method="getEvents", exc=exc_info(), detail={"response": res})
-
-        self._saveID(newid)
-
-        return events
-
-
-    def close(self):
-
-        if hasattr(self, "conn") and hasattr(self.conn, "close"):
-            self.conn.close()
-
-
-    __del__ = close
-
-
-
-def read_cfg(cfgfile):
-    abspath = path.join(path.dirname(__file__), cfgfile)
-    with open(abspath, "r") as f:
-        stripcomments = "\n".join((l for l in f if not l.lstrip().startswith("#")))
-        return json.loads(stripcomments)