Warden-3 experimental code initial commit

warden3/warden_client/LICENSE

+BSD License
+
+Copyright © 2011-2013 Cesnet z.s.p.o
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the Cesnet z.s.p.o nor the names of its
+      contributors may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE Cesnet z.s.p.o BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

warden3/warden_client/warden_client.cfg

+{
+    "url": "https://warden.example.com/warden3",
+
+    "certfile": "cert.pem",
+    "keyfile": "key.pem",
+    "cafile": "ca.pem",
+
+    "timeout": 60,
+    "recv_events_limit": 6000,
+
+    "errlog": {"level": "debug"},
+    "filelog": {"file": "warden_client.log", "level": "warning"},
+    "syslog": {"socket": "/dev/log", "facility": "local7", "level": "warning"},
+
+    "idstore": "warden_client.id",
+
+    "name": "warden_client"
+}

warden3/warden_client/warden_client.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2013 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+import json, httplib, ssl, socket, logging, logging.handlers
+from urlparse import urlparse
+from urllib import urlencode
+from sys import stderr, exc_info
+from UserList import UserList
+from pprint import pformat
+from traceback import format_tb
+from os import path
+
+
+class HTTPSConnection(httplib.HTTPSConnection):
+    '''
+    Overridden to allow peer certificate validation, configuration
+    of SSL/ TLS version and cipher selection.  See:
+    http://hg.python.org/cpython/file/c1c45755397b/Lib/httplib.py#l1144
+    and `ssl.wrap_socket()`
+    '''
+    def __init__(self, host, **kwargs):
+        self.ciphers = kwargs.pop('ciphers',None)
+        self.ca_certs = kwargs.pop('ca_certs',None)
+        self.ssl_version = kwargs.pop('ssl_version',ssl.PROTOCOL_SSLv23)
+
+        httplib.HTTPSConnection.__init__(self,host,**kwargs)
+
+    def connect(self):
+        sock = socket.create_connection( (self.host, self.port), self.timeout)
+
+        if self._tunnel_host:
+            self.sock = sock
+            self._tunnel()
+
+        self.sock = ssl.wrap_socket(
+            sock,
+            keyfile = self.key_file,
+            certfile = self.cert_file,
+            ca_certs = self.ca_certs,
+            cert_reqs = ssl.CERT_REQUIRED if self.ca_certs else ssl.CERT_NONE,
+            ssl_version = self.ssl_version)
+
+
+
+class Error(Exception):
+    """ Object for returning error messages to calling application.
+        Caller can test whether it received data or error by checking
+        isinstance(res, Error).
+        However if he does not want to deal with errors altogether,
+        this error object also returns False value if used in Bool
+        context (e.g. in "if res: print res" print is not evaluated),
+        and also acts as empty iterator (e.g. in "for e in res: print e"
+        print is also not evaluated).
+        Also, it can be raised as an exception.
+    """
+
+    def __init__(self, message, logger=None, error=None, method=None,
+            detail=None, exc=None):
+
+        self.error = error
+        self.method = method
+        self.message = message
+        self.detail = detail
+        (self.exctype, self.excval, self.exctb) = exc or exc_info()
+        self.cause = self.excval # compatibility with other exceptions
+        if logger:
+            logger.error(str(self))
+            logger.info(self.info_str())
+            logger.debug(self.debug_str())
+
+
+    def __len__ (self):
+        """ In list or iterable context we're empty """
+        return 0
+
+
+    def __iter__(self):
+        """ We are the iterator """
+        return self
+
+
+    def next(self):
+        """ In list or iterable context we're empty """
+        raise StopIteration
+
+
+    def __bool__(self):
+        """ In boolean context we're never True """
+        return False
+
+
+    def __str__(self):
+        out = []
+        out.append("(%s)" % (self.error or "local"))
+        if self.method is not None:
+            out.append(" in \"%s\"" % self.method)
+        if self.message is not None:
+            out.append(": %s" % self.message)
+        if self.excval is not None:
+            out.append(" - cause was %s: %s" % (type(self.excval).__name__, str(self.excval)))
+        return "".join(out)
+
+
+    def info_str(self):
+        return ("Detail: %s" % pformat(self.detail)) or ""
+
+
+    def debug_str(self):
+        out = []
+        if self.excval is not None:
+            out.append("Exception %s: %s\n" % (type(self.excval).__name__, str(self.excval)))
+        if self.exctb is not None:
+            out.append("Traceback:\n%s" % "".join(format_tb(self.exctb)))
+        return "".join(out)
+
+
+
+class Client(object):
+
+    def __init__(self,
+            url,
+            certfile=None,
+            keyfile=None,
+            cafile=None,
+            timeout=60,
+            recv_events_limit=6000,
+            errlog={"level": "debug"},
+            syslog=None,
+            filelog=None,
+            idstore=None,
+            name="warden_client"):
+
+        self.name = name
+        # Init logging as soon as possible and make sure we don't
+        # spit out exceptions but just log or return Error objects
+        self.init_log(errlog, syslog, filelog)
+
+        self.url = urlparse(url, allow_fragments=False)
+
+        self.conn = None
+
+        base = path.join(path.dirname(__file__))
+        self.certfile = path.join(base, certfile or "cert.pem")
+        self.keyfile  = path.join(base, keyfile or "key.pem")
+        self.cafile = path.join(base, cafile or "ca.pem")
+        self.timeout = int(timeout)
+        self.recv_events_limit = int(recv_events_limit)
+        self.idstore = path.join(base, idstore) if idstore is not None else None
+
+        self.ciphers = 'TLS_RSA_WITH_AES_256_CBC_SHA'
+        self.sslversion = ssl.PROTOCOL_TLSv1
+
+
+    def init_log(self, errlog, syslog, filelog):
+
+        def loglevel(lev):
+            try:
+                return int(getattr(logging, lev.upper()))
+            except (AttributeError, ValueError):
+                self.logger.warning("Unknown loglevel \"%s\", using \"debug\"" % lev)
+                return logging.DEBUG
+
+        def facility(fac):
+            try:
+                return int(getattr(logging.handlers.SysLogHandler, "LOG_" + fac.upper()))
+            except (AttributeError, ValueError):
+                self.logger.warning("Unknown syslog facility \"%s\", using \"local7\"" % fac)
+                return logging.handlers.SysLogHandler.LOG_LOCAL7
+
+        form = "%(filename)s[%(process)d]: (%(levelname)s) %(name)s %(message)s"
+        format_notime = logging.Formatter(form)
+        format_time = logging.Formatter('%(asctime)s ' + form)
+
+        self.logger = logging.getLogger(self.name)
+        self.logger.propagate = False   # Don't bubble up to root logger
+        self.logger.setLevel(logging.DEBUG)
+
+        if errlog is not None:
+            el = logging.StreamHandler(stderr)
+            el.setFormatter(format_time)
+            el.setLevel(loglevel(errlog.get("level", "debug")))
+            self.logger.addHandler(el)
+
+        if filelog is not None:
+            try:
+                fl = logging.FileHandler(
+                    filename=path.join(
+                        path.dirname(__file__),
+                        filelog.get("file", "%s.log" % self.name)))
+                fl.setLevel(loglevel(filelog.get("level", "warning")))
+                fl.setFormatter(format_time)
+                self.logger.addHandler(fl)
+            except Exception as e:
+                Error("Unable to setup file logging", self.logger)
+
+        if syslog is not None:
+            try:
+                sl = logging.handlers.SysLogHandler(
+                    address=syslog.get("socket", "/dev/log"),
+                    facility=facility(syslog.get("facility", "local7")))
+                sl.setLevel(loglevel(syslog.get("level", "warning")))
+                sl.setFormatter(format_notime)
+                self.logger.addHandler(sl)
+            except Exception as e:
+                Error("Unable to setup syslog logging", self.logger)
+
+        if not (errlog or filelog or syslog):
+            # User wants explicitly no logging, so let him shoot its socks off.
+            # This silences complaining of logging module about no suitable
+            # handler.
+            self.logger.addHandler(logging.NullHandler())
+
+
+    def connect(self):
+
+        try:
+            if self.url.scheme=="https":
+                self.conn = HTTPSConnection(
+                    self.url.netloc,
+                    key_file = self.keyfile,
+                    cert_file = self.certfile,
+                    timeout = self.timeout,
+                    ciphers = self.ciphers,
+                    ca_certs = self.cafile,
+                    ssl_version = self.sslversion)
+            elif self.url.scheme=="http":
+                self.conn = httplib.HTTPConnection(
+                    self.url.netloc,
+                    timeout = self.timeout)
+            else:
+                return Error("Don't know how to connect to \"%s\"" % self.url.scheme, self.logger,
+                        detail={"url": self.url.geturl()})
+        except Exception:
+            return Error("HTTPS connection failed", self.logger,
+                detail={
+                    "url": self.url.geturl(),
+                    "timeout": self.timeout,
+                    "key_file": self.keyfile,
+                    "cert_file": self.certfile,
+                    "cafile": self.cafile,
+                    "ciphers": self.ciphers,
+                    "ssl_version": self.sslversion})
+
+        return True
+
+
+    def sendRequest(self, func="", payload=None, **kwargs):
+
+        if kwargs:
+            for k in kwargs.keys():
+                if kwargs[k] is None:
+                    del kwargs[k]
+            argurl = "?" + urlencode(kwargs)
+        else:
+            argurl = ""
+
+        try:
+            if payload is None:
+                data = ""
+            else:
+                data = json.dumps(payload)
+        except:
+            return Error("Serialization to JSON failed", self.logger,
+                method=func, detail=payload)
+
+        self.headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+            "Content-Length": str(len(data))
+        }
+
+        # We are connecting here at first use instead of in
+        # constructor, because constructor cannot return data/errors
+        # and we don't want to spit exceptions into user's face
+        # And maaaybee sometime we will implement reconnection on errors
+        if self.conn is None:
+            err = self.connect()
+            if not err:
+                return err  # either False of Error instance
+
+        loc = '%s/%s%s' % (self.url.path, func, argurl)
+        try:
+            self.conn.request("POST", loc, data, self.headers)
+        except:
+            return Error("Sending of request to server failed", self.logger,
+                method=func, detail={
+                    "loc": loc,
+                    "headers": self.headers,
+                    "data": data})
+
+        try:
+            res = self.conn.getresponse()
+        except:
+            return Error("HTTP reply failed", self.logger, method=func)
+
+        try:
+            response_data = res.read()
+        except:
+            return Error("Fetching HTTP data from server failed", self.logger, method=func)
+
+        if res.status==httplib.OK:
+            try:
+                data = json.loads(response_data)
+            except:
+                data = Error("JSON message parsing failed", self.logger,
+                    method=func, detail={"response": response_data})
+        else:
+            try:
+                data = json.loads(response_data)
+                data["error"]   # trigger exception if not dict or no error key
+            except:
+                data = Error("Generic server HTTP error", self.logger,
+                    method=func,
+                    error=res.status,
+                    detail={"response": response_data})
+            else:
+                data = Error(data.get("message", None), self.logger,
+                    method=data.get("method", None),
+                    error=res.status,
+                    detail=data.get("detail", None))
+
+        return data
+
+
+    def _saveID(self, id, idstore=None):
+        idf = idstore or self.idstore
+        if not idf:
+            return False
+        try:
+            with open(idf, "w+") as f:
+                f.write(str(id))
+        except (ValueError, IOError) as e:
+            # Use Error instance just for proper logging
+            Error("Writing id file \"%s\" failed" % idf, self.logger, detail={"idstore": idf})
+        return id
+
+
+    def _loadID(self, idstore=None):
+        idf = idstore or self.idstore
+        if not idf:
+            return None
+        try:
+            with open(idf, "r") as f:
+                id = int(f.read())
+        except (ValueError, IOError) as e:
+            Error("Reading id file \"%s\" failed, relying on server" % idf, self.logger, detail={"idstore": idf})
+            id = None
+        return id
+
+
+    def getDebug(self):
+        return self.sendRequest("getDebug")
+
+
+    def getInfo(self):
+        return self.sendRequest("getInfo")
+
+
+    def sendEvents(self, events=[]):
+        res = self.sendRequest(
+            "sendEvents", payload=events)
+        if not res:
+            return res  # Should be Error instance
+
+        return res.get("saved", 0)
+
+
+    def getEvents(self, id=None, idstore=None, count=1,
+            cat=None, nocat=None,
+            tag=None, notag=None,
+            group=None, nogroup=None):
+
+        if not id:
+            id = self._loadID(idstore)
+
+        res = self.sendRequest(
+            "getEvents", id=id, count=count or self.recv_events_limit, cat=cat,
+            nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
+        if not res:
+            return res  # Should be Error instance
+
+        try:
+            events = res["events"]
+            newid = res["lastid"]
+        except KeyError:
+            return Error("Server returned bogus reply", self.logger, method="getEvents", detail={"response": res})
+
+        self._saveID(newid)
+
+        return events
+
+
+    def close(self):
+
+        if hasattr(self, "conn") and hasattr(self.conn, "close"):
+            self.conn.close()
+
+
+    __del__ = close
+
+
+
+def read_cfg(cfgfile):
+    abspath = path.join(path.dirname(__file__), cfgfile)
+    with open(abspath, "r") as f:
+        stripcomments = "\n".join((l for l in f if not l.lstrip().startswith("#")))
+        return json.loads(stripcomments)

warden3/warden_client/warden_client_test.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2013 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+from warden_client import Client, Error, read_cfg
+import json
+from time import time, gmtime
+from math import trunc
+from uuid import uuid4
+from pprint import pprint
+from os import path
+
+def gen_random_idea():
+
+    def get_precise_timestamp():
+        t = time()
+        us = trunc((t-trunc(t))*1000000)
+        g = gmtime(t)
+        iso = '%04d-%02d-%02dT%02d:%02d:%02d.%0dZ' % (g[0:6]+(us,))
+        return iso
+
+    return {
+       "Format": "IDEA0",
+       "ID": str(uuid4()),
+       "DetectTime": get_precise_timestamp(),
+       "Category": ["Test"],
+    }
+
+wclient = Client(**read_cfg("warden_client.cfg"))
+# Also inline arguments are possible:
+# wclient = Client(
+#     url  = 'https://warden.example.com/warden3',
+#     keyfile  = '/opt/warden3/etc/key.pem',
+#     certfile = '/opt/warden3/etc/cert.pem',
+#     cafile = '/opt/warden3/etc/tcs-ca-bundle.pem',
+#     timeout=10,
+#     errlog={"level": "debug"},
+#     filelog={"level": "debug"},
+#     idstore="MyClient.id",
+#     name="MyClient")
+
+print "=== Getting 10 events ==="
+start = time()
+ret = wclient.getEvents(count=10)
+print "Time: %f" % (time()-start)
+for e in ret:
+    print e
+if ret:
+    print len(ret)
+
+print "=== Sending 500 events ==="
+start = time()
+ret = wclient.sendEvents([gen_random_idea() for i in range(500)])
+if ret:
+    print ret
+print "Time: %f" % (time()-start)
+
+print "=== Server info ==="
+info = wclient.getInfo()
+if not isinstance(info, Error):
+    pprint(info)

warden3/warden_client/warden_curl_test.sh

+#!/bin/sh
+#
+# Copyright (C) 2011-2013 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+keyfile='key.pem'
+certfile='cert.pem'
+cafile='tcs-ca-bundle.pem'
+url='https://warden.example.com/warden3'
+
+#    --fail \
+#    --show-error \
+#
+
+echo "Test  404"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    $url/blefub
+echo
+
+echo "Test  404"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    $url/
+echo
+
+echo "Test  Deserialization"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    --data '{#$%^' \
+    $url/getEvents
+echo
+
+echo "Test  Invalid data for getEvents - silently discarded"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    --data '[1]' \
+    $url/getEvents
+echo
+
+echo "Test  Called with internal args - just in log"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    $url/getEvents?self=test
+echo
+
+echo "Test  Called with superfluous args - just in log"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    $url/getEvents?bad=guy
+echo
+
+echo "Test  getEvents with no args - should be OK"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    $url/getEvents
+echo
+
+echo "Test  getEvents - should be OK"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    "$url/getEvents?count=3&id=10"
+echo
+
+echo "Test  getDebug"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    $url/getDebug
+echo
+
+echo "Test  getInfo"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    $url/getInfo
+echo
+
+#curl \
+#    --fail \
+#    --connect-timeout 3 \
+#    --request POST \
+#    $url/getEvents

warden3/warden_server/LICENSE

+BSD License
+
+Copyright © 2011-2014 Cesnet z.s.p.o
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the Cesnet z.s.p.o nor the names of its
+      contributors may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE Cesnet z.s.p.o BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

warden3/warden_server/apache.conf

+SSLEngine on
+
+SSLVerifyClient require
+SSLVerifyDepth 4
+SSLOptions +StdEnvVars +ExportCertData
+
+#SSLCipherSuite ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP:+eNULL
+
+SSLCertificateFile      /opt/warden_server_3/etc/cert.pem
+SSLCertificateKeyFile   /opt/warden_server_3/etc/key.pem
+SSLCACertificateFile    /opt/warden_server_3/etc/tcs-ca-bundle.pem
+
+WSGIScriptAlias /warden3 /opt/warden_server_3/warden_server.wsgi
+
+<Directory /opt/warden_server_3/warden_server.wsgi>
+    Order allow,deny
+    Allow from all
+</Directory>

warden3/warden_server/warden_server.cfg

+{
+    "Log": {
+        # If not specified, FileLogger is default
+        "level": "debug"
+    },
+    "Handler": {
+        "send_events_limit": 500,
+        "get_events_limit": 1000,
+        "description": "Warden 3 not even alpha development server"
+    }
+}

warden3/warden_server/warden_server.wsgi

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2013 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+from sys import path
+from os.path import dirname, join
+
+path.append(dirname(__file__))
+from warden_server import build_server
+
+## JSON configuration with line comments (trailing #)
+from warden_server import read_cfg
+application = build_server(read_cfg(join(dirname(__file__), "warden_server.cfg")))