warden_client/warden_client.cfg.dist

+{
+    "url": "https://warden-hub.example.org/warden3",
+    "certfile": "cert.pem",
+    "keyfile": "key.pem",
+    "filelog": {"level": "debug"},
+    "name": "org.example.warden_client",
+    "secret": "ToP_SeCrEt"
+}

warden_client/warden_client.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2015 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+import hashlib  # Some Python/ssl versions incorrectly initialize hashes, this helps
+from sys import stderr, exc_info, version_info
+import json, ssl, socket, logging, logging.handlers, time
+from traceback import format_tb
+from os import path
+from operator import itemgetter
+
+fix_logging_filename = str if version_info<(2, 7) else lambda x: x
+
+if version_info[0] >= 3:
+    import http.client as httplib
+    from urllib.parse import urlparse
+    from urllib.parse import urlencode
+    basestring = str
+else:
+    import httplib
+    from urlparse import urlparse
+    from urllib import urlencode
+
+
+
+VERSION = "3.0-beta3"
+
+DEFAULT_CA_STORES = [
+    "/etc/ssl/certs/ca-certificates.crt",       # Deb
+    "/etc/pki/tls/certs/ca-bundle.crt",         # RH
+    "/var/lib/ca-certificates/ca-bundle.pem"    # SuSE
+ ]
+
+class HTTPSConnection(httplib.HTTPSConnection):
+    '''
+    Overridden to allow peer certificate validation, configuration
+    of SSL/ TLS version and cipher selection.  See:
+    http://hg.python.org/cpython/file/c1c45755397b/Lib/httplib.py#l1144
+    and `ssl.wrap_socket()`
+    Used only if ssl.SSLContext is not available (Python version < 2.7.9)
+    '''
+    def __init__(self, host, **kwargs):
+        self.ciphers = kwargs.pop('ciphers', None)
+        self.ca_certs = kwargs.pop('ca_certs', None)
+        self.ssl_version = kwargs.pop('ssl_version', getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23))
+
+        httplib.HTTPSConnection.__init__(self,host,**kwargs)
+
+    def connect(self):
+        sock = socket.create_connection( (self.host, self.port), self.timeout)
+
+        if self._tunnel_host:
+            self.sock = sock
+            self._tunnel()
+
+        self.sock = ssl.wrap_socket(
+            sock,
+            keyfile = self.key_file,
+            certfile = self.cert_file,
+            ca_certs = self.ca_certs,
+            cert_reqs = ssl.CERT_REQUIRED if self.ca_certs else ssl.CERT_NONE,
+            ssl_version = self.ssl_version)
+
+
+
+class Error(Exception):
+    """ Object for returning error messages to calling application.
+        Caller can test whether it received data or error by checking
+        isinstance(res, Error).
+        However if he does not want to deal with errors altogether,
+        this error object also returns False value if used in Bool
+        context (e.g. in "if res: print res" print is not evaluated),
+        and also acts as empty iterator (e.g. in "for e in res: print e"
+        print is also not evaluated).
+        Also, it can be raised as an exception.
+    """
+
+    def __init__(self, method=None, req_id=None, errors=None, **kwargs):
+
+        self.errors = []
+        if errors:
+            self.extend(method, req_id, errors)
+        if kwargs:
+            self.append(method, req_id, **kwargs)
+
+
+    def append(self, method=None, req_id=None, **kwargs):
+        # We shift method and req_id into each and every error, because
+        # we want to be able to simply merge more Error arrays (for
+        # returning errors from more Warden calls at once
+        if method and not "method" in kwargs:
+            kwargs["method"] = method
+        if req_id and not "req_id" in kwargs:
+            kwargs["req_id"] = req_id
+        # Ugly, but be paranoid, don't rely on server reply to be well formed
+        try:
+            kwargs["error"] = int(kwargs["error"])
+        except Exception:
+            kwargs["error"] = 0
+        if "events" in kwargs:
+            evlist = kwargs["events"]
+            try:
+                evlist_new = []
+                for ev in evlist:
+                    try:
+                        evlist_new.append(int(ev))
+                    except Exception:
+                        pass
+                kwargs["events"] = evlist_new
+            except Exception:
+                kwargs["events"] = []
+        if "events_id" in kwargs:
+            try:
+                dummy = iter(kwargs["events_id"])
+            except TypeError:
+                kwargs["events_id"] = [None]*len(kwargs["events"])
+        if "send_events_limit" in kwargs:
+            try:
+                kwargs["send_events_limit"] = int(kwargs["send_events_limit"])
+            except Exception:
+                del kwargs["send_events_limit"]
+        if "exc" in kwargs:
+            # Traceback objects cause reference loops, so memory may be not
+            # correctly free'd. We only need traceback to log it in str_debug(),
+            # so let's get the string representation now and forget the 
+            # traceback object, thus preventing the loop.
+            exctype, excvalue, tb = kwargs["exc"]
+            tb = format_tb(tb)
+            kwargs["exc"] = exctype, excvalue, tb
+        self.errors.append(kwargs)
+
+
+    def extend(self, method=None, req_id=None, iterable=[]):
+        try:
+            dummy = iter(iterable)
+        except TypeError:
+            iterable = []       # Bad joke from server
+        for e in iterable:
+            try:
+                args = dict(e)
+            except TypeError:
+                args = {}       # Not funny!
+            self.append(method, req_id, **args)
+
+
+    def __len__ (self):
+        """ In list or iterable context we're empty """
+        return 0
+
+
+    def __iter__(self):
+        """ We are the iterator """
+        return self
+
+
+    def next(self):
+        """ In list or iterable context we're empty """
+        raise StopIteration
+
+    __next__ = next
+
+
+    def __bool__(self):
+        """ In boolean context we're never True """
+        return False
+
+
+    def __str__(self):
+        out = []
+        for e in self.errors:
+            out.append(self.str_err(e))
+            out.append(self.str_info(e))
+        return "\n".join(out)
+
+
+    def log(self, logger=None, prio=logging.ERROR):
+        if not logger:
+            logger = logging.getLogger()
+        for e in self.errors:
+            logger.log(prio, self.str_err(e))
+            info = self.str_info(e)
+            if info:
+                logger.info(info)
+            debug = self.str_debug(e)
+            if debug:
+                logger.debug(debug)
+
+
+    def str_preamble(self, e):
+        return "%08x/%s" % (e.get("req_id", 0), e.get("method", "?"))
+
+
+    def str_err(self, e):
+        out = []
+        out.append(self.str_preamble(e))
+        out.append(" Error(%s) %s " % (e.get("error", 0), e.get("message", "Unknown error")))
+        if "exc" in e and e["exc"]:
+            out.append("(cause was %s: %s)" % (e["exc"][0].__name__, str(e["exc"][1])))
+        return "".join(out)
+
+
+    def str_info(self, e):
+        ecopy = dict(e)    # shallow copy
+        ecopy.pop("req_id", None)
+        ecopy.pop("method", None)
+        ecopy.pop("error", None)
+        ecopy.pop("message", None)
+        ecopy.pop("exc", None)
+        if ecopy:
+            out = "%s Detail: %s" % (self.str_preamble(e), json.dumps(ecopy, default=lambda v: str(v)))
+        else:
+            out = ""
+        return out
+
+
+    def str_debug(self, e):
+        out = []
+        out.append(self.str_preamble(e))
+        if not "exc" in e or not e["exc"]:
+            return ""
+        exc_tb = e["exc"][2] # exc_tb is string repr. of traceback object
+        if exc_tb:
+            out.append("Traceback:\n")
+            out.extend(exc_tb)
+        return "".join(out)
+
+
+
+class Client(object):
+
+    def __init__(self,
+            url,
+            certfile=None,
+            keyfile=None,
+            cafile=None,
+            timeout=60,
+            retry=20,
+            pause=10,
+            get_events_limit=6000,
+            send_events_limit=500,
+            errlog={},
+            syslog=None,
+            filelog=None,
+            idstore=None,
+            name="org.example.warden.test",
+            secret=None):
+
+        self.name = name
+        self.secret = secret
+        # Init logging as soon as possible and make sure we don't
+        # spit out exceptions but just log or return Error objects
+        self.init_log(errlog, syslog, filelog)
+
+        self.url = urlparse(url, allow_fragments=False)
+
+        self.conn = None
+
+        base = path.join(path.dirname(__file__))
+        self.certfile = self.get_readable_file(certfile or "cert.pem", base)
+        self.keyfile  = self.get_readable_file(keyfile or "key.pem", base)
+        self.cafile = self.get_readable_file(cafile if cafile is not None else DEFAULT_CA_STORES, base)
+        self.timeout = int(timeout)
+        self.get_events_limit = int(get_events_limit)
+        self.idstore = path.join(base, idstore) if idstore is not None else None
+
+        self.send_events_limit = int(send_events_limit)
+        self.retry = int(retry)
+        self.pause = int(pause)
+
+        self.ciphers = None
+        self.sslversion = getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23)
+
+        # If Python is new enough to have SSLContext, use it for SSL settings,
+        # otherwise our own class derived from httplib.HTTPSConnection is used
+        # later in connect().
+        if hasattr(ssl, 'SSLContext'):
+            self.sslcontext = ssl.SSLContext(self.sslversion)
+            self.sslcontext.load_cert_chain(self.certfile, self.keyfile)
+            if self.cafile:
+                self.sslcontext.load_verify_locations(self.cafile)
+                self.sslcontext.verify_mode = ssl.CERT_REQUIRED
+            else:
+                self.sslcontext.verify_mode = ssl.CERT_NONE
+        else:
+            self.sslcontext = None
+
+        self.getInfo()  # Call to align limits with server opinion
+
+
+    def get_readable_file(self, name, base):
+        names = [name] if isinstance(name, basestring) else name
+        names = [path.join(base, n) for n in names]
+        errors = []
+        for n in names:
+            try:
+                open(n, "r").close()
+                self.logger.debug("Using %s" % n)
+                return n
+            except IOError as e:
+                errors.append(e)
+        for e in errors:
+            self.logger.error(str(e))
+        return names[0] if names else None
+
+
+    def init_log(self, errlog, syslog, filelog):
+
+        def loglevel(lev):
+            try:
+                return int(getattr(logging, lev.upper()))
+            except (AttributeError, ValueError):
+                self.logger.warning("Unknown loglevel \"%s\", using \"debug\"" % lev)
+                return logging.DEBUG
+
+        def facility(fac):
+            try:
+                return int(getattr(logging.handlers.SysLogHandler, "LOG_" + fac.upper()))
+            except (AttributeError, ValueError):
+                self.logger.warning("Unknown syslog facility \"%s\", using \"local7\"" % fac)
+                return logging.handlers.SysLogHandler.LOG_LOCAL7
+
+        form = "%(filename)s[%(process)d]: %(name)s (%(levelname)s) %(message)s"
+        format_notime = logging.Formatter(form)
+        format_time = logging.Formatter('%(asctime)s ' + form)
+
+        self.logger = logging.getLogger(self.name)
+        self.logger.propagate = False   # Don't bubble up to root logger
+        self.logger.setLevel(logging.DEBUG)
+
+        if errlog is not None:
+            el = logging.StreamHandler(stderr)
+            el.setFormatter(format_time)
+            el.setLevel(loglevel(errlog.get("level", "info")))
+            self.logger.addHandler(el)
+
+        if filelog is not None:
+            try:
+                fl = logging.FileHandler(
+                    filename=path.join(
+                        path.dirname(__file__),
+                        filelog.get("file", "%s.log" % self.name)),
+                        encoding="utf-8")
+                fl.setLevel(loglevel(filelog.get("level", "debug")))
+                fl.setFormatter(format_time)
+                self.logger.addHandler(fl)
+            except Exception as e:
+                Error(message="Unable to setup file logging", exc=exc_info()).log(self.logger)
+
+        if syslog is not None:
+            try:
+                sl = logging.handlers.SysLogHandler(
+                    address=fix_logging_filename(syslog.get("socket", "/dev/log")),
+                    facility=facility(syslog.get("facility", "local7")))
+                sl.setLevel(loglevel(syslog.get("level", "debug")))
+                sl.setFormatter(format_notime)
+                self.logger.addHandler(sl)
+            except Exception as e:
+                Error(message="Unable to setup syslog logging", exc=exc_info()).log(self.logger)
+
+        if not (errlog or filelog or syslog):
+            # User wants explicitly no logging, so let him shoot his socks off.
+            # This silences complaining of logging module about no suitable
+            # handler.
+            self.logger.addHandler(logging.NullHandler())
+
+
+    def log_err(self, err, prio=logging.ERROR):
+        if isinstance(err, Error):
+            err.log(self.logger, prio)
+        return err
+
+
+    def connect(self):
+
+        try:
+            if self.url.scheme=="https":
+                if self.sslcontext:
+                    conn = httplib.HTTPSConnection(
+                        self.url.netloc,
+                        timeout = self.timeout,
+                        context = self.sslcontext)
+                else:
+                    conn = HTTPSConnection(
+                        self.url.netloc,
+                        key_file = self.keyfile,
+                        cert_file = self.certfile,
+                        timeout = self.timeout,
+                        ciphers = self.ciphers,
+                        ca_certs = self.cafile,
+                        ssl_version = self.sslversion)
+            elif self.url.scheme=="http":
+                conn = httplib.HTTPConnection(
+                    self.url.netloc,
+                    timeout = self.timeout)
+            else:
+                return Error(message="Don't know how to connect to \"%s\"" % self.url.scheme,
+                        url=self.url.geturl())
+        except Exception:
+            return Error(message="HTTP(S) connection failed", exc=exc_info(),
+                    url=self.url.geturl(),
+                    timeout=self.timeout,
+                    key_file=self.keyfile,
+                    cert_file=self.certfile,
+                    cafile=self.cafile,
+                    ciphers=self.ciphers,
+                    ssl_version=self.sslversion)
+
+        return conn
+
+
+    def sendRequest(self, func="", payload=None, **kwargs):
+
+        if self.secret is None:
+            kwargs["client"] = self.name
+        else:
+            kwargs["secret"] = self.secret
+
+        if kwargs:
+            for k in list(kwargs.keys()):
+                if kwargs[k] is None:
+                    del kwargs[k]
+            argurl = "?" + urlencode(kwargs, doseq=True)
+        else:
+            argurl = ""
+
+        self.headers = {"Accept": "application/json"}
+        data = None
+
+        if payload is None:
+            method = "GET"
+        else:
+            method = "POST"
+            try:
+                data = json.dumps(payload)
+            except:
+                return Error(message="Serialization to JSON failed",
+                    exc=exc_info(), method=func, payload=payload)
+
+            self.headers.update({
+                "Content-Type": "application/json",
+                "Content-Length": str(len(data))
+            })
+
+        # HTTP(S)Connection is oneshot object (and we don't speak "pipelining")
+        conn = self.connect()
+        if not conn:
+            return conn  # either False of Error instance
+
+        loc = '%s/%s%s' % (self.url.path, func, argurl)
+        try:
+            conn.request(method, loc, data, self.headers)
+        except:
+            conn.close()
+            return Error(message="Sending of request to server failed",
+                exc=exc_info(), method=func, log=loc, headers=self.headers, data=data)
+
+        try:
+            res = conn.getresponse()
+        except:
+            conn.close()
+            return Error(method=func, message="HTTP reply failed",
+                exc=exc_info(), loc=loc, headers=self.headers, data=data)
+
+        try:
+            response_data = res.read()
+        except:
+            conn.close()
+            return Error(method=func, message="Fetching HTTP data from server failed",
+                exc=exc_info(), loc=loc, headers=self.headers, data=data)
+
+        conn.close()
+
+        if res.status==httplib.OK:
+            try:
+                data = json.loads(response_data.decode("utf-8"))
+            except:
+                data = Error(method=func, message="JSON message parsing failed",
+                    exc=exc_info(), response=response_data)
+        else:
+            try:
+                data = json.loads(response_data.decode("utf-8"))
+                data["errors"]   # trigger exception if not dict or no error key
+            except:
+                data = Error(method=func, message="Generic server HTTP error",
+                    error=res.status, exc=exc_info(), response=response_data)
+            else:
+                data = Error(
+                    method=data.get("method", None),
+                    req_id=data.get("req_id", None),
+                    errors=data.get("errors", []))
+
+        return data
+
+
+    def _saveID(self, id, idstore=None):
+        idf = idstore or self.idstore
+        if not idf:
+            return False
+        try:
+            with open(idf, "w+") as f:
+                f.write(str(id))
+        except (ValueError, IOError) as e:
+            # Use Error instance just for proper logging
+            Error(message="Writing id file \"%s\" failed" % idf, exc=exc_info(),
+                  idstore=idf).log(self.logger, logging.INFO)
+        return id
+
+
+    def _loadID(self, idstore=None):
+        idf = idstore or self.idstore
+        if not idf:
+            return None
+        try:
+            with open(idf, "r") as f:
+                id = int(f.read())
+        except (ValueError, IOError) as e:
+            Error(message="Reading id file \"%s\" failed, relying on server" % idf,
+                  exc=exc_info(), idstore=idf).log(self.logger, logging.INFO)
+            id = None
+        return id
+
+
+    def getDebug(self):
+        return self.log_err(self.sendRequest("getDebug"))
+
+
+    def getInfo(self):
+        res = self.sendRequest("getInfo")
+        if isinstance(res, Error):
+            res.log(self.logger)
+        else:
+            try:
+                self.send_events_limit = min(res["send_events_limit"], self.send_events_limit)
+                self.get_events_limit = min(res["get_events_limit"], self.get_events_limit)
+            except (AttributeError, TypeError, KeyError):
+                pass
+        return res
+
+
+    def send_events_raw(self, events=[]):
+        return self.sendRequest("sendEvents", payload=events)
+
+
+    def send_events_chunked(self, events=[]):
+        """ Split potentially long "events" list to send_events_limit
+            long chunks to avoid slap from server.
+        """
+        count = len(events)
+        err = Error()
+        send_events_limit = self.send_events_limit  # object stored value can change during sending
+        for offset in range(0, count, send_events_limit):
+            res = self.send_events_raw(events[offset:min(offset+send_events_limit, count)])
+
+            if isinstance(res, Error):
+                # Shift all error indices by offset to correspond with 'events' list
+                for e in res.errors:
+                    evlist = e.get("events", [])
+                    # Update sending limit advice, if present in error
+                    srv_limit = e.get("send_events_limit")
+                    if srv_limit:
+                        self.send_events_limit = min(self.send_events_limit, srv_limit)
+                    for i in range(len(evlist)):
+                        evlist[i] += offset
+                err.errors.extend(res.errors)
+
+        return err if err.errors else {}
+
+
+    def sendEvents(self, events=[], retry=None, pause=None):
+        """ Send out "events" list to server, retrying on server errors.
+        """
+        ev = events
+        idx_xlat = list(range(len(ev)))
+        err = Error()
+        retry = retry or self.retry
+        attempt = retry
+        while ev and attempt:
+            if attempt<retry:
+                self.logger.info("%d transient errors, retrying (%d to go)" % (len(ev), attempt))
+                time.sleep(pause or self.pause)
+            res = self.send_events_chunked(ev)
+            attempt -= 1
+
+            next_ev = []
+            next_idx_xlat = []
+            if isinstance(res, Error):
+                # Sort to process fatal errors first
+                res.errors.sort(key=itemgetter("error"))
+                for e in res.errors:
+                    errno = e["error"]
+                    evlist = e.get("events", list(range(len(ev))))   # none means all
+                    if errno < 500 or not attempt:
+                        # Fatal error or last try, translate indices
+                        # to original and prepare for returning to caller
+                        for i in range(len(evlist)):
+                            evlist[i] = idx_xlat[evlist[i]]
+                        err.errors.append(e)
+                    else:
+                        # Maybe transient error, prepare to try again
+                        for evlist_i in evlist:
+                            next_ev.append(ev[evlist_i])
+                            next_idx_xlat.append(idx_xlat[evlist_i])
+            ev = next_ev
+            idx_xlat = next_idx_xlat
+
+        return self.log_err(err) if err.errors else {"saved": len(events)}
+
+
+    def getEvents(self, id=None, idstore=None, count=None,
+            cat=None, nocat=None,
+            tag=None, notag=None,
+            group=None, nogroup=None):
+
+        if id is None:
+            id = self._loadID(idstore)
+
+        res = self.sendRequest(
+            "getEvents", id=id, count=self.get_events_limit if count is None else count, cat=cat,
+            nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
+
+        if res:
+            try:
+                events = res["events"]
+                newid = res["lastid"]
+            except KeyError:
+                events = Error(method="getEvents", message="Server returned bogus reply",
+                    exc=exc_info(), response=res)
+            self._saveID(newid)
+        else:
+            events = res
+
+        return self.log_err(events)
+
+
+    def close(self):
+
+        if hasattr(self, "conn") and hasattr(self.conn, "close"):
+            self.conn.close()
+
+
+    __del__ = close
+
+
+
+def format_timestamp(epoch=None, utc=True, utcoffset=None):
+    if utcoffset is None:
+        utcoffset = -(time.altzone if time.daylight else time.timezone)
+    if epoch is None:
+        epoch = time.time()
+    if utc:
+        epoch += utcoffset
+    us = int(epoch % 1 * 1000000 + 0.5)
+    return format_time(*time.gmtime(epoch)[:6], microsec=us, utcoffset=utcoffset)
+
+
+def format_time(year, month, day, hour, minute, second, microsec=0, utcoffset=None):
+    if utcoffset is None:
+        utcoffset = -(time.altzone if time.daylight else time.timezone)
+    tstr = "%04d-%02d-%02dT%02d:%02d:%02d" % (year, month, day, hour, minute, second)
+    usstr = "." + str(microsec).rstrip("0") if microsec else ""
+    offsstr = ("%+03d:%02d" % divmod((utcoffset+30)//60, 60)) if utcoffset else "Z"
+    return tstr + usstr + offsstr
+
+
+def read_cfg(cfgfile):
+    with open(cfgfile, "r") as f:
+        stripcomments = "\n".join((l for l in f if not l.lstrip().startswith(("#", "//"))))
+        return json.loads(stripcomments)

warden_client/warden_client_examples.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2015 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+import string
+from time import time
+from uuid import uuid4
+from pprint import pprint
+from random import randint, randrange, choice, random
+from base64 import b64encode
+from warden_client import Client, Error, read_cfg, format_timestamp
+
+def gen_min_idea():
+
+    return {
+       "Format": "IDEA0",
+       "ID": str(uuid4()),
+       "DetectTime": format_timestamp(),
+       "Category": ["Test"],
+    }
+
+def gen_random_idea(client_name="cz.example.warden.test"):
+
+    def geniprange(gen):
+
+        def iprange():
+            u = v = 0
+            while u==v:
+                u, v = gen(), gen()
+            u, v = min(u, v), max(u, v)
+            return "%s-%s" % (u, v)
+
+        return iprange
+
+    def rand4ip():
+        return "%s%d" % ('192.0.2.', randint(1, 254))
+
+    def rand4cidr():
+        return "%s%d/%d" % ('192.0.2.', randint(1, 254), randint(24, 31))
+
+    def randip4():
+        return [rand4ip, geniprange(rand4ip), rand4cidr][randint(0, 2)]()
+
+    def rand6ip():
+        return "2001:DB8:%s" % ":".join("%x" % randint(0, 65535) for i in range(6))
+
+    def rand6cidr():
+        m = randint(0, 5)
+        return "2001:DB8%s%s::/%d" % (":" if m else "", ":".join("%x" % randint(0, 65535) for i in range(m)), (m+2)*16)
+
+    def randip6():
+        return [rand6ip, geniprange(rand6ip), rand6cidr][randint(0, 2)]()
+
+    def randstr(charlist=string.ascii_letters, maxlen=32, minlen=1):
+        return ''.join(choice(charlist) for i in range(randint(minlen, maxlen)))
+
+    event = {
+       "Format": "IDEA0",
+       "ID": str(uuid4()),
+       "CreateTime": format_timestamp(),
+       "DetectTime": format_timestamp(),
+       "WinStartTime": format_timestamp(),
+       "WinEndTime": format_timestamp(),
+       "EventTime": format_timestamp(),
+       "CeaseTime": format_timestamp(),
+       #"Category": ["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"],
+       # "Category": ["Abusive.Spam","Fraud.Copyright"],
+       "Category": [choice(["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"]) for dummy in range(randint(1, 3))],
+       "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()],
+       "Confidence": random(),
+       "Note": "Random event",
+       "ConnCount": randint(0, 65535),
+#       "ConnCount": choice([randint(0, 65535), "asdf"]),    # Send wrong event sometimes
+       "Source": [
+          {
+             "Type": ["Phishing"],
+             "IP4": [randip4() for i in range(randrange(1, 5))],
+             "IP6": [randip6() for i in range(randrange(1, 5))],
+             "Hostname": ["example.com"],
+             "Port": [randint(1, 65535) for i in range(randrange(1, 3))],
+             "AttachHand": ["att1"],
+             "Netname": ["arin:TEST-NET-1"]
+          }
+       ],
+       "Target": [
+          {
+             "IP4": [randip4() for i in range(randrange(1, 5))],
+             "IP6": [randip6() for i in range(randrange(1, 5))],
+             "URL": ["http://example.com/%s" % randstr()],
+             "Proto": ["tcp", "http"],
+             "Netname": ["arin:TEST-NET-1"]
+          }
+       ],
+       "Attach": [
+          {
+             "Handle": "att1",
+             "FileName": [randstr()],
+             "Type": ["Malware"],
+             "ContentType": "application/octet-stream",
+             "Hash": ["sha1:%s" % randstr(string.hexdigits, 24)],
+             "Size": 46,
+             "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())],
+             "ContentEncoding": "base64",
+             "Content": b64encode(randstr().encode('ascii')).decode("ascii")
+          }
+       ],
+       "Node": [
+          {
+             "Name": client_name,
+             "Type": [choice(["Data", "Protocol", "Honeypot", "Heuristic", "Log"]) for dummy in range(randint(1, 3))],
+             "SW": ["Kippo"],
+             "AggrWin": "00:05:00"
+          },
+          {
+             "Name": "org.example.warden.client",
+             "Type": [choice(["Connection", "Datagram"]) for dummy in range(randint(1, 2))],
+          }
+       ]
+    }
+
+    return event
+
+def main():
+    wclient = Client(**read_cfg("warden_client.cfg"))
+    # Also inline arguments are possible:
+    # wclient = Client(
+    #     url  = 'https://warden.example.com/warden3',
+    #     keyfile  = '/opt/warden3/etc/key.pem',
+    #     certfile = '/opt/warden3/etc/cert.pem',
+    #     cafile = '/opt/warden3/etc/tcs-ca-bundle.pem',
+    #     timeout=10,
+    #     errlog={"level": "debug"},
+    #     filelog={"level": "debug"},
+    #     idstore="MyClient.id",
+    #     name="cz.example.warden.test")
+
+    print("=== Debug ===")
+    info = wclient.getDebug()
+    pprint(info)
+
+    # All methods return something.
+    # If you want to catch possible errors (for example implement some
+    # form of persistent retry, or save failed events for later, you may
+    # check for Error instance and act based on contained info.
+    # If you want just to be informed, this is not necessary, just
+    # configure logging correctly and check logs.
+    if isinstance(info, Error):
+        print(info)
+
+    print("=== Server info ===")
+    info = wclient.getInfo()
+
+    print("=== Sending 10 event(s) ===")
+    start = time()
+    ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)])
+    print(ret)
+    print("Time: %f" % (time()-start))
+
+    print("=== Getting 10 events ===")
+    start = time()
+
+    # cat = ['Availability', 'Abusive.Spam','Attempt.Login']
+    # cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus']
+    # cat = ['Fraud', 'Abusive.Spam']
+    # nocat = ['Availability', 'Information', 'Fraud.Scam']
+    cat = []
+    nocat = []
+
+    #tag = ['Log', 'Data']
+    #notag = ['Flow', 'Datagram']
+    tag = []
+    notag = []
+
+    #group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
+    #nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
+    group = []
+    nogroup = []
+
+    ret = wclient.getEvents(count=10, cat=cat, nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
+    print("Time: %f" % (time()-start))
+    print("Got %i events" % len(ret))
+    for e in ret:
+        print(e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name"))
+
+if __name__ == "__main__":
+    main()

warden_client/warden_curl_test.sh

+#!/bin/sh
+#
+# Copyright (C) 2011-2015 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+if [ "$#" -ne 6 ]; then
+    echo "Run me like:"
+    echo "${0##*/} 'https://warden-hub.example.org/warden3' org.example.warden.client 'ToPsEcReT' key.pem cert.pem tcs-ca-bundle.pem"
+    exit 1
+fi
+
+
+url="$1"
+client="$2"
+secret="$3"
+keyfile="$4"
+certfile="$5"
+cafile="$6"
+
+echo "Test  404"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/blefub?client=$client&secret=$secret"
+echo
+
+echo "Test  404"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/?client=$client&secret=$secret"
+echo
+
+echo "Test  403 - no secret"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=$client"
+echo
+
+echo "Test  403 - no client, no secret"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents"
+echo
+
+echo "Test  403 - wrong client"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=asdf.blefub"
+echo
+
+echo "Test  403 - wrong client, right secret"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=asdf.blefub&secret=$secret"
+echo
+
+echo "Test  403 - right client, wrong secret"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=$client&secret=ASDFblefub"
+echo
+
+echo "Test - no client, but secret, should be ok"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?secret=$secret"
+echo
+
+echo "Test  Deserialization"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    --data '{#$%^' \
+    "$url/getEvents?client=$client&secret=$secret"
+echo
+
+echo "Test  Called with unknown category"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=$client&secret=$secret&cat=bflm"
+echo
+
+echo "Test  Called with both cat and nocat"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=$client&secret=$secret&cat=Other&nocat=Test"
+echo
+
+echo "Test  Invalid data for getEvents - silently discarded"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    --request POST \
+    --data '[1]' \
+    "$url/getEvents?client=$client&secret=$secret"
+echo
+
+echo "Test  Called with internal args - just in log"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=$client&secret=$secret&self=test"
+echo
+
+echo "Test  Called with superfluous args - just in log"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=$client&secret=$secret&bad=guy"
+echo
+
+echo "Test  getEvents with no args - should be OK"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=$client&secret=$secret"
+echo
+
+echo "Test  getEvents - should be OK"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getEvents?client=$client&secret=$secret&count=3&id=10"
+echo
+
+echo "Test  getDebug"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getDebug?client=$client&secret=$secret"
+echo
+
+echo "Test  getInfo"
+curl \
+    --key $keyfile \
+    --cert $certfile \
+    --cacert $cafile \
+    --connect-timeout 3 \
+    "$url/getInfo?client=$client&secret=$secret"
+echo

warden_filer/LICENSE

+BSD License
+
+Copyright © 2011-2015 Cesnet z.s.p.o
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the Cesnet z.s.p.o nor the names of its
+      contributors may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE Cesnet z.s.p.o BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

warden_filer/README

++---------------------------------------+
+| Warden Filer 3.0-beta3 for Warden 3.X |
++---------------------------------------+
+
+Content
+
+  A. Introduction
+  B. Dependencies
+  C. Usage
+  D. Configuration
+  E. Directories and locking issues
+
+------------------------------------------------------------------------------
+A. Introduction
+
+   Warden Filer (executable warden_filer.py) is daemon for easy handling of
+Idea events transfer between plain local files and Warden server. The tool can
+be instructed to run as one of two daemons - reader and sender.
+   In reader mode, Filer polls Warden server and saves incoming events as
+plain files in directory.
+   In writer mode, Filer polls directory and sends out all new files out to
+Warden server.
+
+------------------------------------------------------------------------------
+B. Dependencies
+
+ 1. Platform
+
+    Python 2.7+
+
+ 2. Python packages
+
+    python-daemon 1.5+, warden_client 3.0+
+
+------------------------------------------------------------------------------
+C. Usage
+
+   warden_filer.py [-h] [-c CONFIG] [--oneshot] {sender,receiver}
+
+   Save Warden events as files or send files to Warden
+
+   positional arguments:
+     {sender,receiver}     choose direction: sender picks up files and submits
+                           them to Warden, receiver pulls events from Warden
+                           and saves them as files
+
+   optional arguments:
+     -h, --help            show this help message and exit
+     -c CONFIG, --config CONFIG
+                           configuration file path
+     --oneshot             don't daemonise, run just once
+     -d, --daemon          daemonize
+     -p PID_FILE, --pid_file PID_FILE
+                           create PID file with this name
+
+
+   CONFIG denotes path to configuration file, default is warden_filer.cfg in
+current directory.
+   --oneshot instructs Filer to just do its work once (fetch available events
+or send event files present in directory), but obeys all other applicable
+options from configuration file (concerning logging, filtering, directories,
+etc.)
+   --daemon instructs Filer to go to full unix daemon mode. Without it,
+Filer just stays on foreground.
+   --pid_file makes Filer to create the usual PID file. Without it, no PID
+file gets created.
+
+------------------------------------------------------------------------------
+D. Configuration
+
+   Configuration is JSON object in file - however, lines starting with "#"
+or "//" are allowed and will be ignored as comments. File must contain valid
+JSON object, containing configuration. See also warden_filer.cfg as example.
+
+   warden - can contain Warden 3 configuration (see Warden doc), or path
+            to Warden configuration file
+   sender - configuration section for sender mode
+      dir - directory, whose "incoming" subdir will be checked for Idea
+            events to send out
+      done_dir - directory, into which the messages will be moved after
+            successful sending. If not set, processed messages will get
+            deleted, which is default, and usually what you want. Note that
+            this is just regular directory, no special locking precautions
+            and no subdirectories are done here, however if "done_dir" is on
+            the same filesystem as "dir"
+      filter - filter fields (same as in Warden query, see Warden and Idea
+            doc, possible keys: cat, nocat, group, nogroup, tag, notag),
+            unmatched events get discarded and deleted
+      node - o information about detector to be prepended into event Node
+            array (see Idea doc). Note that Warden server may require it to
+            correspond with client registration
+      poll_time - how often to check incoming directory (in seconds, defaults
+            to 5)
+      owait_timeout - how long to opportunistically wait for possible new
+            incoming files when number of files to process is less than
+            send_events_limit (in seconds, defaults to poll_time)
+      owait_poll_time - how often to check incoming directory during
+            opportunistic timeout (in seconds, defaults to 1)
+   receiver - configuration section for receiver mode
+      dir - directory, whose "incoming" subdir will serve as target for events
+      filter - filter fields for Warden query (see Warden and Idea doc,
+               possible keys: cat, nocat, group, nogroup, tag, notag)
+      node - o information about detector to be prepended into event Node
+             array (see Idea doc). Be careful here, you may ruin Idea
+             messages by wrongly formatted data and they are not checked
+             here in any way
+      poll_time - how often to check Warden server for new events (in seconds,
+             defaults to 5)
+      file_limit - limit number of files in "incoming" directory. When the limit
+             is reached, polling is paused for "limit_wait_time" seconds
+      limit_wait_time - wait this number of seconds if limit on number of files
+             is reached (defaults to 5)
+      
+
+    Both the "sender" and "reciever" sections can also bear daemon
+configuration.
+
+      work_dir - where should daemon chdir
+      chroot_dir - confine daemon into chroot directory
+      umask - explicitly set umask for created files
+      uid, gid - uid/gid, under which daemon will run
+
+------------------------------------------------------------------------------
+E. Directories and locking issues
+
+   Working directories are not just simple paths, but contain structure,
+loosely mimicked from Maildir with slightly changed names to avoid first look
+confusion. Simple path suffers locking issue: when one process saves file
+there, another process has no way to know whether file is already complete
+or not, and starting to read prematurely can lead to corrupted data read.
+Also, two concurrent processes may decide to work on one file, stomping on
+others legs.
+   So, your scripts and tools inserting data or taking data from working
+directories must obey simple protocols, which use atomic "rename" to avoid
+locking issues.
+   Also, your directory (and its structure) _must_ reside on the same
+filesystem to keep "rename" atomic. _Never_ try to mount some of the
+subdirectories ("tmp", "incoming", "errors") from other filesystem.
+
+ 1. Inserting file
+
+   * The file you want to create _must_ be created in the "tmp" subdirectory
+     first, _not_ "incoming". Filename is arbitrary, but must be unique among
+     all subdirectories.
+
+   * When done writing, rename the file into "incoming" subdir. Rename is
+     atomic operation, so for readers, file will appear either nonexistent
+     or complete.
+
+   For simple usage (bash scripts, etc.), just creating sufficiently random
+   filename in "tmp" and then moving into "incoming" may be enough.
+   Concatenating $RANDOM couple of times will do. :)
+   
+   For advanced or potentially concurrent usage inserting enough of unique
+   information into name is recommended - Filer itself uses hostname, pid,
+   unixtime, milliseconds, device number and file inode number to avoid
+   locking issues both on local and network based filesystems and to be
+   prepared for high traffic.
+
+ 2. Picking up file
+
+   * Rename the file to work with into "tmp" directory.
+
+   * Do whatever you want with contents, and when finished, rename file back
+     into "incoming", or remove, or move somewhere else, or move into "errors"
+     directory - what suits your needs, after all, it's your file.
+
+   Note that in concurrent environment file can disappear between directory
+   enumeration and attempt to rename - then just pick another one (and
+   possibly repeat), someone was swifter.
+
+------------------------------------------------------------------------------
+Copyright (C) 2011-2015 Cesnet z.s.p.o

warden_filer/check_file_count

+#!/bin/bash
+
+script=${0##*/}
+warn=0
+crit=65536
+read -rd '' helps <<EOF
+$script: Icinga plugin to check too high number of files in directory.
+
+Usage: $script -d dir [-w num] [-c num] [-h ]
+
+  -d dir    directory to watch
+  -w num    warning if number of files exceeds this value (default $warn)
+  -c num    critical if number of files exceeds this value (default $crit)
+EOF
+
+function bailout {
+	echo -n "$script" | tr '[:lower:]' '[:upper:]'
+	echo " $2 $3"
+	exit "$1"
+}
+
+while getopts hvVd:w:c: opt; do
+	case "$opt" in
+		h) bailout 3 "UNKNOWN" "$helps";;
+		d) dir="$OPTARG";;
+		w) warn="$OPTARG";;
+		c) crit="$OPTARG";;
+		"?") bailout 3 "UNKNOWN" "Unknown option, use -h for help";;
+	esac
+done
+
+[ -z "$dir" ] && bailout 3 "UNKNOWN" "-d not specified"
+
+count=$(find "$dir" -mindepth 1 -maxdepth 1 | wc -l)
+
+[ "$count" -gt "$crit" ] && bailout 2 "CRIT" "$count"
+
+[ "$count" -gt "$warn" ] && bailout 1 "WARN" "$count"
+
+bailout 0 "OK" "$count"

warden_filer/warden_filer.cfg.dist

+// For all options see documentation
+{
+    // Warden config can be also referenced as:
+    // "warden": "/path/to/warden_client.cfg"
+    "warden": {
+        "url": "https://example.com/warden3",
+        "cafile": "tcs-ca-bundle.pem",
+        "keyfile": "my.key.pem",
+        "certfile": "my.cert.pem",
+        "timeout": 60,
+        "retry": 20,
+        "pause": 5,
+        "filelog": {"level": "debug"},
+        "name": "com.example.warden.test",
+        "secret": "SeCrEt"
+    },
+    "sender": {
+        // Maildir like directory, whose "incoming" subdir will be checked
+        // for Idea events to send out
+        "dir": "warden_sender",
+        // Optional filter fields, unmatched events are discarded (and removed)
+        //"filter": {
+        //    "cat": ["Test", "Recon.Scanning"],
+        //    "nocat": null,
+        //    "group": ["cz.example"],
+        //    "nogroup": null,
+        //    "tag": null,
+        //    "notag": ["Honeypot"]
+        //},
+        // Optional information about detector to be prepended into Idea Node array
+        //"node": {
+        //    "Name": "cz.example.warden.test_sender",
+        //    "Type": ["Relay"]
+        //}
+    },
+    "receiver": {
+        // Maildir like directory, whose "incoming" will serve as target for events
+        "dir": "warden_receiver",
+        // Optional filter fields for Warden query
+        //"filter": {
+        //    "cat": ["Test", "Recon.Scanning"],
+        //    "nocat": null,
+        //    "group": ["cz.cesnet"],
+        //    "nogroup": null,
+        //    "tag": null,
+        //    "notag": ["Honeypot"]
+        //},
+        // Optional information about detector to be prepended into Idea Node array
+        //"node": {
+        //    "Name": "cz.example.warden.test_receiver",
+        //    "Type": ["Relay"]
+        //},
+        // Optional limit on number of files in "incoming" directory
+        //"file_limit": 10000
+    }
+}

warden_filer/warden_filer.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2015 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+from warden_client import Client, Error, read_cfg
+import json
+import string
+import os
+import sys
+import errno
+import socket
+import time
+import logging
+import signal
+import resource
+import atexit
+import argparse
+from os import path, mkdir
+from random import choice, randint
+
+# for py2/py3 compatibility
+try:
+    basestring
+except NameError:
+    basestring = str
+
+VERSION = "3.0-beta3"
+
+class NamedFile(object):
+    """ Wrapper class for file objects, which allows and tracks filename
+        changes.
+    """
+
+    def __init__(self, pth, name, fd=None):
+        self.name = name
+        self.path = pth
+        if fd:
+            self.f = os.fdopen(fd, "w+b")
+        else:
+            self.f = None
+
+
+    def __str__(self):
+        return "%s(%s, %s)" % (type(self).__name__, self.path, self.name)
+
+
+    def get_path(self, basepath=None, name=None):
+        return path.join(basepath or self.path, name or self.name)
+
+
+    def open(self, mode):
+        return open(self.get_path(), mode)
+
+
+    def moveto(self, destpath):
+        os.rename(self.get_path(), self.get_path(basepath=destpath))
+        self.path = destpath
+
+
+    def rename(self, newname):
+        os.rename(self.get_path(), self.get_path(name=newname))
+        self.name = newname
+
+
+    def remove(self):
+        os.remove(self.get_path())
+
+
+
+class SafeDir(object):
+    """ Maildir like directory for safe file exchange.
+        - Producers are expected to drop files into "tmp" under globally unique
+          filename and rename it into "incoming" atomically (newfile method)
+        - Workers pick files in "incoming", rename them into "tmp",
+          do whatever they want, and either discard them or move into
+          "errors" directory
+    """
+
+    def __init__(self, p):
+        self.path = self._ensure_path(p)
+        self.incoming = self._ensure_path(path.join(self.path, "incoming"))
+        self.errors = self._ensure_path(path.join(self.path, "errors"))
+        self.temp = self._ensure_path(path.join(self.path, "tmp"))
+        self.hostname = socket.gethostname()
+        self.pid = os.getpid()
+
+
+    def __str__(self):
+        return "%s(%s)" % (type(self).__name__, self.path)
+
+
+    def _ensure_path(self, p):
+        try:
+            mkdir(p)
+        except OSError:
+            if not path.isdir(p):
+                raise
+        return p
+
+
+    def _get_new_name(self, device=0, inode=0):
+        return "%s.%d.%f.%d.%d.idea" % (
+            self.hostname, self.pid, time.time(), device, inode)
+
+
+    def newfile(self):
+        """ Creates file with unique filename within this SafeDir.
+            - hostname takes care of network filesystems
+            - pid distinguishes two daemons on one machine
+              (we are not multithreaded, so this is enough)
+            - time in best precision supported narrows window within process
+            - device/inode makes file unique on particular filesystem
+            In fact, device/inode is itself enough for uniqueness, however
+            if we mandate wider format, users can use simpler form with
+            random numbers instead of device/inode, if they choose to,
+            and it will still ensure reasonable uniqueness.
+        """
+
+        # Note: this simpler device/inode algorithm replaces original,
+        #       which checked uniqueness among all directories by atomic
+        #       links.
+
+        # First find and open name unique within tmp
+        tmpname = None
+        while not tmpname:
+            tmpname = self._get_new_name()
+            try:
+                fd = os.open(path.join(self.temp, tmpname), os.O_CREAT | os.O_RDWR | os.O_EXCL)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise   # other errors than duplicates should get noticed
+                tmpname = None
+        # Now we know device/inode, rename to make unique within system
+        stat = os.fstat(fd)
+        newname = self._get_new_name(stat.st_dev, stat.st_ino)
+        nf = NamedFile(self.temp, tmpname, fd)
+        nf.rename(newname)
+        return nf
+
+
+    def get_incoming(self):
+        return [NamedFile(self.incoming, n) for n in os.listdir(self.incoming)]
+
+
+    def get_incoming_cnt(self):
+        """Get number of files in the incoming directory"""
+        return len(os.listdir(self.incoming))
+
+
+
+def receiver(config, wclient, sdir, oneshot):
+    poll_time = config.get("poll_time", 5)
+    node = config.get("node", None)
+    conf_filt = config.get("filter", {})
+    file_limit = config.get("file_limit", None)
+    wait_time = config.get("limit_wait_time", 5)
+    filt = {}
+    # Extract filter explicitly to be sure we have right param names for getEvents
+    for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"):
+        filt[s] = conf_filt.get(s, None)
+
+    while running_flag:
+        count_ok = count_err = 0
+        limit_reached = False
+        if file_limit:
+            cnt_files = sdir.get_incoming_cnt() # Count files in 'incoming' dir
+            remain_to_limit = file_limit - cnt_files
+            # Query server, but not for more events than what can fit into limit
+            if remain_to_limit > 0:
+                events = wclient.getEvents(count=remain_to_limit, **filt)
+            else:
+                events = []
+            # Check whether limit was reached
+            if len(events) >= remain_to_limit:
+                limit_reached = True
+        else:
+            events = wclient.getEvents(**filt)
+
+        for event in events:
+            if node:
+                nodelist = event.setdefault("Node", [])
+                nodelist.insert(0, node)
+            try:
+                nf = None
+                nf = sdir.newfile()
+                with nf.f as f:
+                    data = json.dumps(event)
+                    f.write(data.encode('utf-8'))
+                nf.moveto(sdir.incoming)
+                count_ok += 1
+            except Exception as e:
+                Error(message="Error saving event", exc=sys.exc_info(), file=str(nf),
+                      event_ids=[event.get("ID")], sdir=sdir.path).log(wclient.logger)
+                count_err += 1
+
+        if events:
+            wclient.logger.info(
+                "warden_filer: received %d, errors %d"
+                % (count_ok, count_err))
+
+        if limit_reached:
+            wclient.logger.info("Limit on number of files in 'incoming' dir reached.")
+
+        if oneshot:
+            if not events or limit_reached:
+                terminate_me(None, None)
+        else:
+            if limit_reached:
+                time.sleep(wait_time)
+            elif not events:
+                time.sleep(poll_time)
+
+
+
+def match_event(event, cat=None, nocat=None, tag=None, notag=None, group=None, nogroup=None):
+
+    cat_match = tag_match = group_match = True
+
+    if cat or nocat:
+        event_cats = event.get("Category")
+        event_full_cats = set(event_cats) | set(cat.split(".", 1)[0] for cat in event_cats)
+        cat_match = set(cat or nocat) & event_full_cats
+        cat_match = not cat_match if nocat else cat_match
+
+    try:
+        event_node = event.get("Node", [])[0]
+    except IndexError:
+        event_node = {}
+
+    if tag or notag:
+        event_tags = set(event_node.get("Type", []))
+        tag_match = set(tag or notag) & event_tags
+        tag_match = not tag_match if notag else tag_match
+
+    if group or nogroup:
+        event_name = event_node.get("Name")
+        namesplit = event_name.split(".")
+        allnames = set([".".join(namesplit[0:l]) for l in range(1, len(namesplit)+1)])
+        group_match = set(group or nogroup) & allnames
+        group_match = not group_match if nogroup else group_match
+
+    return cat_match and tag_match and group_match
+
+
+
+def get_dir_list(sdir, owait_poll_time, owait_timeout, nfchunk, oneshot):
+    nflist = sdir.get_incoming()
+    if oneshot and not nflist:
+        terminate_me(None, None)
+    timeout = time.time() + owait_timeout
+    while len(nflist)<nfchunk and time.time()<timeout and running_flag:
+        time.sleep(owait_poll_time)
+        nflist = sdir.get_incoming()
+    return nflist
+
+
+
+def sender(config, wclient, sdir, oneshot):
+    poll_time = config.get("poll_time", 5)
+    owait_poll_time = config.get("owait_poll_time", 1)
+    owait_timeout = config.get("owait_timeout", poll_time)
+    node = config.get("node", None)
+    done_dir = config.get("done_dir", None)
+    conf_filt = config.get("filter", {})
+    filt = {}
+    # Extract filter explicitly to be sure we have right param names for match_event
+    for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"):
+        filt[s] = conf_filt.get(s, None)
+
+    nfchunk = wclient.send_events_limit
+    while running_flag:
+        nflist = get_dir_list(sdir, owait_poll_time, owait_timeout, nfchunk, oneshot)
+        if oneshot and not nflist:
+            terminate_me(None, None)
+        while running_flag and not nflist:
+            # No new files, wait and try again
+            time.sleep(poll_time)
+            nflist = get_dir_list(sdir, owait_poll_time, owait_timeout, nfchunk, oneshot)
+
+        # Loop over all chunks. However:
+        # - omit the last loop, if there is less data than the optimal window;
+        #   next get_dir_list will still get it again, possibly together with
+        #   new files, which may have appeared meanwhile
+        # - unless it's the sole loop (so that at least _something_ gets sent)
+        nfindex = 0
+        while nfindex<len(nflist) and ((len(nflist)-nfindex>=nfchunk) or not nfindex):
+            events = []
+            nf_sent = []
+            count_ok = count_err = count_unmatched = count_local = 0
+            for nf in nflist[nfindex:nfindex+nfchunk]:
+                # prepare event array from files
+                try:
+                    nf.moveto(sdir.temp)
+                except Exception:
+                    continue    # Silently go to next filename, somebody else might have interfered
+                try:
+                    with nf.open("rb") as fd:
+                        data = fd.read().decode('utf-8')
+                        event = json.loads(data)
+                        if not match_event(event, **filt):
+                            wclient.logger.debug("Unmatched event: %s" % data)
+                            count_unmatched += 1
+                            nf.remove()
+                            continue
+                        if node:
+                            nodelist = event.setdefault("Node", [])
+                            nodelist.insert(0, node)
+                        events.append(event)
+                        nf_sent.append(nf)
+                except Exception as e:
+                    Error(message="Error loading event", exc=sys.exc_info(), file=str(nf),
+                          sdir=sdir.path).log(wclient.logger)
+                    nf.moveto(sdir.errors)
+                    count_local += 1
+
+            res = wclient.sendEvents(events)
+
+            if isinstance(res, Error):
+                for e in res.errors:
+                    errno = e["error"]
+                    evlist = e.get("events", range(len(nf_sent)))  # None means all
+                    for i in evlist:
+                        if nf_sent[i]:
+                            nf_sent[i].moveto(sdir.errors)
+                            nf_sent[i] = None
+                            count_err += 1
+
+            # Cleanup rest - the succesfully sent events
+            for name in nf_sent:
+                if name:
+                    if done_dir:
+                        name.moveto(done_dir)
+                    else:
+                        name.remove()
+                    count_ok += 1
+            wclient.logger.info(
+                "warden_filer: saved %d, warden errors %d, local errors %d, unmatched %d" % (count_ok, count_err, count_local, count_unmatched))
+
+            nfindex += nfchunk  # skip to next chunk of files
+            nfchunk = wclient.send_events_limit # might get changed by server
+
+
+
+def get_logger_files(logger):
+    """ Return file objects of loggers """
+    files = []
+    for handler in logger.handlers:
+        if hasattr(handler, 'stream') and hasattr(handler.stream, 'fileno'):
+            files.append(handler.stream)
+        if hasattr(handler, 'socket') and hasattr(handler.socket, 'fileno'):
+            files.append(handler.socket)
+    return files
+
+
+
+def daemonize(
+        work_dir = None, chroot_dir = None,
+        umask = None, uid = None, gid = None,
+        pidfile = None, files_preserve = [], signals = {}):
+    # Dirs, limits, users
+    if chroot_dir is not None:
+        os.chdir(chroot_dir)
+        os.chroot(chroot_dir)
+    if umask is not None:
+        os.umask(umask)
+    if work_dir is not None:
+        os.chdir(work_dir)
+    if gid is not None:
+        os.setgid(gid)
+    if uid is not None:
+        os.setuid(uid)
+    # Doublefork, split session
+    if os.fork()>0:
+        os._exit(0)
+    os.setsid()
+    if os.fork()>0:
+        os._exit(0)
+    # Setup signal handlers
+    for (signum, handler) in signals.items():
+        signal.signal(signum, handler)
+    # Close descriptors
+    descr_preserve = set(f.fileno() for f in files_preserve)
+    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    if maxfd==resource.RLIM_INFINITY:
+        maxfd = 65535
+    for fd in range(maxfd, 3, -1):  # 3 means omit stdin, stdout, stderr
+        if fd not in descr_preserve:
+            try:
+                os.close(fd)
+            except Exception:
+                pass
+    # Redirect stdin, stdout, stderr to /dev/null
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for fd in range(3):
+        os.dup2(devnull, fd)
+    # PID file
+    if pidfile is not None:
+        pidd = os.open(pidfile, os.O_RDWR|os.O_CREAT|os.O_EXCL|os.O_TRUNC)
+        os.write(pidd, (str(os.getpid())+"\n").encode())
+        os.close(pidd)
+        # Define and setup atexit closure
+        @atexit.register
+        def unlink_pid():
+            try:
+                os.unlink(pidfile)
+            except Exception:
+                pass
+
+
+
+running_flag = True     # Daemon cleanly exits when set to False
+
+def terminate_me(signum, frame):
+    global running_flag
+    running_flag = False
+
+
+
+class DummyContext(object):
+    """ In one shot mode we use this instead of DaemonContext """
+    def __enter__(self): pass
+    def __exit__(self, *exc): pass
+
+
+
+def get_args():
+    argp = argparse.ArgumentParser(
+        description="Save Warden events as files or send files to Warden")
+    argp.add_argument("func",
+        choices=["sender", "receiver"],
+        action="store",
+        help="choose direction: sender picks up files and submits them to "
+              "Warden, receiver pulls events from Warden and saves them as files")
+    argp.add_argument("-c", "--config",
+        default=path.splitext(__file__)[0]+".cfg",
+        dest="config",
+        help="configuration file path")
+    argp.add_argument("-o", "--oneshot",
+        default=False,
+        dest="oneshot",
+        action="store_true",
+        help="don't daemonise, run just once")
+    argp.add_argument("-d", "--daemon",
+        default=False,
+        dest="daemon",
+        action="store_true",
+        help="daemonize")
+    argp.add_argument("-p", "--pid_file",
+        default=None,
+        dest="pid_file",
+        action="store",
+        help="create PID file with this name")
+    return argp.parse_args()
+
+
+
+def get_configs():
+    config = read_cfg(args.config)
+
+    # Allow inline or external Warden config
+    wconfig = config.get("warden", "warden_client.cfg")
+    if isinstance(wconfig, basestring):
+        wconfig = read_cfg(wconfig)
+
+    fconfig = config.get(args.func, {})
+
+    return wconfig, fconfig
+
+
+
+def get_uid_gid(str_id, get_nam_func):
+    if str_id:
+        try:
+            id = int(str_id)
+        except ValueError:
+            id = get_nam_func(str_id)[2]
+    else:
+        id = None
+    return id
+
+
+
+if __name__ == "__main__":
+
+    args = get_args()
+
+    function = sender if args.func=="sender" else receiver
+
+    wconfig, fconfig = get_configs()
+
+    wclient = Client(**wconfig)
+
+    try:
+        if args.daemon:
+            from pwd import getpwnam
+            from grp import getgrnam
+            uid = get_uid_gid(fconfig.get("uid"), getpwnam)
+            gid = get_uid_gid(fconfig.get("gid"), getgrnam)
+
+            daemonize(
+                work_dir = fconfig.get("work_dir", "."),
+                chroot_dir = fconfig.get("chroot_dir"),
+                umask = fconfig.get("umask"),
+                uid = uid,
+                gid = gid,
+                pidfile = args.pid_file,
+                files_preserve = get_logger_files(wclient.logger),
+                signals = {
+                    signal.SIGTERM: terminate_me,
+                    signal.SIGINT: terminate_me,
+                    signal.SIGHUP: signal.SIG_IGN,
+                    signal.SIGTTIN: signal.SIG_IGN,
+                    signal.SIGTTOU: signal.SIG_IGN})
+
+        safe_dir = SafeDir(fconfig.get("dir", args.func))
+        wclient.logger.info("Starting %s" % args.func)
+        function(fconfig, wclient, safe_dir, args.oneshot)
+        wclient.logger.info("Exiting %s" % args.func)
+
+    except Exception as e:
+        Error(message="%s daemon error" % args.func, exc=sys.exc_info()).log(wclient.logger)

warden_filer/warden_filer_logrotate.dist

+# You may want to review and/or change the logfile path, user/group in
+# 'create' and daemon to restart in 'postrotate'
+
+/var/log/warden_filer.log
+{
+	rotate 52
+	weekly
+	missingok
+	notifempty
+	compress
+	delaycompress
+	dateext
+	create 640 mentat mentat
+	postrotate
+		/etc/init.d/warden_filer_sender restart
+		/etc/init.d/warden_filer_receiver restart
+	endscript
+}

warden_filer/warden_filer_receiver

+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:          warden_filer_receiver
+# Required-Start:    $local_fs $syslog
+# Required-Stop:     $local_fs $syslog
+# Should-Start:      $network $named
+# Should-Stop:       $network $named
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Warden Filer - receiver
+### END INIT INFO
+
+DAEMON_NAME=warden_filer
+FUNC=receiver
+DAEMON_PATH=/usr/local/bin/"$DAEMON_NAME".py
+SERVICE_NAME="${DAEMON_NAME}_${FUNC}"
+PID=/var/run/"$DAEMON_NAME"/"$FUNC".pid
+CONFIG=/etc/"$DAEMON_NAME".cfg
+
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$SERVICE_NAME" ] && . /etc/$n/"$SERVICE_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
+
+ACTION="$1"
+
+case "$ACTION" in
+	start)
+		mkdir -p "${PID%/*}"
+		log_daemon_msg "Starting $SERVICE_NAME" "$SERVICE_NAME"
+		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" --pid_file "$PID" --daemon "$FUNC"
+		log_end_msg $?
+		;;
+	stop)
+		log_daemon_msg "Stopping $SERVICE_NAME" "$SERVICE_NAME"
+		killproc -p "$PID" "$DAEMON_PATH"
+		log_end_msg $?
+		;;
+	restart|force-reload)
+		$0 stop && sleep 2 && exec $0 start
+		;;
+	status)
+		status_of_proc -p "$PID" "$DAEMON_PATH" "$SERVICE_NAME"
+		;;
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 2
+		;;
+esac

warden_filer/warden_filer_sender

+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:          warden_filer_sender
+# Required-Start:    $local_fs $syslog
+# Required-Stop:     $local_fs $syslog
+# Should-Start:      $network $named
+# Should-Stop:       $network $named
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Warden Filer - sender
+### END INIT INFO
+
+DAEMON_NAME=warden_filer
+FUNC=sender
+DAEMON_PATH=/usr/local/bin/"$DAEMON_NAME".py
+SERVICE_NAME="${DAEMON_NAME}_${FUNC}"
+PID=/var/run/"$DAEMON_NAME"/"$FUNC".pid
+CONFIG=/etc/"$DAEMON_NAME".cfg
+
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$SERVICE_NAME" ] && . /etc/$n/"$SERVICE_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
+
+ACTION="$1"
+
+case "$ACTION" in
+	start)
+		mkdir -p "${PID%/*}"
+		log_daemon_msg "Starting $SERVICE_NAME" "$SERVICE_NAME"
+		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" --pid_file "$PID" --daemon "$FUNC"
+		log_end_msg $?
+		;;
+	stop)
+		log_daemon_msg "Stopping $SERVICE_NAME" "$SERVICE_NAME"
+		killproc -p "$PID" "$DAEMON_PATH"
+		log_end_msg $?
+		;;
+	restart|force-reload)
+		$0 stop && sleep 2 && exec $0 start
+		;;
+	status)
+		status_of_proc -p "$PID" "$DAEMON_PATH" "$SERVICE_NAME"
+		;;
+	*)
+		echo "Usage: $0 {start|stop|restart|status}"
+		exit 2
+		;;
+esac

warden_ra/LICENSE

+Copyright (c) 2011-2016 Cesnet z.s.p.o <warden-info@cesnet.cz>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\ No newline at end of file

warden_ra/README

+Warden Registration Authority for Warden 3.X
+============================================
+
+Introduction
+------------
+
+Warden RA is a certificate registration authority for Warden_ server.
+It is meant to support the client registration process and simplification
+of the credential transport.
+
+As Warden clients are authenticated by X509 certificate, the usual certificate
+generation process can be used - local key and certificate request gets
+generated, the request is submitted to registration authority, and after
+review, certificate is issued and delivered back.
+
+However in centralised Warden setup, it is useful to be able to preallow
+certificate for particular client during negotiation phase (thus removing
+another round-trip).
+
+This is done by issuing 'token' by Warden registration officer during client
+registration, which is just a oneshot password, allowing sending the request
+and getting new certificate in one step through web API.
+
+Password is meant only for completely new clients or unusual situations,
+however RA also allows prolongation - generating new certificate by using
+old certificate (which must be still valid, of course) instead of password.
+
+The application script, which can be distributed to newly registered clients,
+is also included.
+
+Dependencies
+------------
+
+ 1. Platform
+
+    Python 2.7+
+    Apache 2.2+
+    mod_wsgi 3.3+
+
+
+Registration process
+--------------------
+
+New client credentials
+``````````````````````
+
+After succesful negotiation of new Warden client parameters, the registration
+officer enables new certificate generation by issuing (on the server side):
+
+    warden_ra.py applicant --name org.example.warden.client
+
+The tool generates and sets one time password on the registration authority
+side, and this password can be sent (preferably through the secured channel)
+to the new client administrator along with other setup information.
+
+The client administrator runs the application script with application
+password:
+
+    warden_apply.sh https://warden.example.org/warden-ra/ org.example.warden.client P4SSW0RD
+
+The script creates new X509 key, CSR certificate request and makes call to
+the Warden RA web service, where it obtains the new complete certificate.
+
+
+Prolonging existing client credentials
+``````````````````````````````````````
+
+The client administrator runs the application script with his existing valid
+Warden credentials, which he needs to prolong:
+
+    warden_apply.sh https://warden.example.org/warden-ra/ org.example.warden.client cert.pem key.pem
+
+The script creates new X509 key, CSR certificate request and makes call to
+the Warden RA web service, where it obtains the new complete certificate.
+
+
+Installation
+------------
+
+This depends heavily on your distribution and Apache configuration.
+Basically you need to create and include apache.conf:
+
+  Include /opt/warden_ra/apache22.conf
+
+or paste the contents into whichever Directory, Location or VirtualHost
+you dedicate for Warden RA. Note that you have to use different host than
+the one for Warden server, as Warden RA needs different Apache options
+for SSL authentication.
+
+You may need to change paths to certificate/key/ca material, path to
+warden_server.wsgi and web path alias.
+
+Note that warden_ra itself is NOT thread safe, so included configuration
+expects Apache with mpm-prefork worker, or you will have to configure
+mod_wsgi as separate process with threads=1.
+
+Also, for warden_server.wsgi, you can use warden_server.wsgi.dist as
+a template. You will possibly need to change at least configuration
+file path.
+
+ * Now install and/or configure RA backend (see README.openssl or README.ejbca)
+
+ * Configure Warden RA (see next chapter)
+
+ * Reload Apache
+
+
+Configuration
+-------------
+
+   Configuration is JSON object in file (warden_server.cfg by default),
+however, lines starting with "#" or "//" are allowed and will be ignored as
+comments. File must contain valid JSON object, containing configuration. See
+also warden_server.cfg.dist as example.
+
+   Top level JSON object contains subsections, which configure particular
+aspect of the server. Each subsection can contain "type" key, which chooses
+particular implementation object of the aspect, for example type of logger
+(file, syslog), such as:
+
+   {
+      "Log": {
+         "type": "SysLogger",
+         ...
+      },
+      "DB": { ... }
+   }
+
+Sections and their "type" objects can be:
+
+    Log: FileLogger, SysLogger
+    Auth: OptionalAuthenticator
+    Registry: EjbcaRegistry, OpenSSLRegistry
+    Handler: CertHandler
+
+"type" keyword is not mandatory, if not specified, first implementation
+object from particular section list is used ("FileLogger" for example).
+
+Object function and configuration keys are as follows:
+
+   FileLogger: logging into file on the filesystem
+      filename: name of the file, defaults to "warden_ra.log" at
+                installation directory
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+   SysLogger: logging into unix syslog
+      socket: path to syslog socket, defaults to "/dev/log"
+      facility: syslog facility, defaults to "daemon"
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+    OptionalAuthenticator: authenticate based on X509 certificate, or
+        signal the password auth for the registry
+
+    CertHandler: the main certificate requestor implementation
+
+    For OpenSSLRegistry or EJBCARegistry configuration please see
+    README.openssl or README.ejbca respectively.
+
+
+Command line
+------------
+
+When run from the command line, RA allows for client and request management.
+
+
+    warden_ra.py [--help] [-c CONFIG] [-v]
+                        {list,register,applicant,request,gencert} ...
+
+    Warden server certificate registry
+    
+    arguments:
+      --help                show this help message and exit
+      -c CONFIG, --config CONFIG
+                            path to configuration file
+      -v, --verbose         be more chatty
+    
+    commands:
+      {list,register,applicant,request,gencert}
+        list                list clients
+        register            register client
+        applicant           allow for certificate application
+        request             generate CSR
+        gencert             get new certificate
+
+
+    warden_ra.py list [--help] [--name NAME]
+    
+    List registered clients.
+    
+    arguments:
+      --help       show this help message and exit
+      --name NAME  client name
+
+
+    warden_ra.py register [--help] --name NAME --admins
+                                 [ADMINS [ADMINS ...]]
+    
+    Add client registration entry.
+    
+    arguments:
+      --help                show this help message and exit
+      --name NAME           client name
+      --admins [ADMINS [ADMINS ...]]
+                            administrator list
+
+
+    warden_ra.py applicant [--help] --name NAME [--password PASSWORD]
+    
+    Set client into certificate application mode and set its password
+    
+    optional arguments:
+      --help               show this help message and exit
+      --name NAME          client name
+      --password PASSWORD  password for application (will be autogenerated if not
+                           set)
+
+    
+.. _Warden: https://warden.cesnet.cz/
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

warden_ra/README.ejbca

+EJBCA backend for Warden 3.# Registration Authority
+===================================================
+
+Introduction
+------------
+
+EJBCA_ is an open source CA management software. To use this backend
+with Warden RA, you need to have it already installed and running.
+Tested with EJBCA_ 3.9.
+
+.. _EJBCA: https://www.ejbca.org/
+
+
+Configuration
+-------------
+
+    Options for "Registry: EjbcaRegistry" section.
+
+    url: EJBCA API URL, for example "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl"
+    cert: certificate for authentication to EJBCA, defaults to "warden_ra.cert.pem"
+    key: key for authentication to EJBCA, defaults to "warden_ra.key.pem"
+    ca_name: name of the CA, dedicated for Warden, defaults to "Example CA"
+    certificate_profile_name: name of the EJBCA certificate profile, defaults to "Example"
+    end_entity_profile_name: name of the EJBCA entity profile, defaults to "Example EE"
+    subject_dn_template: template for the DN generation, defaults to "DC=cz,DC=example-ca,DC=warden,CN=%s"
+    username_suffix: suffix, which will be added to EJBCA entities, defaults to "@warden"
+
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

warden_ra/README.openssl

+OpenSSL local backed for Warden 3.# Registration Authority
+==========================================================
+
+Introduction
+------------
+
+This backend allows using basic `openssl ca`_ facility for certificate
+emission. Client information is kept as plain config files within "clients"
+subdirectory. Also, received CSRs and issued certificates are saved in "csr"
+and "newcerts" subdirectories, respectively. File "lock" is used to conduct
+concurrent access to running openssl binary.
+
+.. _openssl ca: https://www.openssl.org/docs/manmaster/man1/openssl-ca.html
+
+
+Installation
+------------
+
+Choose directory where OpenSSL CA structure will reside (for example
+"ca").
+
+    # mkdir ca
+    # cd ca/
+    /ca# mkdir certs crl newcerts private clients csr
+    /ca# chmod 700 private
+    /ca# touch index.txt
+    /ca# echo 1024 > serial
+
+Adjust permissions.
+
+   # s-bit, so newly created files receive permissions of parent
+   # directory, not of creator
+   ca# find . -type d | xargs chmod g+s
+   # owner - apache group (this is for Debian, adjust accordingly for
+   # different distribution)
+   ca# chgrp -R www-data .
+
+Generate CA root certificate.
+
+    ca# openssl genrsa -out private/ca.key.pem 4096
+    ca# openssl req -config openssl.cnf \
+                -key private/ca.key.pem \
+                -new -x509 -days 7300 -sha256 -extensions v3_ca \
+                -out certs/ca.cert.pem
+    ca# chmod 444 private/ca.key.pem certs/ca.cert.pem
+
+Create "openssl.cnf" in base directory. You can use "openssl.cnf.example" as
+a basis.
+
+
+Configuration
+-------------
+
+    Options for "Registry: OpenSSLRegistry" section.
+
+    base_dir: Base directory where OpenSSL CA environment is managed
+    subject_dn_template: Template for DN of issued certs, defaults to "DC=cz,DC=example-ca,DC=warden,CN=%s"
+    openssl_sign: OpenSSL command and arguments to run for signing, defaults to "openssl ca -config %(cnf)s -batch -extensions server_cert -days 375 -notext -md sha256 -in %(csr)s -subj '%(dn)s'"
+
+
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

warden_ra/apache22.conf

+SSLEngine on
+
+SSLVerifyClient optional
+SSLOptions +StdEnvVars +ExportCertData
+
+SSLCertificateFile      /opt/warden_server/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/key.pem
+SSLCACertificateFile    /opt/warden_server/chain_TERENA_SSL_CA_3.pem
+
+WSGIScriptAlias /warden_ra /opt/warden-ra/warden_ra.wsgi
+
+<Directory /opt/warden-ra/warden_ra.wsgi>
+    Order allow,deny
+    Allow from all
+</Directory>

warden_ra/apache24.conf

+SSLEngine on
+
+SSLVerifyClient optional
+SSLOptions +StdEnvVars +ExportCertData
+
+SSLCertificateFile      /opt/warden_server/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/key.pem
+SSLCACertificateFile    /opt/warden_server/chain_TERENA_SSL_CA_3.pem
+
+WSGIScriptAlias /warden_ra /opt/warden-ra/warden_ra.wsgi
+
+<Directory /opt/warden-ra/warden_ra.wsgi>
+    Require all granted
+</Directory>

warden_ra/ejbcaws.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2016, CESNET, z. s. p. o.
+# Use of this source is governed by an ISC license, see LICENSE file.
+
+import sys
+import socket
+import base64
+import suds.transport.http
+import suds.client
+import M2Crypto
+
+
+if sys.version_info[0] >= 3:
+    import urllib.request, urllib.error, urllib.parse
+    import http.client
+
+    def get_https_handler():
+        return urllib.request.HTTPSHandler
+else:
+    import urllib2
+    import httplib
+
+    def get_https_handler():
+        return urllib2.HTTPSHandler
+
+
+STATUS_FAILED = 11
+STATUS_GENERATED = 40
+STATUS_HISTORICAL = 60
+STATUS_INITIALIZED = 20
+STATUS_INPROCESS = 30
+STATUS_KEYRECOVERY = 70
+STATUS_NEW = 10
+STATUS_REVOKED = 50
+
+MATCH_TYPE_BEGINSWITH = 1
+MATCH_TYPE_CONTAINS = 2
+MATCH_TYPE_EQUALS = 0
+MATCH_WITH_CA = 5
+MATCH_WITH_CERTIFICATEPROFILE = 4
+MATCH_WITH_COMMONNAME = 101
+MATCH_WITH_COUNTRY = 112
+MATCH_WITH_DIRECTORYNAME = 204
+MATCH_WITH_DN = 7
+MATCH_WITH_DNSERIALNUMBER = 102
+MATCH_WITH_DNSNAME = 201
+MATCH_WITH_DOMAINCOMPONENT = 111
+MATCH_WITH_EDIPARTNAME = 205
+MATCH_WITH_EMAIL = 1
+MATCH_WITH_ENDENTITYPROFILE = 3
+MATCH_WITH_GIVENNAME = 103
+MATCH_WITH_GUID = 209
+MATCH_WITH_INITIALS = 104
+MATCH_WITH_IPADDRESS = 202
+MATCH_WITH_LOCALE = 109
+MATCH_WITH_ORGANIZATION = 108
+MATCH_WITH_ORGANIZATIONUNIT = 107
+MATCH_WITH_REGISTEREDID = 207
+MATCH_WITH_RFC822NAME = 200
+MATCH_WITH_STATE = 110
+MATCH_WITH_STATUS = 2
+MATCH_WITH_SURNAME = 105
+MATCH_WITH_TITLE = 106
+MATCH_WITH_TOKEN = 6
+MATCH_WITH_UID = 100
+MATCH_WITH_UPN = 208
+MATCH_WITH_URI = 206
+MATCH_WITH_USERNAME = 0
+MATCH_WITH_X400ADDRESS = 203
+
+TOKEN_TYPE_JKS = "JKS"
+TOKEN_TYPE_P12 = "P12"
+TOKEN_TYPE_PEM = "PEM"
+TOKEN_TYPE_USERGENERATED = "USERGENERATED"
+
+VIEW_RIGHTS = "/view_end_entity"
+EDIT_RIGHTS = "/edit_end_entity"
+CREATE_RIGHTS = "/create_end_entity"
+DELETE_RIGHTS = "/delete_end_entity"
+REVOKE_RIGHTS = "/revoke_end_entity"
+HISTORY_RIGHTS = "/view_end_entity_history"
+APPROVAL_RIGHTS = "/approve_end_entity"
+HARDTOKEN_RIGHTS = "/view_hardtoken"
+HARDTOKEN_PUKDATA_RIGHTS = "/view_hardtoken/puk_data"
+KEYRECOVERY_RIGHTS = "/keyrecovery"
+
+ENDENTITYPROFILEBASE = "/endentityprofilesrules"
+ENDENTITYPROFILEPREFIX = "/endentityprofilesrules/"
+USERDATASOURCEBASE = "/userdatasourcesrules"
+USERDATASOURCEPREFIX = "/userdatasourcesrules/"
+UDS_FETCH_RIGHTS = "/fetch_userdata"
+UDS_REMOVE_RIGHTS = "/remove_userdata"
+
+CABASE = "/ca"
+CAPREFIX = "/ca/"
+ROLE_PUBLICWEBUSER = "/public_web_user"
+ROLE_ADMINISTRATOR = "/administrator"
+ROLE_SUPERADMINISTRATOR = "/super_administrator"
+REGULAR_CAFUNCTIONALTY = "/ca_functionality"
+REGULAR_CABASICFUNCTIONS = "/ca_functionality/basic_functions"
+REGULAR_ACTIVATECA = "/ca_functionality/basic_functions/activate_ca"
+REGULAR_RENEWCA = "/ca_functionality/renew_ca"
+REGULAR_VIEWCERTIFICATE = "/ca_functionality/view_certificate"
+REGULAR_APPROVECAACTION = "/ca_functionality/approve_caaction"
+REGULAR_CREATECRL = "/ca_functionality/create_crl"
+REGULAR_EDITCERTIFICATEPROFILES = "/ca_functionality/edit_certificate_profiles"
+REGULAR_CREATECERTIFICATE = "/ca_functionality/create_certificate"
+REGULAR_STORECERTIFICATE = "/ca_functionality/store_certificate"
+REGULAR_RAFUNCTIONALITY = "/ra_functionality"
+REGULAR_EDITENDENTITYPROFILES = "/ra_functionality/edit_end_entity_profiles"
+REGULAR_EDITUSERDATASOURCES = "/ra_functionality/edit_user_data_sources"
+REGULAR_VIEWENDENTITY = "/ra_functionality/view_end_entity"
+REGULAR_CREATEENDENTITY = "/ra_functionality/create_end_entity"
+REGULAR_EDITENDENTITY = "/ra_functionality/edit_end_entity"
+REGULAR_DELETEENDENTITY = "/ra_functionality/delete_end_entity"
+REGULAR_REVOKEENDENTITY = "/ra_functionality/revoke_end_entity"
+REGULAR_VIEWENDENTITYHISTORY = "/ra_functionality/view_end_entity_history"
+REGULAR_APPROVEENDENTITY = "/ra_functionality/approve_end_entity"
+REGULAR_LOGFUNCTIONALITY = "/log_functionality"
+REGULAR_VIEWLOG = "/log_functionality/view_log"
+REGULAR_LOGCONFIGURATION = "/log_functionality/edit_log_configuration"
+REGULAR_LOG_CUSTOM_EVENTS = "/log_functionality/log_custom_events"
+REGULAR_SYSTEMFUNCTIONALITY = "/system_functionality"
+REGULAR_EDITADMINISTRATORPRIVILEDGES = "/system_functionality/edit_administrator_privileges"
+REGULAR_EDITSYSTEMCONFIGURATION = "/system_functionality/edit_systemconfiguration"
+REGULAR_VIEWHARDTOKENS = "/ra_functionality/view_hardtoken"
+REGULAR_VIEWPUKS = "/ra_functionality/view_hardtoken/puk_data"
+REGULAR_KEYRECOVERY = "/ra_functionality/keyrecovery"
+
+HARDTOKEN_HARDTOKENFUNCTIONALITY = "/hardtoken_functionality"
+HARDTOKEN_EDITHARDTOKENISSUERS = "/hardtoken_functionality/edit_hardtoken_issuers"
+HARDTOKEN_EDITHARDTOKENPROFILES = "/hardtoken_functionality/edit_hardtoken_profiles"
+HARDTOKEN_ISSUEHARDTOKENS = "/hardtoken_functionality/issue_hardtokens"
+HARDTOKEN_ISSUEHARDTOKENADMINISTRATORS = "/hardtoken_functionality/issue_hardtoken_administrators"
+
+RESPONSETYPE_CERTIFICATE = "CERTIFICATE"
+RESPONSETYPE_PKCS7 = "PKCS7"
+RESPONSETYPE_PKCS7WITHCHAIN = "PKCS7WITHCHAIN"
+
+NOT_REVOKED = -1
+REVOKATION_REASON_UNSPECIFIED = 0
+REVOKATION_REASON_KEYCOMPROMISE = 1
+REVOKATION_REASON_CACOMPROMISE = 2
+REVOKATION_REASON_AFFILIATIONCHANGED = 3
+REVOKATION_REASON_SUPERSEDED = 4
+REVOKATION_REASON_CESSATIONOFOPERATION = 5
+REVOKATION_REASON_CERTIFICATEHOLD = 6
+REVOKATION_REASON_REMOVEFROMCRL = 8
+REVOKATION_REASON_PRIVILEGESWITHDRAWN = 9
+REVOKATION_REASON_AACOMPROMISE = 10
+
+
+class HTTPSClientAuthHandler(get_https_handler()):
+
+    def __init__(self, key, cert):
+        get_https_handler().__init__(self)
+        self.key = key
+        self.cert = cert
+
+    def https_open(self, req):
+        return self.do_open(self.get_connection, req)
+
+    def get_connection(self, host, timeout=5):
+        if sys.version_info[0] >= 3:
+            return http.client.HTTPSConnection(host, key_file=self.key, cert_file=self.cert, timeout=timeout)
+        else:
+            return httplib.HTTPSConnection(host, key_file=self.key, cert_file=self.cert, timeout=timeout)
+
+
+class HTTPSClientCertTransport(suds.transport.http.HttpTransport):
+
+    def __init__(self, key, cert, *args, **kwargs):
+        suds.transport.http.HttpTransport.__init__(self, *args, **kwargs)
+        self.key = key
+        self.cert = cert
+
+    def u2open(self, u2request, timeout=None):
+        tm = timeout or self.options.timeout
+        if sys.version_info[0] >= 3:
+            url = urllib.request.build_opener(HTTPSClientAuthHandler(self.key, self.cert))
+        else:
+            url = urllib2.build_opener(HTTPSClientAuthHandler(self.key, self.cert))
+        if self.u2ver() < 2.6:
+            socket.setdefaulttimeout(tm)
+            return url.open(u2request)
+        else:
+            return url.open(u2request, timeout=tm)
+
+
+class Ejbca(object):
+
+    def __init__(self, url, cert=None, key=None):
+        self.url = url
+        self.cert = cert
+        self.key = key
+        self.transport = HTTPSClientCertTransport(self.key, self.cert) if self.cert else None
+        self.wsclient = suds.client.Client(self.url, transport=self.transport)
+
+
+    def get_version(self):
+        return self.wsclient.service.getEjbcaVersion()
+
+
+    def get_users(self):
+        return self.find_user(MATCH_WITH_DN, MATCH_TYPE_CONTAINS, "=")
+
+
+    def find_user(self, matchwith, matchtype, matchvalue):
+        usermatch = self.wsclient.factory.create('userMatch')
+        usermatch.matchwith = matchwith
+        usermatch.matchtype = matchtype
+        usermatch.matchvalue = matchvalue
+        return self.wsclient.service.findUser(usermatch)
+
+
+    def edit_user(self, user):
+        return self.wsclient.service.editUser(user)
+
+
+    def _decode_ejbca_cert(self, double_mess):
+        single_mess = base64.b64decode(double_mess)
+        cert_data = base64.b64decode(single_mess)
+        cert = M2Crypto.X509.load_cert_string(cert_data, M2Crypto.X509.FORMAT_DER)
+        return cert
+
+
+    def pkcs10_request(self, username, password, pkcs10, hardTokenSN, responseType):
+        res = self.wsclient.service.pkcs10Request(
+            arg0=username,
+            arg1=password,
+            arg2=pkcs10,
+            arg3=hardTokenSN,
+            arg4=responseType)
+        return self._decode_ejbca_cert(res["data"])
+
+
+    def find_certs(self, loginName, validOnly=False):
+        reslist = self.wsclient.service.findCerts(
+            arg0=loginName,
+            arg1=validOnly)
+        certs = []
+        for res in reslist:
+            double_mess = res["certificateData"]
+            if double_mess is not None:
+                cert = self._decode_ejbca_cert(double_mess)
+                cert.ejbca_status = res["type"]
+                certs.append(cert)
+        return certs

warden_ra/openssl.cnf.example

+# OpenSSL root CA configuration file.
+# Copy to `/root/ca/openssl.cnf`.
+
+[ ca ]
+# `man ca`
+default_ca = CA_default
+
+[ CA_default ]
+# Directory and file locations.
+dir               = /var/spool/example-ca
+certs             = $dir/certs
+crl_dir           = $dir/crl
+new_certs_dir     = $dir/newcerts
+database          = $dir/index.txt
+serial            = $dir/serial
+RANDFILE          = $dir/private/.rand
+unique_subject    = no
+
+# The root key and root certificate.
+private_key       = $dir/private/ca.key.pem
+certificate       = $dir/certs/ca.cert.pem
+
+# For certificate revocation lists.
+crlnumber         = $dir/crlnumber
+crl               = $dir/crl/ca.crl.pem
+crl_extensions    = crl_ext
+default_crl_days  = 30
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md        = sha256
+
+name_opt          = ca_default
+cert_opt          = ca_default
+default_days      = 375
+preserve          = no
+policy            = policy_loose
+
+[ policy_loose ]
+# Allow the CA to sign a more diverse range of certificates.
+# See the POLICY FORMAT section of the `ca` man page.
+countryName             = optional
+stateOrProvinceName     = optional
+localityName            = optional
+organizationName        = optional
+organizationalUnitName  = optional
+commonName              = supplied
+emailAddress            = optional
+
+[ req ]
+# Options for the `req` tool (`man req`).
+default_bits        = 2048
+distinguished_name  = req_distinguished_name
+string_mask         = utf8only
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md          = sha256
+
+# Extension to add when the -x509 option is used.
+x509_extensions     = v3_ca
+
+[ req_distinguished_name ]
+# See <https://en.wikipedia.org/wiki/Certificate_signing_request>.
+countryName                     = Country Name (2 letter code)
+stateOrProvinceName             = State or Province Name
+localityName                    = Locality Name
+0.organizationName              = Organization Name
+organizationalUnitName          = Organizational Unit Name
+commonName                      = Common Name
+emailAddress                    = Email Address
+
+# Optionally, specify some defaults.
+countryName_default             = CZ
+stateOrProvinceName_default     = Czech Republic
+localityName_default            =
+0.organizationName_default      = Example
+organizationalUnitName_default  =
+emailAddress_default            =
+
+[ v3_ca ]
+# Extensions for a typical CA (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+basicConstraints = critical, CA:true
+keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+
+[ server_cert ]
+# Extensions for server certificates (`man x509v3_config`).
+basicConstraints = CA:FALSE
+nsCertType = client
+nsComment = "OpenSSL Generate Client Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+keyUsage = critical, digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth
+