warden_ra/apache22.conf

+SSLEngine on
+
+SSLVerifyClient optional
+SSLOptions +StdEnvVars +ExportCertData
+
+SSLCertificateFile      /opt/warden_server/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/key.pem
+SSLCACertificateFile    /opt/warden_server/chain_TERENA_SSL_CA_3.pem
+
+WSGIScriptAlias /warden_ra /opt/warden-ra/warden_ra.wsgi
+
+<Directory /opt/warden-ra/warden_ra.wsgi>
+    Order allow,deny
+    Allow from all
+</Directory>

warden_ra/apache24.conf

+SSLEngine on
+
+SSLVerifyClient optional
+SSLOptions +StdEnvVars +ExportCertData
+
+SSLCertificateFile      /opt/warden_server/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/key.pem
+SSLCACertificateFile    /opt/warden_server/chain_TERENA_SSL_CA_3.pem
+
+WSGIScriptAlias /warden_ra /opt/warden-ra/warden_ra.wsgi
+
+<Directory /opt/warden-ra/warden_ra.wsgi>
+    Require all granted
+</Directory>

warden_ra/ejbcaws.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2016, CESNET, z. s. p. o.
+# Use of this source is governed by an ISC license, see LICENSE file.
+
+import sys
+import socket
+import base64
+import suds.transport.http
+import suds.client
+import M2Crypto
+
+
+if sys.version_info[0] >= 3:
+    import urllib.request, urllib.error, urllib.parse
+    import http.client
+
+    def get_https_handler():
+        return urllib.request.HTTPSHandler
+else:
+    import urllib2
+    import httplib
+
+    def get_https_handler():
+        return urllib2.HTTPSHandler
+
+
+STATUS_FAILED = 11
+STATUS_GENERATED = 40
+STATUS_HISTORICAL = 60
+STATUS_INITIALIZED = 20
+STATUS_INPROCESS = 30
+STATUS_KEYRECOVERY = 70
+STATUS_NEW = 10
+STATUS_REVOKED = 50
+
+MATCH_TYPE_BEGINSWITH = 1
+MATCH_TYPE_CONTAINS = 2
+MATCH_TYPE_EQUALS = 0
+MATCH_WITH_CA = 5
+MATCH_WITH_CERTIFICATEPROFILE = 4
+MATCH_WITH_COMMONNAME = 101
+MATCH_WITH_COUNTRY = 112
+MATCH_WITH_DIRECTORYNAME = 204
+MATCH_WITH_DN = 7
+MATCH_WITH_DNSERIALNUMBER = 102
+MATCH_WITH_DNSNAME = 201
+MATCH_WITH_DOMAINCOMPONENT = 111
+MATCH_WITH_EDIPARTNAME = 205
+MATCH_WITH_EMAIL = 1
+MATCH_WITH_ENDENTITYPROFILE = 3
+MATCH_WITH_GIVENNAME = 103
+MATCH_WITH_GUID = 209
+MATCH_WITH_INITIALS = 104
+MATCH_WITH_IPADDRESS = 202
+MATCH_WITH_LOCALE = 109
+MATCH_WITH_ORGANIZATION = 108
+MATCH_WITH_ORGANIZATIONUNIT = 107
+MATCH_WITH_REGISTEREDID = 207
+MATCH_WITH_RFC822NAME = 200
+MATCH_WITH_STATE = 110
+MATCH_WITH_STATUS = 2
+MATCH_WITH_SURNAME = 105
+MATCH_WITH_TITLE = 106
+MATCH_WITH_TOKEN = 6
+MATCH_WITH_UID = 100
+MATCH_WITH_UPN = 208
+MATCH_WITH_URI = 206
+MATCH_WITH_USERNAME = 0
+MATCH_WITH_X400ADDRESS = 203
+
+TOKEN_TYPE_JKS = "JKS"
+TOKEN_TYPE_P12 = "P12"
+TOKEN_TYPE_PEM = "PEM"
+TOKEN_TYPE_USERGENERATED = "USERGENERATED"
+
+VIEW_RIGHTS = "/view_end_entity"
+EDIT_RIGHTS = "/edit_end_entity"
+CREATE_RIGHTS = "/create_end_entity"
+DELETE_RIGHTS = "/delete_end_entity"
+REVOKE_RIGHTS = "/revoke_end_entity"
+HISTORY_RIGHTS = "/view_end_entity_history"
+APPROVAL_RIGHTS = "/approve_end_entity"
+HARDTOKEN_RIGHTS = "/view_hardtoken"
+HARDTOKEN_PUKDATA_RIGHTS = "/view_hardtoken/puk_data"
+KEYRECOVERY_RIGHTS = "/keyrecovery"
+
+ENDENTITYPROFILEBASE = "/endentityprofilesrules"
+ENDENTITYPROFILEPREFIX = "/endentityprofilesrules/"
+USERDATASOURCEBASE = "/userdatasourcesrules"
+USERDATASOURCEPREFIX = "/userdatasourcesrules/"
+UDS_FETCH_RIGHTS = "/fetch_userdata"
+UDS_REMOVE_RIGHTS = "/remove_userdata"
+
+CABASE = "/ca"
+CAPREFIX = "/ca/"
+ROLE_PUBLICWEBUSER = "/public_web_user"
+ROLE_ADMINISTRATOR = "/administrator"
+ROLE_SUPERADMINISTRATOR = "/super_administrator"
+REGULAR_CAFUNCTIONALTY = "/ca_functionality"
+REGULAR_CABASICFUNCTIONS = "/ca_functionality/basic_functions"
+REGULAR_ACTIVATECA = "/ca_functionality/basic_functions/activate_ca"
+REGULAR_RENEWCA = "/ca_functionality/renew_ca"
+REGULAR_VIEWCERTIFICATE = "/ca_functionality/view_certificate"
+REGULAR_APPROVECAACTION = "/ca_functionality/approve_caaction"
+REGULAR_CREATECRL = "/ca_functionality/create_crl"
+REGULAR_EDITCERTIFICATEPROFILES = "/ca_functionality/edit_certificate_profiles"
+REGULAR_CREATECERTIFICATE = "/ca_functionality/create_certificate"
+REGULAR_STORECERTIFICATE = "/ca_functionality/store_certificate"
+REGULAR_RAFUNCTIONALITY = "/ra_functionality"
+REGULAR_EDITENDENTITYPROFILES = "/ra_functionality/edit_end_entity_profiles"
+REGULAR_EDITUSERDATASOURCES = "/ra_functionality/edit_user_data_sources"
+REGULAR_VIEWENDENTITY = "/ra_functionality/view_end_entity"
+REGULAR_CREATEENDENTITY = "/ra_functionality/create_end_entity"
+REGULAR_EDITENDENTITY = "/ra_functionality/edit_end_entity"
+REGULAR_DELETEENDENTITY = "/ra_functionality/delete_end_entity"
+REGULAR_REVOKEENDENTITY = "/ra_functionality/revoke_end_entity"
+REGULAR_VIEWENDENTITYHISTORY = "/ra_functionality/view_end_entity_history"
+REGULAR_APPROVEENDENTITY = "/ra_functionality/approve_end_entity"
+REGULAR_LOGFUNCTIONALITY = "/log_functionality"
+REGULAR_VIEWLOG = "/log_functionality/view_log"
+REGULAR_LOGCONFIGURATION = "/log_functionality/edit_log_configuration"
+REGULAR_LOG_CUSTOM_EVENTS = "/log_functionality/log_custom_events"
+REGULAR_SYSTEMFUNCTIONALITY = "/system_functionality"
+REGULAR_EDITADMINISTRATORPRIVILEDGES = "/system_functionality/edit_administrator_privileges"
+REGULAR_EDITSYSTEMCONFIGURATION = "/system_functionality/edit_systemconfiguration"
+REGULAR_VIEWHARDTOKENS = "/ra_functionality/view_hardtoken"
+REGULAR_VIEWPUKS = "/ra_functionality/view_hardtoken/puk_data"
+REGULAR_KEYRECOVERY = "/ra_functionality/keyrecovery"
+
+HARDTOKEN_HARDTOKENFUNCTIONALITY = "/hardtoken_functionality"
+HARDTOKEN_EDITHARDTOKENISSUERS = "/hardtoken_functionality/edit_hardtoken_issuers"
+HARDTOKEN_EDITHARDTOKENPROFILES = "/hardtoken_functionality/edit_hardtoken_profiles"
+HARDTOKEN_ISSUEHARDTOKENS = "/hardtoken_functionality/issue_hardtokens"
+HARDTOKEN_ISSUEHARDTOKENADMINISTRATORS = "/hardtoken_functionality/issue_hardtoken_administrators"
+
+RESPONSETYPE_CERTIFICATE = "CERTIFICATE"
+RESPONSETYPE_PKCS7 = "PKCS7"
+RESPONSETYPE_PKCS7WITHCHAIN = "PKCS7WITHCHAIN"
+
+NOT_REVOKED = -1
+REVOKATION_REASON_UNSPECIFIED = 0
+REVOKATION_REASON_KEYCOMPROMISE = 1
+REVOKATION_REASON_CACOMPROMISE = 2
+REVOKATION_REASON_AFFILIATIONCHANGED = 3
+REVOKATION_REASON_SUPERSEDED = 4
+REVOKATION_REASON_CESSATIONOFOPERATION = 5
+REVOKATION_REASON_CERTIFICATEHOLD = 6
+REVOKATION_REASON_REMOVEFROMCRL = 8
+REVOKATION_REASON_PRIVILEGESWITHDRAWN = 9
+REVOKATION_REASON_AACOMPROMISE = 10
+
+
+class HTTPSClientAuthHandler(get_https_handler()):
+
+    def __init__(self, key, cert):
+        get_https_handler().__init__(self)
+        self.key = key
+        self.cert = cert
+
+    def https_open(self, req):
+        return self.do_open(self.get_connection, req)
+
+    def get_connection(self, host, timeout=5):
+        if sys.version_info[0] >= 3:
+            return http.client.HTTPSConnection(host, key_file=self.key, cert_file=self.cert, timeout=timeout)
+        else:
+            return httplib.HTTPSConnection(host, key_file=self.key, cert_file=self.cert, timeout=timeout)
+
+
+class HTTPSClientCertTransport(suds.transport.http.HttpTransport):
+
+    def __init__(self, key, cert, *args, **kwargs):
+        suds.transport.http.HttpTransport.__init__(self, *args, **kwargs)
+        self.key = key
+        self.cert = cert
+
+    def u2open(self, u2request, timeout=None):
+        tm = timeout or self.options.timeout
+        if sys.version_info[0] >= 3:
+            url = urllib.request.build_opener(HTTPSClientAuthHandler(self.key, self.cert))
+        else:
+            url = urllib2.build_opener(HTTPSClientAuthHandler(self.key, self.cert))
+        if self.u2ver() < 2.6:
+            socket.setdefaulttimeout(tm)
+            return url.open(u2request)
+        else:
+            return url.open(u2request, timeout=tm)
+
+
+class Ejbca(object):
+
+    def __init__(self, url, cert=None, key=None):
+        self.url = url
+        self.cert = cert
+        self.key = key
+        self.transport = HTTPSClientCertTransport(self.key, self.cert) if self.cert else None
+        self.wsclient = suds.client.Client(self.url, transport=self.transport)
+
+
+    def get_version(self):
+        return self.wsclient.service.getEjbcaVersion()
+
+
+    def get_users(self):
+        return self.find_user(MATCH_WITH_DN, MATCH_TYPE_CONTAINS, "=")
+
+
+    def find_user(self, matchwith, matchtype, matchvalue):
+        usermatch = self.wsclient.factory.create('userMatch')
+        usermatch.matchwith = matchwith
+        usermatch.matchtype = matchtype
+        usermatch.matchvalue = matchvalue
+        return self.wsclient.service.findUser(usermatch)
+
+
+    def edit_user(self, user):
+        return self.wsclient.service.editUser(user)
+
+
+    def _decode_ejbca_cert(self, double_mess):
+        single_mess = base64.b64decode(double_mess)
+        cert_data = base64.b64decode(single_mess)
+        cert = M2Crypto.X509.load_cert_string(cert_data, M2Crypto.X509.FORMAT_DER)
+        return cert
+
+
+    def pkcs10_request(self, username, password, pkcs10, hardTokenSN, responseType):
+        res = self.wsclient.service.pkcs10Request(
+            arg0=username,
+            arg1=password,
+            arg2=pkcs10,
+            arg3=hardTokenSN,
+            arg4=responseType)
+        return self._decode_ejbca_cert(res["data"])
+
+
+    def find_certs(self, loginName, validOnly=False):
+        reslist = self.wsclient.service.findCerts(
+            arg0=loginName,
+            arg1=validOnly)
+        certs = []
+        for res in reslist:
+            double_mess = res["certificateData"]
+            if double_mess is not None:
+                cert = self._decode_ejbca_cert(double_mess)
+                cert.ejbca_status = res["type"]
+                certs.append(cert)
+        return certs

warden_ra/openssl.cnf.example

+# OpenSSL root CA configuration file.
+# Copy to `/root/ca/openssl.cnf`.
+
+[ ca ]
+# `man ca`
+default_ca = CA_default
+
+[ CA_default ]
+# Directory and file locations.
+dir               = /var/spool/example-ca
+certs             = $dir/certs
+crl_dir           = $dir/crl
+new_certs_dir     = $dir/newcerts
+database          = $dir/index.txt
+serial            = $dir/serial
+RANDFILE          = $dir/private/.rand
+unique_subject    = no
+
+# The root key and root certificate.
+private_key       = $dir/private/ca.key.pem
+certificate       = $dir/certs/ca.cert.pem
+
+# For certificate revocation lists.
+crlnumber         = $dir/crlnumber
+crl               = $dir/crl/ca.crl.pem
+crl_extensions    = crl_ext
+default_crl_days  = 30
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md        = sha256
+
+name_opt          = ca_default
+cert_opt          = ca_default
+default_days      = 375
+preserve          = no
+policy            = policy_loose
+
+[ policy_loose ]
+# Allow the CA to sign a more diverse range of certificates.
+# See the POLICY FORMAT section of the `ca` man page.
+countryName             = optional
+stateOrProvinceName     = optional
+localityName            = optional
+organizationName        = optional
+organizationalUnitName  = optional
+commonName              = supplied
+emailAddress            = optional
+
+[ req ]
+# Options for the `req` tool (`man req`).
+default_bits        = 2048
+distinguished_name  = req_distinguished_name
+string_mask         = utf8only
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md          = sha256
+
+# Extension to add when the -x509 option is used.
+x509_extensions     = v3_ca
+
+[ req_distinguished_name ]
+# See <https://en.wikipedia.org/wiki/Certificate_signing_request>.
+countryName                     = Country Name (2 letter code)
+stateOrProvinceName             = State or Province Name
+localityName                    = Locality Name
+0.organizationName              = Organization Name
+organizationalUnitName          = Organizational Unit Name
+commonName                      = Common Name
+emailAddress                    = Email Address
+
+# Optionally, specify some defaults.
+countryName_default             = CZ
+stateOrProvinceName_default     = Czech Republic
+localityName_default            =
+0.organizationName_default      = Example
+organizationalUnitName_default  =
+emailAddress_default            =
+
+[ v3_ca ]
+# Extensions for a typical CA (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+basicConstraints = critical, CA:true
+keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+
+[ server_cert ]
+# Extensions for server certificates (`man x509v3_config`).
+basicConstraints = CA:FALSE
+nsCertType = client
+nsComment = "OpenSSL Generate Client Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+keyUsage = critical, digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth
+

warden_ra/warden_apply.sh

+#!/bin/bash
+
+key=key.pem
+csr=csr.pem
+cert=cert.pem
+result=${TMPDIR:-${TMP:-/tmp}}/cert.$$.$RANDOM
+config=${TMPDIR:-${TMP:-/tmp}}/conf.$$.$RANDOM
+if [ "$1" == "--cacert" ]; then
+  cacert="--cacert $2"
+  shift
+  shift
+fi
+url="$1"
+client="$2"
+password="$3"
+incert="$3"
+inkey="$4"
+
+trap 'rm -f "$config $result"' INT TERM HUP EXIT
+
+function flee { echo -e "$1"; exit $2; }
+
+[ -z "$client" -o -z "$password" ] && flee "Usage: ${0%.*} [--cacert CERT] url client.name password\n       ${0%.*} [--cacert CERT] url client.name cert_file key_file" 255
+
+url="${url%/}/getCert"
+
+for n in openssl curl; do
+    command -v "$n" 2>&1 >/dev/null || flee "Haven't found $n binary." 251
+done
+for n in "$csr" "$key" "$cert"; do
+    [ -e "$n" ] && flee "$n already exists, I won't overwrite, move them away first, please." 254
+done
+for n in "$result" "$config"; do
+    touch "$n" || flee "Error creating temporary file ($n)." 253
+done
+
+echo -e "default_bits=2048\ndistinguished_name=rdn\nprompt=no\n[rdn]\ncommonName=dummy" > "$config"
+
+openssl req -new -nodes -batch -keyout "$key" -out "$csr" -config "$config" || flee "Error generating key/certificate request." 252
+
+if [ -z "$inkey" ]; then
+    curl --progress-bar $cacert --request POST --data-binary '@-' "$url?name=$client&password=$password" < "$csr" > "$result"
+else
+    # local cert file name may be interpreted as a "nickname", add "./" to force interpretation as a file
+    if [[ ! "$incert" =~ "/" ]]; then
+        incert="./$incert"
+    fi
+    curl --progress-bar $cacert --request POST --data-binary '@-' --cert "$incert" --key "$inkey" "$url?name=$client" < "$csr" > "$result"
+fi
+
+case $(<$result) in '-----BEGIN CERTIFICATE-----'*)
+    mv "$result" "$cert"
+    flee "Succesfully generated key ($key) and obtained certificate ($cert)." 0
+esac
+
+flee "$(<$result)\n\nCertificate request failed. Please save all error messages for communication with registration authority representative." 252

warden_ra/warden_ra.cfg.dist

+{
+    "Log": {
+        "filename": "/var/log/warden_ra.log",
+        "level": "info"
+    },
+    "Registry": {
+
+// Example configuration for OpenSSL CA backend
+//        "type": "OpenSSLRegistry",
+//        "base_dir": "/var/spool/example-ca",
+//        "subject_dn_template": "DC=cz,DC=example-ca,DC=warden,CN=%s"
+
+// Example configuration for EJBCA backend
+//        "type": "EjbcaRegistry",
+//        "url": "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl",
+//        "cert": "warden_ra.cert.pem",
+//        "key": "warden_ra.key.pem",
+//        "ca_name": "Example CA",
+//        "certificate_profile_name": "Example",
+//        "end_entity_profile_name": "Example EE",
+//        "subject_dn_template": "DC=cz,DC=example-ca,DC=warden,CN=%s",
+//        "username_suffix": "@warden"
+
+    }
+}

warden_ra/warden_ra.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2016, CESNET, z. s. p. o.
+# Use of this source is governed by an ISC license, see LICENSE file.
+
+from __future__ import print_function
+
+import sys
+import os
+import time
+import fcntl
+import errno
+import string
+import random
+import struct
+import operator
+import argparse
+import json
+import logging
+import os.path as pth
+import subprocess
+import shlex
+import tempfile
+import M2Crypto
+
+import ejbcaws
+
+if sys.version_info[0] >= 3:
+    import configparser as ConfigParser
+else:
+    import ConfigParser
+
+# usual path to warden server
+sys.path.append(pth.join(pth.dirname(__file__), "..", "warden_server"))
+import warden_server
+from warden_server import Request, ObjectBase, FileLogger, SysLogger, Server, expose, read_cfg
+
+VERSION = "3.0-beta3"
+
+class ClientDisabledError(Exception): pass
+class ClientNotIssuableError(Exception): pass
+class AuthenticationError(Exception): pass
+class PopenError(Exception): pass
+
+
+class Client(object):
+
+    def __init__(self, name, admins=None, status=None, pwd=None, opaque=None):
+        self.name = name
+        self.admins = admins or []
+        self.status = status or "New"
+        self.pwd = pwd
+        self.opaque = opaque or {}
+
+    def update(self, admins=None, status=None, pwd=None):
+        if admins is not None:
+            self.admins = admins
+        if status:
+            if self.status == "Disabled" and status not in ("Passive", "Disabled"):
+                raise ClientDisabledError("This client is disabled")
+            self.status = status
+        self.pwd = pwd if status=="Issuable" and pwd else None
+
+    def __str__(self):
+        return (
+            "Client:   %s\n"
+            "Admins:   %s\n"
+            "Status:   %s\n"
+        ) % (self.name, ", ".join(self.admins), self.status)
+
+    def str(self, verbose=False):
+        return str(self) + (str(self.opaque) if self.opaque and verbose else "")
+
+
+class OpenSSLRegistry(object):
+
+    def __init__(self, log, base_dir,
+                 subject_dn_template, openssl_sign, lock_timeout):
+        self.base_dir = base_dir
+        self.cnf_file = pth.join(base_dir, "openssl.cnf")
+        self.client_dir = pth.join(base_dir, "clients")
+        self.serial_file = pth.join(base_dir, "serial")
+        self.newcerts_dir = pth.join(base_dir, "newcerts")
+        self.csr_dir = pth.join(base_dir, "csr")
+        self.lock_file = pth.join(base_dir, "lock")
+        self.lock_timeout = lock_timeout
+        self.log = log
+        self.subject_dn_template = subject_dn_template
+        self.openssl_sign = openssl_sign
+        os.umask(0o0002)    # read privilege for usual apache group
+
+    def get_clients(self):
+        return [self.get_client(c) for c in os.listdir(self.client_dir) if pth.isdir(pth.join(self.client_dir, c))]
+
+    def get_client(self, name):
+        config = ConfigParser.RawConfigParser()
+        try:
+            with open(pth.join(self.client_dir, name, "state")) as cf:
+                config.readfp(cf)
+        except IOError as e:
+            if e.errno == errno.ENOENT:
+                return None
+            raise
+        datum = dict(config.items("Client"))
+        return Client(name, admins=datum["admins"].split(","), status=datum["status"], pwd=datum.get("password"))
+
+    def new_client(self, name, admins=None):
+        user = self.get_client(name)
+        if user:
+            raise LookupError("Client %s already exists" % name)
+        return Client(name, admins)
+
+    def save_client(self, client):
+        config = ConfigParser.RawConfigParser()
+        config.add_section("Client")
+        config.set("Client", "admins", ",".join(client.admins))
+        config.set("Client", "status", client.status)
+        if client.pwd:
+            config.set("Client", "password", client.pwd)
+        client_path = pth.join(self.client_dir, client.name)
+        try:
+            os.makedirs(client_path)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        with tempfile.NamedTemporaryFile(dir=client_path, delete=False, mode="w") as cf:
+            config.write(cf)
+        os.chmod(cf.name, 0o660)    # read privilege for usual apache group
+        os.rename(cf.name, pth.join(client_path, "state")) # atomic + rewrite, so no need for locking
+
+    def get_certs(self, client):
+        files = [fname for fname in os.listdir(pth.join(self.client_dir, client.name)) if not fname.startswith(".") and fname.endswith(".pem")]
+        certs = [M2Crypto.X509.load_cert(pth.join(self.client_dir, client.name, fname)) for fname in files]
+        return certs
+
+    def __enter__(self):
+        self._lockfd = os.open(self.lock_file, os.O_CREAT)
+        start = time.time()
+        while True:
+            try:
+                fcntl.flock(self._lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                return
+            except (OSError, IOError) as e:
+                if e.errno != errno.EAGAIN or time.time() > start + self.lock_timeout:
+                   raise
+            time.sleep(0.5)
+
+    def __exit__(self, type_, value, traceback):
+        fcntl.flock(self._lockfd, fcntl.LOCK_UN)
+        os.close(self._lockfd)
+        try:
+            os.unlink(self.lock_file)
+        except:
+            pass
+
+    def run_openssl(self, command, **kwargs):
+        cmdline = shlex.split(command % kwargs)
+        process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        res = process.communicate()
+        if process.returncode:
+            raise PopenError("Popen returned nonzero code", process.returncode, ' '.join(cmdline), res[0], res[1])
+        return res
+
+    def new_cert(self, client, csr, pwd):
+        if client.status != "Issuable" or not client.pwd:
+            raise ClientNotIssuableError("Client not allowed to issue request or password not set")
+        if client.pwd != pwd:
+            raise AuthenticationError("Wrong credentials")
+        dn = self.subject_dn_template.replace("/", "//").replace(",", "/") % client.name
+        if not dn.startswith("/"):
+            dn = "/" + dn
+        with tempfile.NamedTemporaryFile(dir=self.csr_dir, delete=False) as csr_file:
+            csr_file.write(csr)
+        with self:  # lock dance
+            with open(self.serial_file) as f:
+                serial = f.read().strip()
+            output = self.run_openssl(self.openssl_sign, cnf = self.cnf_file, csr = csr_file.name, dn = dn)
+        self.log.debug(output)
+        os.rename(csr_file.name, pth.join(self.csr_dir, serial + ".csr.pem"))
+        client_pem_name = pth.join(self.client_dir, client.name, serial + ".cert.pem")
+        os.symlink(pth.join(self.newcerts_dir, serial + ".pem"), client_pem_name)
+        with open(client_pem_name) as pem:
+            cert = M2Crypto.X509.load_cert_string(pem.read(), M2Crypto.X509.FORMAT_PEM)
+        client.update(status="Passive", pwd=None)
+        self.save_client(client)
+        return cert
+
+    def __str__(self):
+        return "%s<%s>" % (type(self).__name__, self.base_dir)
+
+
+class EjbcaRegistry(OpenSSLRegistry):
+
+    status_ejbca_to_str = {
+        ejbcaws.STATUS_NEW: "Issuable",
+        ejbcaws.STATUS_GENERATED: "Passive",
+        ejbcaws.STATUS_INITIALIZED: "New",
+        ejbcaws.STATUS_HISTORICAL: "Disabled"
+    }
+    status_str_to_ejbca = dict((v, k) for k, v in status_ejbca_to_str.items())
+
+    def __init__(self, log, url, cert=None, key=None,
+                 ca_name="", certificate_profile_name="", end_entity_profile_name="",
+                 subject_dn_template="%s", username_suffix=""):
+        self.log = log
+        self.ejbca = ejbcaws.Ejbca(url, cert, key)
+        self.ca_name = ca_name
+        self.certificate_profile_name = certificate_profile_name
+        self.end_entity_profile_name = end_entity_profile_name
+        self.subject_dn_template = subject_dn_template
+        self.username_suffix = username_suffix
+
+    def client_data(self, ejbca_data):
+        ejbca_username = ejbca_data["username"]
+        username = ejbca_username[:-len(self.username_suffix)] if ejbca_username.endswith(self.username_suffix) else ejbca_username
+        try:
+            alt_name = ejbca_data["subjectAltName"]
+        except KeyError:
+            alt_name = None
+        if alt_name:
+            admins = [u if not u.startswith("RFC822NAME") else u[11:] for u in alt_name.split(",")]
+        else:
+            admins = []
+        status = self.status_ejbca_to_str.get(ejbca_data["status"], "Other")
+        return username, admins, status, None, ejbca_data
+
+    def get_clients(self):
+        return [Client(*self.client_data(u)) for u in self.ejbca.get_users()]
+
+    def get_client(self, name):
+        users = self.ejbca.find_user(ejbcaws.MATCH_WITH_USERNAME, ejbcaws.MATCH_TYPE_EQUALS, name + self.username_suffix)
+        if len(users) > 1:
+            raise LookupError("%d users %s found (more than one?!)" % (len(users), name))
+        if not users:
+            return None
+        return Client(*self.client_data(users[0]))
+
+    def save_client(self, client):
+        edata = client.opaque or dict(
+            caName=self.ca_name,
+            certificateProfileName=self.certificate_profile_name,
+            endEntityProfileName=self.end_entity_profile_name,
+            keyRecoverable=False,
+            sendNotification=False,
+            tokenType=ejbcaws.TOKEN_TYPE_USERGENERATED,
+            password = "".join((random.choice(string.ascii_letters + string.digits) for dummy in range(16))),
+            clearPwd = True,
+            username = client.name + self.username_suffix,
+            subjectDN = self.subject_dn_template % client.name
+        )
+        edata["subjectAltName"] = ",".join(("RFC822NAME=%s" % a for a in client.admins))
+        try:
+            edata["status"] = self.status_str_to_ejbca.get(client.status)
+        except KeyError:
+            # Unknown status - either came from EJBCA and translated to
+            # "Other", or something wrong came in later. Let's just
+            # keep original EJBCA status unchanged.
+            pass
+        if client.pwd:
+            edata["password"] = client.pwd
+            edata["clearPwd"] = True
+        self.ejbca.edit_user(edata)
+
+    def get_certs(self, client):
+        return self.ejbca.find_certs(client.opaque["username"], validOnly=False)
+
+    def new_cert(self, client, csr, pwd):
+        cert = self.ejbca.pkcs10_request(
+            client.opaque["username"],
+            pwd, csr, 0, ejbcaws.RESPONSETYPE_CERTIFICATE)
+        return cert
+
+    def __str__(self):
+        return self.ejbca.get_version()
+
+
+def relaxed_ord(c):
+    # Compatibility wrapper for py2/py3
+    try:
+        return ord(c)
+    except TypeError:
+        return c
+
+
+def format_cert(cert):
+    return (
+        "Subject:     %s\n"
+        "Validity:    %s - %s\n"
+        "Serial:      %s\n"
+        "Fingerprint: md5:%s, sha1:%s\n"
+        "Issuer:      %s\n"
+    ) % (
+        cert.get_subject().as_text(),
+        cert.get_not_before().get_datetime().isoformat(),
+        cert.get_not_after().get_datetime().isoformat(),
+        ":".join(["%02x" % relaxed_ord(c) for c in struct.pack('!Q', cert.get_serial_number())]),
+        cert.get_fingerprint("md5"),
+        cert.get_fingerprint("sha1"),
+        cert.get_issuer().as_text()
+    )
+
+
+# Server side
+
+class OptionalAuthenticator(ObjectBase):
+
+    def __init__(self, req, log):
+        ObjectBase.__init__(self, req, log)
+
+
+    def __str__(self):
+        return "%s(req=%s)" % (type(self).__name__, type(self.req).__name__)
+
+
+    def authenticate(self, env, args):
+        cert_name = env.get("SSL_CLIENT_S_DN_CN")
+
+        if cert_name:
+            if cert_name != args.setdefault("name", [cert_name])[0]:
+                exception = self.req.error(message="authenticate: client name does not correspond with certificate", error=403, cn = cert_name, args = args)
+                exception.log(self.log)
+                return None
+
+            verify = env.get("SSL_CLIENT_VERIFY")
+            if verify != "SUCCESS":
+                exception = self.req.error(message="authenticate: certificate present but verification failed", error=403, cn = cert_name, args = args, verify=verify)
+                exception.log(self.log)
+                return None
+
+            return "cert"   # Ok, client authorized by valid certificate
+
+        else:
+            try:
+                args["password"][0]
+                return "pwd"    # Ok, pass on, but getCert will have to rely on certificate registry password
+            except (KeyError, IndexError):
+                exception = self.req.error(message="authenticate: no certificate nor password present", error=403, cn = cert_name, args = args)
+                exception.log(self.log)
+                return None
+
+
+    def authorize(self, env, client, path, method):
+        return True
+
+
+class CertHandler(ObjectBase):
+
+    def __init__(self, req, log, registry):
+        ObjectBase.__init__(self, req, log)
+        self.registry = registry
+
+    @expose(read=1, debug=1)
+    def getCert(self, csr_data=None, name=None, password=None):
+        if not (name and csr_data):
+            raise self.req.error(message="Wrong or missing arguments", error=400, name=name, password=password)
+        client = self.registry.get_client(name[0])
+        if not client:
+            raise self.req.error(message="Unknown client", error=403, name=name, password=password)
+        self.log.info("Client %s" % client)
+        if self.req.client == "cert":
+            # Correctly authenticated by cert, most probably not preactivated with password,
+            # so generate oneshot password and allow now
+            password = ["".join((random.choice(string.ascii_letters + string.digits) for dummy in range(16)))]
+            self.log.debug("Authorized by X509, enabling cert generation with password %s" % password)
+            try:
+                client.update(status="Issuable", pwd=password[0])
+                self.registry.save_client(client)
+            except ClientDisabledError as e:
+                raise self.req.error(message="Error enabling cert generation", error=403, exc=sys.exc_info())
+        if not password:
+            raise self.req.error(message="Missing password and certificate validation failed", error=403, name=name, password=password)
+        try:
+            newcert = self.registry.new_cert(client, csr_data.decode('latin1'), password[0])
+        except Exception as e:
+            raise self.req.error(message="Processing error", error=403, exc=sys.exc_info())
+        self.log.info("Generated.")
+        return [("Content-Type", "application/x-pem-file")], newcert.as_pem()
+
+
+# Order in which the base objects must get initialized
+section_order = ("log", "auth", "registry", "handler", "server")
+
+# List of sections and objects, configured by them
+# First object in each object list is the default one, otherwise
+# "type" keyword in section may be used to choose other
+section_def = {
+    "log": [FileLogger, SysLogger],
+    "auth": [OptionalAuthenticator],
+    "registry": [OpenSSLRegistry, EjbcaRegistry],
+    "handler": [CertHandler],
+    "server": [Server]
+}
+
+# Object parameter conversions and defaults
+param_def = {
+    FileLogger: warden_server.param_def[FileLogger],
+    SysLogger: warden_server.param_def[SysLogger],
+    Server: warden_server.param_def[Server],
+    OptionalAuthenticator: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"}
+    },
+    OpenSSLRegistry: {
+        "log":  {"type": "obj", "default": "log"},
+        "base_dir": {"type": "str", "default": pth.join(pth.dirname(__file__), "ca")},
+        "subject_dn_template": {"type": "str", "default": "DC=cz,DC=example-ca,DC=warden,CN=%s"},
+        "openssl_sign": {"type": "str", "default": "openssl ca -config %(cnf)s -batch -extensions server_cert -days 375 -notext -md sha256 -in %(csr)s -subj '%(dn)s'"},
+        "lock_timeout": {"type": "natural", "default": "3"}
+    },
+    EjbcaRegistry: {
+        "log":  {"type": "obj", "default": "log"},
+        "url": {"type": "str", "default": "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl"},
+        "cert": {"type": "filepath", "default": pth.join(pth.dirname(__file__), "warden_ra.cert.pem")},
+        "key": {"type": "filepath", "default": pth.join(pth.dirname(__file__), "warden_ra.key.pem")},
+        "ca_name": {"type": "str", "default": "Example CA"},
+        "certificate_profile_name": {"type": "str", "default": "Example"},
+        "end_entity_profile_name": {"type": "str", "default": "Example EE"},
+        "subject_dn_template": {"type": "str", "default": "DC=cz,DC=example-ca,DC=warden,CN=%s"},
+        "username_suffix": {"type": "str", "default": "@warden"}
+    },
+    CertHandler: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "registry": {"type": "obj", "default": "registry"}
+    }
+}
+
+param_def[FileLogger]["filename"] = {"type": "filepath", "default": pth.join(pth.dirname(__file__), pth.splitext(pth.split(__file__)[1])[0] + ".log")}
+
+
+def build_server(conf):
+    return warden_server.build_server(conf, section_order, section_def, param_def)
+
+
+# Command line
+
+def list_clients(registry, name=None, verbose=False, show_cert=True):
+    if name is not None:
+        client = registry.get_client(name)
+        if client is None:
+            print("No such client.")
+            return
+        else:
+            print(client.str(verbose))
+            if show_cert:
+                for cert in sorted(registry.get_certs(client), key=lambda c: c.get_not_after().get_datetime()):
+                    print(format_cert(cert))
+                    if verbose:
+                        print(cert.as_text())
+    else:
+        clients = registry.get_clients()
+        for client in sorted(clients, key=operator.attrgetter("name")):
+            print(client.str(verbose))
+
+
+def register_client(registry, name, admins=None, verbose=False):
+    try:
+        client = registry.new_client(name, admins)
+    except LookupError as e:
+        print(e)
+        return
+    registry.save_client(client)
+    list_clients(registry, name, verbose, show_cert=False)
+
+
+def applicant(registry, name, password=None, verbose=False):
+    client = registry.get_client(name)
+    if not client:
+        print("No such client.")
+        return
+    if password is None:
+        password = "".join((random.choice(string.ascii_letters + string.digits) for dummy in range(16)))
+    try:
+        client.update(status="Issuable", pwd=password)
+    except ClientDisabledError:
+        print("This client is disabled. Use 'enable' first.")
+        return
+    registry.save_client(client)
+    list_clients(registry, name, verbose, show_cert=False)
+    print("Application password is: %s\n" % password)
+
+
+def enable(registry, name, verbose=False):
+    client = registry.get_client(name)
+    if not client:
+        print("No such client.")
+        return
+    client.update(status="Passive")
+    registry.save_client(client)
+    list_clients(registry, name, verbose, show_cert=False)
+
+
+def disable(registry, name, verbose=False):
+    client = registry.get_client(name)
+    if not client:
+        print("No such client.")
+        return
+    client.update(status="Disabled")
+    registry.save_client(client)
+    list_clients(registry, name, verbose, show_cert=False)
+
+
+def request(registry, key, csr, verbose=False):
+    openssl = subprocess.Popen(
+        [
+            "openssl", "req", "-new", "-nodes", "-batch",
+            "-keyout", key,
+            "-out", csr,
+            "-config", "/dev/stdin"
+        ], stdin=subprocess.PIPE
+    )
+    openssl.stdin.write(
+        "distinguished_name=req_distinguished_name\n"
+        "prompt=no\n"
+        "\n"
+        "[req_distinguished_name]\n"
+        "commonName=dummy".encode("ascii")
+    )
+    openssl.stdin.close()
+    openssl.wait()
+    if verbose:
+        with open(csr, "r") as f:
+            print(f.read())
+
+
+def gen_cert(registry, name, csr, cert, password, verbose=False):
+    with open(csr, "r") as f:
+        csr_data = f.read()
+    client = registry.get_client(name)
+    newcert = registry.new_cert(client, csr_data, password)
+    print(format_cert(newcert))
+    if verbose:
+        print(newcert.as_text())
+        print(newcert.as_pem())
+    with open(cert, "w") as f:
+        f.write(newcert.as_text())
+        f.write(newcert.as_pem().decode("ascii"))
+
+
+def get_args():
+    argp = argparse.ArgumentParser(
+        description="Warden server certificate registry", add_help=False)
+    argp.add_argument("--help", action="help",
+        help="show this help message and exit")
+    argp.add_argument("-c", "--config",
+        help="path to configuration file")
+    argp.add_argument("-v", "--verbose", action="store_true", default=False,
+        help="be more chatty")
+    subargp = argp.add_subparsers(title="commands")
+
+    subargp_list = subargp.add_parser("list", add_help=False,
+        description="List registered clients.",
+        help="list clients")
+    subargp_list.set_defaults(command=list_clients)
+    subargp_list.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_list.add_argument("--name", action="store", type=str,
+        help="client name")
+
+    subargp_reg = subargp.add_parser("register", add_help=False,
+        description="Add client registration entry.",
+        help="register client")
+    subargp_reg.set_defaults(command=register_client)
+    subargp_reg.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_reg.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+    subargp_reg.add_argument("--admins", action="store", type=str,
+        required=True, nargs="*", help="administrator list")
+
+    subargp_apply = subargp.add_parser("applicant", add_help=False,
+        description="Set client into certificate application mode and set its password",
+        help="allow for certificate application")
+    subargp_apply.set_defaults(command=applicant)
+    subargp_apply.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_apply.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+    subargp_apply.add_argument("--password", action="store", type=str,
+        help="password for application (will be autogenerated if not set)")
+
+    subargp_enable = subargp.add_parser("enable", add_help=False,
+        description="Enable this client",
+        help="enable this client")
+    subargp_enable.set_defaults(command=enable)
+    subargp_enable.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_enable.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+
+    subargp_disable = subargp.add_parser("disable", add_help=False,
+        description="Disable this client",
+        help="disable this client (no more applications until enabled again)")
+    subargp_disable.set_defaults(command=disable)
+    subargp_disable.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_disable.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+
+    subargp_req = subargp.add_parser("request", add_help=False,
+        description="Generate certificate request",
+        help="generate CSR")
+    subargp_req.set_defaults(command=request)
+    subargp_req.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_req.add_argument("--key", action="store", type=str,
+        required=True, help="file for saving the key")
+    subargp_req.add_argument("--csr", action="store", type=str,
+        required=True, help="file for saving the request")
+
+    subargp_cert = subargp.add_parser("gencert", add_help=False,
+        description="Request new certificate from registry",
+        help="get new certificate")
+    subargp_cert.set_defaults(command=gen_cert)
+    subargp_cert.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_cert.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+    subargp_cert.add_argument("--csr", action="store", type=str,
+        required=True, help="file for saving the request")
+    subargp_cert.add_argument("--cert", action="store", type=str,
+        required=True, help="file for saving the new certificate")
+    subargp_cert.add_argument("--password", action="store", type=str,
+        required=True, help="password for application")
+
+    return argp.parse_args()
+
+
+if __name__ == "__main__":
+    args = get_args()
+    config = pth.join(pth.dirname(__file__), args.config or "warden_ra.cfg")
+    server = build_server(read_cfg(config))
+    registry = server.handler.registry
+    if args.verbose:
+        print(registry)
+    command = args.command
+    subargs = vars(args)
+    del subargs["command"]
+    del subargs["config"]
+    sys.exit(command(registry, **subargs))

warden_ra/warden_ra.wsgi

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from sys import path
+from os.path import dirname, join
+
+path.append(dirname(__file__))
+from warden_ra import build_server
+
+## JSON configuration with line comments (trailing #)
+from warden_ra import read_cfg
+application = build_server(read_cfg(join(dirname(__file__), "warden_ra.cfg")))

warden_server/LICENSE

+BSD License
+
+Copyright © 2011-2015 Cesnet z.s.p.o
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of the Cesnet z.s.p.o nor the names of its
+      contributors may be used to endorse or promote products derived from this
+      software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE Cesnet z.s.p.o BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

warden_server/README

++-------------------------+
+| Warden Server 3.0-beta3 |
++-------------------------+
+
+Content
+
+  A. Introduction
+  B. Dependencies
+  C. Installation
+  D. Configuration
+  E. Command line
+
+------------------------------------------------------------------------------
+A. Introduction
+
+   Warden is a system for efficient sharing of information about detected
+events (threats). Warden Server is server-side part of the software, the
+communication hub, allowing to publish detected events and download yet
+unprocessed ones.
+
+   The events are exchanged in Idea, flexible and descriptive event JSON
+serialized format. Warden 3 protocol is based on plain HTTPS queries with
+help of JSON, so the clients can be thin and simple.
+
+   For description of API see Warden Client, this document describes basics
+of Warden Server administration.
+
+   Warden Server is Python/WSGI based, written primarily with Apache mod_wsgi
+in mind. Other WSGI servers/frameworks are not yet tested, so your mileage
+may vary. Authentication is X509 certificate (for machine or client
+identification) + shared secret (for client identification, where
+certificate does not suffice).
+
+------------------------------------------------------------------------------
+B. Dependencies
+
+ 1. Platform
+
+    Python 2.7+
+	Apache 2.2/2.4
+	mod_wsgi 3.3+
+
+ 2. Python modules
+
+    python-mysqldb 5.3.3+
+    python-m2crypto 0.20+
+    jsonschema 2.4+
+
+ 3. Database
+
+    MySQL | MariaDB >= 5.5
+
+------------------------------------------------------------------------------
+C. Installation
+
+ * Untar distribution archive into required destination
+   (We're using /opt as an example.)
+
+   # cd /opt
+   # tar xjf warden_server_3.0.tar.bz2
+   # mv warden_server_3.0 warden_server
+
+ * Create database and desired database users
+   (We're using db "warden3" and user "warden@localhost" as an example.)
+
+   # mysql -p
+
+   > CREATE DATABASE warden3;
+   > CREATE USER 'warden'@'localhost' IDENTIFIED BY 'example';
+   > GRANT ALL ON warden3.* TO `warden`@`localhost`;
+   > FLUSH PRIVILEGES;
+
+ * Create necessary table structure
+
+   mysql -p -u warden warden3 < warden_3.0.sql
+
+ * Get up to date Idea schema
+
+   wget -O warden_server/idea.schema https://idea.cesnet.cz/_media/en/idea0.schema
+
+ * Enable mod_wsgi, mod_ssl, include Warden configuration
+
+   This depends heavily on your distribution and Apache configuration.
+   Basically you need to create and include apache.conf:
+
+      Include /opt/warden_server/apache.conf
+
+   or paste the contents into whichever Directory, Location or VirtualHost
+   you dedicate for Warden.  You can use apache22.conf.dist or
+   apache24.conf.dist (for Apache version 2.2 or 2.4, respectively) as an
+   example.
+
+   You may need to change paths to certificate/key/ca material, path to
+   warden_server.wsgi and web path alias.
+
+   Also note that warden_server itself is NOT thread safe, so included
+   configuration expects Apache with mpm-prefork worker, or you will have
+   to configure mod_wsgi as separate process with threads=1.
+
+   Also, for warden_server.wsgi, you can use warden_server.wsgi.dist as
+   a template. You will possibly need to change at least configuration
+   file path.
+
+ * Configure Warden server (see next chapter)
+
+ * Reload Apache
+
+------------------------------------------------------------------------------
+D. Configuration
+
+   Configuration is JSON object in file (warden_server.cfg by default),
+however, lines starting with "#" or "//" are allowed and will be ignored as
+comments. File must contain valid JSON object, containing configuration. See
+also warden_server.cfg.dist as example.
+
+   Top level JSON object contains subsections, which configure particular
+aspect of the server. Each subsection can contain "type" key, which chooses
+particular implementation object of the aspect, for example type of logger
+(file, syslog), such as:
+
+   {
+      "Log": {
+         "type": "SysLogger",
+         ...
+      },
+      "DB": { ... }
+   }
+
+   Sections and their "type" objects can be:
+
+		Log: FileLogger, SysLogger
+		DB: MySQL
+		Auth: X509Authenticator, X509NameAuthenticator,
+              X509MixMatchAuthenticator,PlainAuthenticator
+		Validator: JSONSchemaValidator, NoValidator
+		Handler: WardenHandler
+
+	"type" keyword is not mandatory, if not specified, first implementation
+object from particular section list is used ("FileLogger" for example).
+
+   Object function and configuration keys are as follows:
+
+   FileLogger: logging into file on the filesystem
+      filename: name of the file, defaults to "warden_server.log" at
+		        installation directory
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+   SysLogger: logging into unix syslog
+      socket: path to syslog socket, defaults to "/dev/log"
+      facility: syslog facility, defaults to "daemon"
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+   X509Authenticator: authenticate based on certificate chain validation,
+       hostname corresponding with certificate CN or SubjectAltName and
+       optionally shared secret (note that more clients on one machine
+       will have to have the certificate with the same hostname, clients
+       than can be differentiated by separate secrets).
+       This method is OBSOLETE.
+
+    X509NameAuthenticator: authenticate based on certificate chain validation,
+        certificate CN must correspond with client _name_, NOT hostname.
+
+    X509MixMatchAuthenticator: automatically choose X509Authenticator or
+        X509NameAuthenticator based on existence of 'secret' in query. Allows
+        for seamless transition of clients between two authentication methods.
+
+   PlainAuthenticator: authenticate based on client name or shared secret, usable
+      over plain HTTP connection or HTTPS without client certificate - note that
+      this pretty much spoils security, and is meant only for testing and
+      debugging purposes, NOT for production servers
+
+   NoValidator: forego event JSON validation, for debug purposes
+
+   JSONSchemaValidator: validate incoming events based on JSON schema file
+      filename: path to schema file, defaults to "idea.schema" at
+                installation directory, for information on obtaining current
+                IDEA schema file, refer to https://idea.cesnet.cz/en/schema
+
+   MySQL: database storage backend
+      host: database server host, default "localhost"
+      user: database user, default "warden"
+      password: database password
+      dbname: database name, default "warden3"
+      port: database server port, default 3306
+      retry_pause: retry in case of database errors, in seconds, defaults to 5
+      retry_count: number of retries, defaults to 3
+      event_size_limit: max size of serialized event, defaults to 5 MB
+      catmap_filename: IDEA category mapping to database ids, defaults to
+                       "catmap_mysql.json" at installation directory
+      tagmap_filename": IDEA node type mapping to database ids, defaults to
+                        "tagmap_mysql.json" at installation directory
+
+   WardenHandler: Main Warden RPC worker
+      send_events_limit: max events sent in one bunch, defaults to 10000
+      get_events_limit: max events received in one bunch, defaults to 10000
+      description: human readable description, sent in server info
+	
+------------------------------------------------------------------------------
+E. Command line
+
+   When run from command line, server offers set of commands and options for
+runtime and database management. You can also use --help option for each
+command and for server itself.
+   
+warden_server.py [--help] [-c CONFIG] <command>
+
+optional arguments:
+  -c CONFIG, --config CONFIG
+                        configuration file path
+
+commands:
+  {check,register,modify,list,loadmaps}
+    check               check configuration
+    register            register new client
+    modify              modify client registration
+    list                list registered clients
+    loadmaps            load catmap and tagmap into db
+
+
+warden_server.py register [--help] -n NAME -h HOSTNAME -r REQUESTOR
+                          [-s SECRET] [--note NOTE]
+                          [--valid | --novalid] [--read | --noread]
+                          [--nowrite | --write] [--nodebug | --debug]
+                          [--test | --notest]
+
+   Add new client registration entry.
+
+   optional arguments:
+     -n NAME, --name NAME  client name (in dotted reverse path notation)
+     -h HOSTNAME, --hostname HOSTNAME
+                           client FQDN hostname
+     -r REQUESTOR, --requestor REQUESTOR
+                           requestor email
+     -s SECRET, --secret SECRET
+                           authentication token (use explicit empty string to
+                           disable)
+     --note NOTE           client freetext description
+     --valid               valid client (default)
+     --novalid
+     --read                client is allowed to read (default)
+     --noread
+     --nowrite             client is allowed to send (default - no)
+     --write
+     --nodebug             client is allowed receive debug output (default - no)
+     --debug
+     --test                client is yet in testing phase (default - yes)
+     --notest
+
+
+warden_server.py modify [--help] -i ID [-n NAME] [-h HOSTNAME]
+                        [-r REQUESTOR] [-s SECRET] [--note NOTE]
+                        [--valid | --novalid] [--read | --noread]
+                        [--nowrite | --write] [--nodebug | --debug]
+                        [--test | --notest]
+
+   Modify details of client registration entry.
+
+   optional arguments:
+     -i ID, --id ID        client id
+
+   Rest of the arguments is same as in "register" command.
+
+
+warden_server.py list [--help] [--id ID]
+
+   List details of client registration entries.
+
+   optional arguments:
+     --id ID  client id
+
+
+warden_server.py purge [--help] [-l] [-e] [-d DAYS]
+
+   Purge old events or lastlog records. Note that lastlog purge retains at
+   least one newest record for each client, even if it is more than number of
+   'days' old.
+
+   optional arguments:
+     --help                show this help message and exit
+     -l, --lastlog         purge lastlog records
+     -e, --events          purge events
+     -d DAYS, --days DAYS  records older than 'days' back from today will get
+                           purged
+
+
+warden_server.py loadmaps [--help]
+
+   Load 'categories' and 'tags' table from 'catmap_mysql.json' and
+   'tagmap_mysql.json'. Note that this is NOT needed for server at all, load
+   them into db at will, should you need to run your own specific SQL queries
+   on data directly. Note also that previous content of both tables will be
+   lost.
+
+------------------------------------------------------------------------------
+Copyright (C) 2011-2015 Cesnet z.s.p.o

warden_server/README.test

++----------------------------------+
+| Warden3 Server Test Suite README |
++----------------------------------+
+
+Content
+
+  A. Introduction
+  B. Compatibility
+  C. Dependencies
+  D. Usage
+
+-------------------------------------------------------------------------------
+A. Introduction
+
+The Warden Server Test Suite is a collection of high-level functional tests
+(black-box testing), covering the most important interfaces of the Warden
+Server.
+
+-------------------------------------------------------------------------------
+B. Compatibility
+
+* The test suite, just like the Warden Server, is compatible with both Python2
+  (tested on 2.7) and Python3 (tested on 3.6).
+* Just like Warden Server, the test suite requires a local MySQL installation.
+* It is safe to run the test suite on a production system. For testing,
+  a database distinct from the default production one is used. Also, the user
+  account used for accessing the testing database is set for local login only.
+  To be extra safe, make sure not to name the production database `w3test`.
+
+-------------------------------------------------------------------------------
+C. Dependencies
+
+In addition to the regular Warden Server dependencies, package `unittest2` is
+required to run the test suite. It can be installed by running:
+	pip install unittest2
+or `pip3 install unittest2` for Python3 version
+	or on Debian:
+apt-get install python-unittest2
+	or alternatively:
+apt-get install python3-unittest2
+for Python3 version.
+
+An optional dependency is a code coverage measurement tool `Coverage.py`,
+which can be installed by:
+	pip install coverage
+or `pip3 install coverage` for Python3 version
+or on Debian:
+	apt-get install python-coverage
+or alternatively:
+	apt-get install python3-coverage
+for Python3 version.
+
+-------------------------------------------------------------------------------
+D. Usage
+
+Before running the tests (for the first time), a DB user with required rights
+must be created. An easy way to do it is:
+	./test_warden_server.py --init
+This will prompt for MySQL root password.
+
+Standard usage for testing:
+	./test_warden_server.py
+
+Advanced usage:
+
+	./test_warden_server.py --help
+	usage: test_warden_server.py [-h] [-i] [-n]
+
+	Warden3 Server Test Suite
+
+	optional arguments:
+	  -h, --help     show this help message and exit
+	  -i, --init     Set up an user with rights to CREATE/DROP the
+			 test database
+	  -n, --nopurge  Skip the database purge after running the tests
+
+
+Option -n (--nopurge) is meant for debugging purposes and test development, it
+keeps the test database around for inspection after running the tests.
+
+-------------------------------------------------------------------------------

warden_server/apache22.conf.dist

+SSLEngine on
+
+SSLVerifyClient optional
+SSLVerifyDepth 4
+SSLOptions +StdEnvVars +ExportCertData
+
+#SSLCipherSuite ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP:+eNULL
+
+SSLCertificateFile      /opt/warden_server/etc/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/etc/key.pem
+SSLCACertificateFile    /opt/warden_server/etc/tcs-ca-bundle.pem
+
+WSGIScriptAlias /warden3 /opt/warden_server/warden_server.wsgi
+
+<Directory /opt/warden_server/warden_server.wsgi>
+    Order allow,deny
+    Allow from all
+</Directory>

warden_server/apache24.conf.dist

+SSLEngine on
+
+SSLVerifyClient optional
+SSLVerifyDepth 4
+SSLOptions +StdEnvVars +ExportCertData
+
+#SSLCipherSuite ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP:+eNULL
+
+SSLCertificateFile      /opt/warden_server/etc/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/etc/key.pem
+SSLCACertificateFile    /opt/warden_server/etc/tcs-ca-bundle.pem
+
+WSGIScriptAlias /warden3 /opt/warden_server/warden_server.wsgi
+
+<DirectoryMatch /opt/warden_server/warden_server.wsgi>
+	Require all granted
+</DirectoryMatch>

warden_server/catmap_mysql.json

+{
+    "Abusive" : 100,  
+    "Abusive.Spam" : 101,
+    "Abusive.Harassment" : 102,
+    "Abusive.Child" : 103,
+    "Abusive.Sexual" : 104,
+    "Abusive.Violence" : 105,
+    "Malware" : 200,
+    "Malware.Virus" : 201,
+    "Malware.Worm" : 202,
+    "Malware.Trojan" : 203,
+    "Malware.Spyware" : 204,
+    "Malware.Dialer" : 205,
+    "Malware.Rootkit" : 206,
+    "Recon" : 300,
+    "Recon.Scanning" : 301,
+    "Recon.Sniffing" : 302,
+    "Recon.SocialEngineering" : 303,
+    "Recon.Searching" : 304,
+    "Attempt" : 400,
+    "Attempt.Exploit" : 401,
+    "Attempt.Login" : 402,
+    "Attempt.NewSignature" : 403,
+    "Intrusion" : 500,
+    "Intrusion.AdminCompromise" : 501,
+    "Intrusion.UserCompromise" : 502,
+    "Intrusion.AppCompromise" : 503,
+    "Intrusion.Botnet" : 504,
+    "Availability" : 600,
+    "Availability.DoS" : 601,
+    "Availability.DDoS" : 602,
+    "Availability.Sabotage" : 603,
+    "Availability.Outage" : 604,
+    "Information" : 700,
+    "Information.UnauthorizedAccess" : 701,
+    "Information.UnauthorizedModification" : 702,
+    "Fraud" : 800,
+    "Fraud.UnauthorizedUsage" : 801,
+    "Fraud.Copyright" : 802,
+    "Fraud.Masquerade" : 803,
+    "Fraud.Phishing" : 804,
+    "Fraud.Scam" : 805,
+    "Vulnerable" : 900,
+    "Vulnerable.Open" : 901,
+    "Vulnerable.Config" : 902,
+    "Anomaly" : 1000,
+    "Anomaly.Traffic" : 1001,
+    "Anomaly.Connection" : 1002,
+    "Anomaly.Protocol" : 1003,
+    "Anomaly.System" : 1004,        
+    "Anomaly.Application" : 1005,
+    "Anomaly.Behaviour" : 1006,
+    "Other" : 9998,
+    "Test" : 9999
+}
+

warden_server/tagmap_mysql.json

+{
+    "Connection": 1, 
+    "Datagram": 2, 
+    "Content": 3, 
+    "Data": 4, 
+    "File": 5, 
+    "Flow": 6, 
+    "Log": 7, 
+    "Protocol": 8, 
+    "Host": 9, 
+    "Network": 10, 
+    "Correlation": 11, 
+    "External": 12, 
+    "Reporting": 13,
+    "Relay": 14,
+    "Auth": 15,
+    "Blackhole": 30,
+    "Signature": 31,
+    "Statistical": 32,
+    "Heuristic": 33,
+    "Integrity": 34,
+    "Policy": 35,
+    "Honeypot": 36,
+    "Tarpit": 37,
+    "Recon": 38,
+    "Monitor": 39,
+    "Other" : 99
+}

warden_server/test_warden_server.py

+#!/usr/bin/python
+"""Warden3 Server Test Suite"""
+
+from __future__ import print_function
+import argparse
+import getpass
+import sys
+import warnings
+from os import path
+from copy import deepcopy
+import MySQLdb as my
+from warden_server import build_server
+import warden_server
+
+if sys.version_info >= (3, 10):
+    import unittest
+else:
+    import unittest2 as unittest
+
+if sys.version_info[0] >= 3:
+    from io import StringIO
+else:
+    from StringIO import StringIO
+
+USER = 'warden3test'
+PASSWORD = 'h7w*D>4B)3omcvLM$oJp'
+DB = 'w3test'
+
+
+def setUpModule():      # pylint: disable = locally-disabled, invalid-name
+    """Initialize the test database"""
+    print(__doc__)
+    conn = None
+    try:
+        conn = my.connect(user=USER, passwd=PASSWORD)
+        cur = conn.cursor()
+        with warnings.catch_warnings():                         # The database is not supposed to exist
+            warnings.simplefilter("ignore")
+            cur.execute("DROP DATABASE IF EXISTS %s" % (DB,))   # NOT SECURE
+            cur.execute("CREATE DATABASE %s" % (DB,))           # NOT SECURE
+            cur.execute("USE %s" % (DB,))                       # NOT SECURE
+            with open(path.join(path.dirname(__file__), 'warden_3.0.sql')) as script:
+                statements = ''.join([line.replace('\n', '') for line in script if line[0:2] != '--']).split(';')[:-1]
+                for statement in statements:
+                    cur.execute(statement)
+            cur.execute("INSERT INTO clients VALUES(NULL, NOW(), 'warden-info@cesnet.cz', 'test.server.warden.cesnet.cz', NULL, 1, 'cz.cesnet.warden3test', 'abc', 1, 1, 1, 0)")
+            conn.commit()
+    except my.OperationalError as ex:
+        if conn:
+            conn.rollback()
+            conn.close()
+            conn = None
+        print('Setup failed, have you tried --init ? Original exception: %s' % (str(ex),))
+        exit()
+    finally:
+        if conn:
+            conn.close()
+
+
+NO_PURGE = False
+
+
+def tearDownModule():  # pylint: disable = locally-disabled, invalid-name
+    """Clean up by purging the test database"""
+    if not NO_PURGE:
+        conn = my.connect(user=USER, passwd=PASSWORD)
+        cur = conn.cursor()
+        cur.execute("DROP DATABASE IF EXISTS %s" % (DB,))       # NOT SECURE
+        conn.commit()
+        conn.close()
+
+
+class ReadableSTR(str):
+    """Mission: To boldly quack like a buffer, like no str has quacked before"""
+    def read(self, content_length=None):
+        """Return own content"""
+        return ReadableSTR(self[0:content_length])
+
+    if getattr(str, 'decode', None) is None:
+        def decode(self, encoding="UTF-8", errors="strict"):    # pylint: disable = locally-disabled, unused-argument
+            """For Py3 return own content, no decoding necessary"""
+            return self
+
+
+class Request(object):
+    """Abstraction layer to perform an WSGI request"""
+    def __init__(self, app, uri, payload=""):
+        env = self.get_environ(uri, payload)
+        self.status = None
+        self.headers = None
+        raw_out = app(env, self.start_response)
+        self.out = [item.decode('ascii') for item in raw_out]
+
+    def __call__(self):
+        return self.status, self.headers, self.out
+
+    @staticmethod
+    def get_environ(uri, payload):
+        """Prepares an (partial) environ for WSGI app, almost like an WSGI server would"""
+        try:
+            full_path, query_string = uri.split('?')
+        except ValueError:
+            full_path = uri
+            query_string = ''
+        path_info = '/' + full_path.split('/')[-1]
+        env = {
+            "REQUEST_URI": uri,
+            "PATH_INFO": path_info,
+            "QUERY_STRING": query_string,
+            "SSL_CLIENT_VERIFY": "SUCCESS",
+            "SSL_CLIENT_S_DN_CN": "cz.cesnet.warden3test",
+            "SSL_CLIENT_CERT": "-----BEGIN CERTIFICATE-----\nMIIDgDCCAmgCCQDEG431XDXZjDANBgkqhkiG9w0BAQsFADCBgTELMAkGA1UEBhMCQ1oxFzAVBgNVBAoMDkNFU05FVCwgYS5sLmUuMQwwCgYDVQQLDAM3MDkxJTAjBgNVBAMMHHRlc3Quc2VydmVyLndhcmRlbi5jZXNuZXQuY3oxJDAiBgkqhkiG9w0BCQEWFXdhcmRlbi1pbmZvQGNlc25ldC5jejAeFw0xODA3MjMxMzMyMjFaFw0xODA4MjIxMzMyMjFaMIGBMQswCQYDVQQGEwJDWjEXMBUGA1UECgwOQ0VTTkVULCBhLmwuZS4xDDAKBgNVBAsMAzcwOTElMCMGA1UEAwwcdGVzdC5zZXJ2ZXIud2FyZGVuLmNlc25ldC5jejEkMCIGCSqGSIb3DQEJARYVd2FyZGVuLWluZm9AY2VzbmV0LmN6MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvgwOv1bv44hyWF7UDAPGdm+PqcbITi/6SVEfCENbMx6DAT+M3ZJlg7aOZyiZ16CRNxrjWizXYYY1H+NhOvlPZwsBcHFvnaBrcBciURMW6AQ+OiIHUONDUV7zqTcyiZ6NDMoNy472UpfNBMYXMtaUjPO33aRYwtl+QjoivU8bhzcSxyr/4P6WnZ7rW2nuHWfUNcGWGVxsRw7E2r4OY3Yr6M4SjKEDTEalByApoOYj2s3oEmeiNPjxKhN0wgD4h38+HcnpmKGZLNFbOEdT/7luA6IwzJ7l0p4ktjgCl/x3/Y6ZBrIZuFCNxjYrdfciD27LmcA5A6nEJ083fa4d+O/H8QIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBc6EtV6FYnFBd735h4zwe2SIaFs2bu1d6COsOsaWe5loInI+oEATThaBlA9QiVamikkug3t2wgro8YcYhp0CMPN1gMxR6GstrBrKafprWp/Dv3+IP8RY+Z2lJ0ivw1MTMipqsCMiB+Lvs2wRVV3xBIXslgI3dbceZXos2bj6CPf3Frho7Z7oRaHetI+1a0T9QqZSug7dUSmYNCd9ZXQ8kFzU3eCFP0JKMqOy75KHIE00xowarDDFjTyyPoHmZviIOsY8ByKGNRDQz/WnZWzghAQjb+7tTFm2deOQua0XIyO7GSIU2xdGbTje4wA3/YiWhkpF8HWpCEAN8G6sMTDEXF\n-----END CERTIFICATE-----",      # pylint: disable = locally-disabled, line-too-long
+            "wsgi.input": ReadableSTR(payload),
+            "CONTENT_LENGTH": len(payload)
+            }
+        return env
+
+    def start_response(self, status, headers):
+        """Mocked start_response to record returned status and headers"""
+        self.status = status
+        self.headers = headers
+
+
+class Warden3ServerTest(unittest.TestCase):
+    """High level Warden3 Server tests"""
+    config = {'log': {'level': 'debug'}, 'validator': {'type': 'NoValidator'}, 'auth': {'type': 'PlainAuthenticator'},
+              'db': {'user': USER, 'password': PASSWORD, 'dbname': DB}, 'handler': {'description': 'Warden Test Server'}}
+
+    getInfo_interface_tests_specific = [
+        ("/getInfo", "403 I'm watching. Authenticate."),
+        ("/getInfo?client=", "403 I'm watching. Authenticate."),
+        ("/getInfo?client=cz.cesnet.warden3test", "403 I'm watching. Authenticate."),
+        ("/getInfo?client=cz.cesnet.warden3test&secret=123", "403 I'm watching. Authenticate."),
+        ("/getInfo?secret=123", "403 I'm watching. Authenticate."),
+    ]
+    getEvents_interface_tests_specific = [
+        ("/getEvents", "403 I'm watching. Authenticate.", None),
+        ("/getEvents?secret=123", "403 I'm watching. Authenticate.", None),
+    ]
+
+    @classmethod
+    def setUpClass(cls):
+        """Pre-test cleanup"""
+        cls.clean_lastid()
+        cls.app = build_server(cls.config)
+
+    @classmethod
+    def clean_lastid(cls):
+        """Cleans the lastid information for all clients"""
+        conn = my.connect(user=USER, passwd=PASSWORD, db=DB)
+        cur = conn.cursor()
+        cur.execute("DELETE FROM events")
+        cur.execute("DELETE FROM last_events")
+        cur.close()
+        conn.commit()
+        conn.close()
+
+    def test_getInfo_interface(self):       # pylint: disable = locally-disabled, invalid-name
+        """Tests the getInfo method invocation"""
+        tests_common = [
+            ("/getInfo?secret=abc", "200 OK"),
+            ("/getInfo?secret=abc&evil=false", "200 OK"),       # RFC3514
+            ("/getInfo?client=cz.cesnet.warden3test&secret=abc", "200 OK"),
+            ("/getInfo?client=asdf.blefub", "403 I'm watching. Authenticate."),
+            ("/getInfo?client=asdf.blefub&secret=abc", "403 I'm watching. Authenticate."),
+            ("/getInfo?secret=abc&self=test", "200 OK"),        # Internal parameter
+        ]
+        for query, expected_status in tests_common + self.getInfo_interface_tests_specific:
+            with self.subTest(query=query, expected_status=expected_status):
+                status, _, _ = Request(self.app, query)()
+                self.assertEqual(status, expected_status)
+
+    def test_getEvents_interface(self):     # pylint: disable = locally-disabled, invalid-name
+        """Tests the getEvents method invocation"""
+        tests_common = [
+            ("/getEvents?secret=abc", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?client=foo", "403 I'm watching. Authenticate.", None),
+            ("/getEvents?secret=abc&foo=bar", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?secret=abc&lastid=1", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?secret=abc&lastid=0", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?secret=abc&lastid=9", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?secret=abc&cat=bflm", "422 Wrong tag or category used in query.", None),
+            ("/getEvents?secret=abc&cat=Other", "200 OK", None),
+            ("/getEvents?secret=abc&tag=Other", "200 OK", None),
+            ("/getEvents?secret=abc&group=Other", "200 OK", None),
+            ("/getEvents?secret=abc&cat=Other&nocat=Test", "422 Unrealizable conditions. Choose cat or nocat option.", None),
+            ("/getEvents?secret=abc&tag=Other&notag=Test", "422 Unrealizable conditions. Choose tag or notag option.", None),
+            ("/getEvents?secret=abc&group=Other&nogroup=Test", "422 Unrealizable conditions. Choose group or nogroup option.", None),
+            ("/getEvents?client=cz.cesnet.warden3test&secret=abc&count=3&id=10", "200 OK", None)
+        ]
+        for query, expected_status, expected_response in tests_common + self.getEvents_interface_tests_specific:
+            with self.subTest(query=query, expected_status=expected_status, expected_response=expected_response):
+                status, _, out = Request(self.app, query)()
+                self.assertEqual(status, expected_status)
+                if expected_response is not None:
+                    self.assertEqual(out, expected_response)
+
+    def test_getDebug_interface(self):      # pylint: disable = locally-disabled, invalid-name
+        """Tests the getDebug method invocation"""
+        tests = [
+            ("/getDebug?secret=abc", "200 OK"),
+            ("/getDebug?client=cz.cesnet.warden3test&secret=abc", "200 OK"),
+            ("/getDebug?secret=abc&self=test", "200 OK"),
+        ]
+        for query, expected_status in tests:
+            with self.subTest(query=query, expected_status=expected_status):
+                status, _, _ = Request(self.app, query)()
+                self.assertEqual(status, expected_status)
+
+    def test_methods(self):
+        """Tests application behaviour in method parsing"""
+        tests = [
+            ("", "404 You've fallen off the cliff."),
+            ("/blefub?client=client&secret=secret", "404 You've fallen off the cliff."),
+            ("/?client=client&secret=secret", "404 You've fallen off the cliff."),
+        ]
+        for query, expected_status in tests:
+            with self.subTest(query=query, expected_status=expected_status):
+                status, _, _ = Request(self.app, query)()
+                self.assertEqual(status, expected_status)
+
+    def test_payload(self):
+        """Tests parsing of transported data"""
+        tests = [
+            ("/getInfo?secret=abc", "", "200 OK", None),
+            ("/getInfo?secret=abc", "[1]", "200 OK", None),
+            ("/getInfo?secret=abc", "{#$%^", "200 OK", None),
+            ("/sendEvents?secret=abc", "", "200 OK", ['{"saved": 0}']),
+            ("/sendEvents?secret=abc", "{'test': 'true'}", "400 Deserialization error.", None),
+            ("/sendEvents?secret=abc", '{"test": "true"}', "400 List of events expected.", None),
+            ("/sendEvents?secret=abc", '[{"test": "true"}]', "422 Event does not bear valid Node attribute", None),
+            ("/sendEvents?secret=abc", '[{"Node": ["test", "test2"]}]', "422 Event does not bear valid Node attribute", None),
+            ("/sendEvents?secret=abc", '[{"Node": ["Name", "test"]}]', "422 Event does not bear valid Node attribute", None),
+            ("/sendEvents?secret=abc", '[{"Node": [{"Name"}]}]', "400 Deserialization error.", None),
+            ("/sendEvents?secret=abc", '[{"Node": [{"Name": "test"}]}]', "422 Node does not correspond with saving client", None),
+            ("/sendEvents?secret=abc", '[{"Node": [{"Name": "cz.cesnet.warden3test"}]}]', "200 OK", ['{"saved": 1}']),
+        ]
+        for query, payload, expected_status, expected_response in tests:
+            with self.subTest(query=query, payload=payload, expected_status=expected_status, expected_response=expected_response):
+                status, _, out = Request(self.app, query, payload)()
+                self.assertEqual(status, expected_status)
+                if expected_response is not None:
+                    self.assertEqual(out, expected_response)
+
+
+class X509AuthenticatorTest(Warden3ServerTest):
+    """Performs the basic test suite using the X509Authenticator"""
+    config = deepcopy(Warden3ServerTest.config)
+    config['auth']['type'] = 'X509Authenticator'
+
+
+class X509NameAuthenticatorTest(Warden3ServerTest):
+    """Performs the basic test suite using the X509NameAuthenticator"""
+    config = deepcopy(Warden3ServerTest.config)
+    config['auth']['type'] = 'X509NameAuthenticator'
+
+    getInfo_interface_tests_specific = [
+        ("/getInfo", "200 OK"),
+        ("/getInfo?client=", "200 OK"),
+        ("/getInfo?client=cz.cesnet.warden3test", "200 OK"),
+        ("/getInfo?client=cz.cesnet.warden3test&secret=123", "200 OK"),
+        ("/getInfo?secret=123", "200 OK"),
+    ]
+    getEvents_interface_tests_specific = [
+        ("/getEvents", "200 OK", None),
+        ("/getEvents?secret=123", "200 OK", None),
+    ]
+
+
+class WScliTest(unittest.TestCase):
+    """Tester of the Warden Server command line interface"""
+    @classmethod
+    def setUpClass(cls):
+        cls.config = {'log': {'level': 'debug'}, 'validator': {'type': 'NoValidator'}, 'auth': {'type': 'PlainAuthenticator'},
+                      'db': {'user': USER, 'password': PASSWORD, 'dbname': DB}, 'handler': {'description': 'Warden Test Server'}}
+        warden_server.server = build_server(cls.config)
+
+    @staticmethod
+    def do_cli(command_line):
+        """Performs the command line action requested by argv and presents the results"""
+        argv_backup = sys.argv
+        sys.argv = command_line
+        out = StringIO()
+        err = StringIO()
+        sys.stdout = out
+        sys.stderr = err
+        try:
+            args = warden_server.get_args()
+            command = args.command
+            subargs = vars(args)
+            del subargs["command"]
+            del subargs["config"]
+            ret = command(**subargs)
+        except SystemExit as sys_exit:
+            ret = sys_exit.code
+        sys.stdout = sys.__stdout__
+        sys.stderr = sys.__stderr__
+        sys.argv = argv_backup
+        return ret, out.getvalue(), err.getvalue()
+
+    @staticmethod
+    def do_sql_select(query, params):
+        """Reads data from database"""
+        conn = my.connect(user=USER, passwd=PASSWORD, db=DB)
+        cur = conn.cursor()
+        cur.execute(query, params)
+        result = cur.fetchall()
+        cur.close()
+        conn.close()
+        return result
+
+    def test_list(self):
+        """Tests the list command line option"""
+        tests = [
+            (['list'], 0, 4),
+            (['list', '--id=1'], 0, 4),
+            (['list', '--id=1000'], 0, 3),
+            (['list', '--id', '1'], 0, 4),
+            (['list', '--id', '1000'], 0, 3),
+        ]
+        for supplied_arguments, expected_return, output_lines in tests:
+            with self.subTest(supplied_arguments=supplied_arguments, expected_return=expected_return, output_lines=output_lines):
+                ret, out, _ = self.do_cli(['./warden_server.py'] + supplied_arguments)
+                self.assertEqual(ret, expected_return)
+                self.assertEqual(len(out.split('\n')), output_lines)
+
+    def test_register_modify(self):
+        """Tests the client registration and its modification"""
+        tests = [
+            (['register', '-n', 'cz.cesnet.warden.test2', '-h', 'test2.warden.cesnet.cz', '-r', 'warden-info@cesnet.cz'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--novalid'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 0, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--valid'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--valid', '--novalid'], 2,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--read'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--noread', '--write'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 0, 0, 1, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--debug', '--read'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 1, 1, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--notest', '--nodebug'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '--notest', '--nodebug'], 2,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', '1000', '--notest', '--nodebug'], 251,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-n', 'cz.cesnet.warden.test3'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-n', '..'], 254,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-h', 'test3.warden.cesnet.cz'], 0,
+             (('warden-info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-h', ''.zfill(256)], 253,
+             (('warden-info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-h', '..'], 253,
+             (('warden-info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'warden-info@cesnet.cz, info@cesnet.cz'], 0,
+             (('warden-info@cesnet.cz, info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'warden-info@cesnet.cz ,info@cesnet.cz'], 0,
+             (('warden-info@cesnet.cz ,info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'Warden Info <warden-info@cesnet.cz>'], 0,
+             (('Warden Info <warden-info@cesnet.cz>', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'Other Info <other-info@x.cz'], 252,
+             (('Warden Info <warden-info@cesnet.cz>', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'Other other@x.cz'], 252,
+             (('Warden Info <warden-info@cesnet.cz>', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'a@b, '], 252,
+             (('Warden Info <warden-info@cesnet.cz>', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'a@b'], 0,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', '@'], 252,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'abc'], 252,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'a@b@c'], 252,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-n', 'cz.cesnet.warden.test3'], 250,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-s', 'abc'], 249,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-s', 'top_secret'], 0,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-s', 'top_secret'], 249,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--note', ''.zfill(1024)], 0,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, ''.zfill(1024)),)),
+            (['modify', '-i', 'CLIENT_ID', '--note', 'Valid until: 18.01.2038'], 0,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, 'Valid until: 18.01.2038'),)),
+            (['modify', '-i', 'CLIENT_ID', '--note', 'Valid until:', '20.1.2038'], 2,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, 'Valid until: 18.01.2038'),)),
+        ]
+        test_sql = "SELECT requestor, hostname, name, secret, valid, clients.read, debug, clients.write, test, note FROM clients WHERE id = %s"
+        client_id = None
+        for supplied_arguments, expected_return, expected_sql_result in tests:
+            with self.subTest(supplied_arguments=supplied_arguments, expected_return=expected_return, expected_sql_result=expected_sql_result):
+                supplied_arguments = [entry.replace('CLIENT_ID', str(client_id)) for entry in supplied_arguments]
+                ret, out, _ = self.do_cli(['./warden_server.py'] + supplied_arguments)
+                self.assertEqual(ret, expected_return)
+                try:
+                    client_id = int(out.split('\n')[-2].split(' ')[0])
+                except IndexError:      # No modification was performed, keep the previous client_id
+                    pass
+                result = self.do_sql_select(test_sql, (client_id,))
+                self.assertEqual(result, expected_sql_result)
+
+
+def init_user():
+    """DB user rights setup"""
+    conn = None
+    try:
+        conn = my.connect(user='root', passwd=getpass.getpass('Enter MySQL Root password:'))
+        with conn.cursor() as cur:
+            cur.execute("CREATE USER IF NOT EXISTS %s@'localhost' IDENTIFIED BY %s", (USER, PASSWORD))
+            cur.execute("GRANT SELECT, INSERT, UPDATE, CREATE, DELETE, DROP ON *.* TO %s@'localhost'", (USER,))
+        conn.commit()
+        print("DB User set up successfuly")
+    except my.OperationalError as ex:
+        if conn:
+            conn.rollback()
+            conn.close()
+            conn = None
+        print('Connection unsuccessful, bad password? Original exception: %s' % (str(ex)))
+        exit()
+    except KeyboardInterrupt:
+        print("\nCancelled!")
+        exit()
+    finally:
+        if conn:
+            conn.close()
+
+
+def main():
+    """Parses arguments and acts accordingly"""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('-i', '--init', action='store_true', help='Set up an user with rights to CREATE/DROP the test database')
+    parser.add_argument('-n', '--nopurge', action='store_true', help='Skip the database purge after running the tests')
+
+    args = parser.parse_args()
+    if args.init:
+        init_user()
+    else:
+        if args.nopurge:
+            global NO_PURGE    # pylint: disable = locally-disabled, global-statement
+            NO_PURGE = True
+        sys.argv = [sys.argv[0]]
+        unittest.main()
+
+
+if __name__ == "__main__":
+    main()

warden_server/warden_3.0.sql

+-- phpMyAdmin SQL Dump
+-- version 3.4.11.1deb2+deb7u1
+-- http://www.phpmyadmin.net
+--
+-- Host: localhost
+-- Generation Time: Dec 04, 2014 at 02:54 PM
+-- Server version: 5.5.38
+-- PHP Version: 5.4.4-14+deb7u14
+
+SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO";
+SET time_zone = "+00:00";
+
+
+/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
+/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
+/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
+/*!40101 SET NAMES utf8 */;
+
+--
+-- Database: `warden3`
+--
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `categories`
+--
+
+CREATE TABLE IF NOT EXISTS `categories` (
+  `id` int UNSIGNED NOT NULL,
+  `category` varchar(64) NOT NULL,
+  `subcategory` varchar(64) DEFAULT NULL,
+  `cat_subcat` varchar(129) NOT NULL,
+  KEY `cat_sub` (`cat_subcat`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci;
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `clients`
+--
+
+CREATE TABLE IF NOT EXISTS `clients` (
+  `id` int UNSIGNED NOT NULL AUTO_INCREMENT,
+  `registered` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `requestor` varchar(256) NOT NULL,
+  `hostname` varchar(256) NOT NULL,
+  `note` text NULL,
+  `valid` tinyint UNSIGNED NOT NULL DEFAULT '1',
+  `name` varchar(64) NOT NULL,
+  `secret` varchar(16) NULL,
+  `read` tinyint UNSIGNED NOT NULL DEFAULT '1',
+  `debug` tinyint UNSIGNED NOT NULL DEFAULT '0',
+  `write` tinyint UNSIGNED NOT NULL DEFAULT '0',
+  `test` tinyint UNSIGNED NOT NULL DEFAULT '0',
+  PRIMARY KEY (`id`),
+  KEY `clients_1` (`valid`, `secret`, `hostname`),
+  KEY `clients_2` (`valid`, `name`)
+) ENGINE=InnoDB  DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci AUTO_INCREMENT=1 ;
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `events`
+--
+
+CREATE TABLE IF NOT EXISTS `events` (
+  `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT,
+  `received` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  `client_id` int UNSIGNED NOT NULL,
+  `data` longtext NOT NULL,
+  `valid` tinyint UNSIGNED NOT NULL DEFAULT '1',
+  PRIMARY KEY (`id`),
+  KEY `id` (`id`,`client_id`),
+  KEY `received` (`received`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8  DEFAULT COLLATE utf8_unicode_ci AUTO_INCREMENT=2 ;
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `event_category_mapping`
+--
+
+CREATE TABLE IF NOT EXISTS `event_category_mapping` (
+  `event_id` bigint UNSIGNED NOT NULL,
+  `category_id` int UNSIGNED NOT NULL,
+  KEY `event_id_2` (`event_id`,`category_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci;
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `event_tag_mapping`
+--
+
+CREATE TABLE IF NOT EXISTS `event_tag_mapping` (
+  `event_id` bigint UNSIGNED NOT NULL,
+  `tag_id` int UNSIGNED NOT NULL,
+  KEY `event_id_2` (`event_id`,`tag_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci;
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `last_events`
+--
+
+CREATE TABLE IF NOT EXISTS `last_events` (
+  `id` int UNSIGNED NOT NULL AUTO_INCREMENT,
+  `client_id` int UNSIGNED NOT NULL,
+  `event_id` bigint UNSIGNED NOT NULL,
+  `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  KEY `client_id` (`client_id`,`event_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci AUTO_INCREMENT=1 ;
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `tags`
+--
+
+CREATE TABLE IF NOT EXISTS `tags` (
+  `id` int UNSIGNED NOT NULL,
+  `tag` varchar(64) NOT NULL,
+  KEY `id_tag_name` (`id`,`tag`),
+  KEY `tag_name` (`tag`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci;
+
+/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
+/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
+/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;

warden_server/warden_server.cfg.dist

+{
+    "Log": {
+        // If not specified, FileLogger is default
+        "level": "debug"
+    },
+    "Handler": {
+        "send_events_limit": 500,
+        "get_events_limit": 1000,
+        "description": "Warden 3 distribution config"
+    },
+    "DB": {
+        "user": "warden",
+        "password": "EXAMPLE",
+        "dbname": "warden3"
+    }
+}
\ No newline at end of file

warden_server/warden_server.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2015 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+from __future__ import print_function
+
+import sys
+import os
+import io
+from os import path
+import logging
+import logging.handlers
+import json
+import re
+from traceback import format_tb
+from collections import namedtuple
+from time import sleep
+from random import randint
+import M2Crypto.X509
+import MySQLdb as my
+import MySQLdb.cursors as mycursors
+
+if sys.version_info[0] >= 3:
+    import configparser as ConfigParser
+    from urllib.parse import parse_qs
+    unicode = str
+
+    def get_method_params(method):
+        return method.__code__.co_varnames[:method.__code__.co_argcount]
+
+else:
+    import ConfigParser
+    from urlparse import parse_qs
+
+    def get_method_params(method):
+        return method.func_code.co_varnames[:method.func_code.co_argcount]
+
+
+# for local version of up to date jsonschema
+sys.path.append(path.join(path.dirname(__file__), "..", "lib"))
+
+from jsonschema import Draft4Validator
+
+
+VERSION = "3.0-beta3"
+
+
+class Error(Exception):
+
+    def __init__(self, method=None, req_id=None, errors=None, **kwargs):
+        self.method = method
+        self.req_id = req_id
+        self.errors = [kwargs] if kwargs else []
+        if errors:
+            self.errors.extend(errors)
+
+    def append(self, _events=None, **kwargs):
+        self.errors.append(kwargs)
+
+    def get_http_err_msg(self):
+        try:
+            err = self.errors[0]["error"]
+            msg = self.errors[0]["message"].replace("\n", " ")
+        except (IndexError, KeyError):
+            err = 500
+            msg = "There's NO self-destruction button! Ah, you've just found it..."
+        for e in self.errors:
+            next_err = e.get("error", 500)
+            if err != next_err:
+                # errors not same, round to basic err code (400, 500)
+                # and use the highest one
+                err = max(err//100, next_err//100)*100
+            next_msg = e.get("message", "Unknown error").replace("\n", " ")
+            if msg != next_msg:
+                msg = "Multiple errors"
+        return err, msg
+
+    def __str__(self):
+        return "\n".join(self.str_err(e) for e in self.errors)
+
+    def log(self, logger, prio=logging.ERROR):
+        for e in self.errors:
+            logger.log(prio, self.str_err(e))
+            info = self.str_info(e)
+            if info:
+                logger.info(info)
+            debug = self.str_debug(e)
+            if debug:
+                logger.debug(debug)
+
+    def str_err(self, e):
+        out = []
+        out.append("Error(%s) %s " % (e.get("error", 0), e.get("message", "Unknown error")))
+        if "exc" in e and e["exc"]:
+            out.append("(cause was %s: %s)" % (e["exc"][0].__name__, str(e["exc"][1])))
+        return "".join(out)
+
+    def str_info(self, e):
+        ecopy = dict(e)    # shallow copy
+        ecopy.pop("req_id", None)
+        ecopy.pop("method", None)
+        ecopy.pop("error", None)
+        ecopy.pop("message", None)
+        ecopy.pop("exc", None)
+        if ecopy:
+            out = "Detail: %s" % (json.dumps(ecopy, default=lambda v: str(v)))
+        else:
+            out = ""
+        return out
+
+    def str_debug(self, e):
+        out = []
+        if not e.get("exc"):
+            return ""
+        exc_tb = e["exc"][2]
+        if exc_tb:
+            out.append("Traceback:\n")
+            out.extend(format_tb(exc_tb))
+        return "".join(out)
+
+    def to_dict(self):
+        errlist = []
+        for e in self.errors:
+            ecopy = dict(e)
+            ecopy.pop("exc", None)
+            errlist.append(ecopy)
+        d = {
+            "method": self.method,
+            "req_id": self.req_id,
+            "errors": errlist
+        }
+        return d
+
+
+def get_clean_root_logger(level=logging.INFO):
+    """ Attempts to get logging module into clean slate state """
+
+    # We want to be able to set up at least stderr logger before any
+    # configuration is read, and then later get rid of it and set up
+    # whatever administrator requires.
+    # However, there can exist only one logger, but we want to get a clean
+    # slate everytime we initialize StreamLogger or FileLogger... which
+    # is not exactly supported by logging module.
+    # So, we look directly inside logger class and clean up handlers/filters
+    # manually.
+    logger = logging.getLogger(__name__)
+    logger.setLevel(level)
+    while logger.handlers:
+        logger.handlers[0].close()
+        logger.removeHandler(logger.handlers[0])
+    while logger.filters:
+        logger.removeFilter(logger.filters[0])
+    logger.propagate = False
+    return logger
+
+
+def StreamLogger(stream=sys.stderr, level=logging.DEBUG):
+    """ Fallback handler just for setup, not meant to be used from
+        configuration file because during wsgi query stdout/stderr
+        is forbidden.
+    """
+
+    fhand = logging.StreamHandler(stream)
+    fform = logging.Formatter('%(asctime)s %(filename)s[%(process)d]: (%(levelname)s) %(message)s')
+    fhand.setFormatter(fform)
+    logger = get_clean_root_logger(level)
+    logger.addHandler(fhand)
+    return logger
+
+
+class LogRequestFilter(logging.Filter):
+    """ Filter class, instance of which is added to logger class to add
+        info about request automatically into every logline, no matter
+        how it came into existence.
+    """
+
+    def __init__(self, req):
+        logging.Filter.__init__(self)
+        self.req = req
+
+    def filter(self, record):
+        if self.req.env:
+            record.req_preamble = "%08x/%s: " % (self.req.req_id or 0, self.req.path)
+        else:
+            record.req_preamble = ""
+        return True
+
+
+def FileLogger(req, filename, level=logging.INFO):
+
+    fhand = logging.FileHandler(filename)
+    fform = logging.Formatter('%(asctime)s %(filename)s[%(process)d]: (%(levelname)s) %(req_preamble)s%(message)s')
+    fhand.setFormatter(fform)
+    ffilt = LogRequestFilter(req)
+    logger = get_clean_root_logger(level)
+    logger.addFilter(ffilt)
+    logger.addHandler(fhand)
+    logger.info("Initialized FileLogger(req=%r, filename=\"%s\", level=%s)" % (req, filename, level))
+    return logger
+
+
+def SysLogger(req, socket="/dev/log", facility=logging.handlers.SysLogHandler.LOG_DAEMON, level=logging.INFO):
+
+    fhand = logging.handlers.SysLogHandler(address=socket, facility=facility)
+    fform = logging.Formatter('%(filename)s[%(process)d]: (%(levelname)s) %(req_preamble)s%(message)s')
+    fhand.setFormatter(fform)
+    ffilt = LogRequestFilter(req)
+    logger = get_clean_root_logger(level)
+    logger.addFilter(ffilt)
+    logger.addHandler(fhand)
+    logger.info("Initialized SysLogger(req=%r, socket=\"%s\", facility=\"%d\", level=%s)" % (req, socket, facility, level))
+    return logger
+
+
+Client = namedtuple("Client", [
+    "id", "registered", "requestor", "hostname", "name",
+    "secret", "valid", "read", "debug", "write", "test", "note"])
+
+
+class Object(object):
+
+    def __str__(self):
+        attrs = get_method_params(self.__init__)[1:]
+        eq_str = ["%s=%r" % (attr, getattr(self, attr, None)) for attr in attrs]
+        return "%s(%s)" % (type(self).__name__, ", ".join(eq_str))
+
+
+class Request(Object):
+    """ Simple container for info about ongoing request.
+        One instance gets created before server startup, and all other
+        configured objects get it as parameter during instantiation.
+
+        Server then takes care of populating this instance on the start
+        of wsgi request (and resetting at the end). All other objects
+        then can find this actual request info in their own self.req.
+
+        However, only Server.wsgi_app, handler (WardenHandler) exposed
+        methods and logging related objects should use self.req directly.
+        All other objects should use self.req only as source of data for
+        error/exception handling/logging, and should take/return
+        necessary data as arguments/return values for clarity on
+        which data their main codepaths work with.
+    """
+
+    def reset(self, env=None, client=None, path=None, req_id=None):
+        self.env = env
+        self.client = client
+        self.path = path or ""
+        if req_id is not None:
+            self.req_id = req_id
+        else:
+            self.req_id = 0 if env is None else randint(0x00000000, 0xFFFFFFFF)
+
+    __init__ = reset
+
+    def error(self, **kwargs):
+        return Error(self.path, self.req_id, **kwargs)
+
+
+class ObjectBase(Object):
+
+    def __init__(self, req, log):
+        Object.__init__(self)
+        self.req = req
+        self.log = log
+
+
+class PlainAuthenticator(ObjectBase):
+
+    def __init__(self, req, log, db):
+        ObjectBase.__init__(self, req, log)
+        self.db = db
+
+    def authenticate(self, env, args, hostnames=None, check_secret=True):
+        name = args.get("client", [None])[0]
+        secret = args.get("secret", [None])[0] if check_secret else None
+
+        client = self.db.get_client_by_name(hostnames, name, secret)
+
+        if not client:
+            self.log.info("authenticate: client not found by name: \"%s\", secret: %s, hostnames: %s" % (
+                name, secret, str(hostnames)))
+            return None
+
+        # Clients with 'secret' set must get authenticated by it.
+        # No secret turns secret auth off for this particular client.
+        if client.secret is not None and secret is None and check_secret:
+            self.log.info("authenticate: missing secret argument")
+            return None
+
+        self.log.info("authenticate: %s" % str(client))
+
+        # These args are not for handler
+        args.pop("client", None)
+        args.pop("secret", None)
+
+        return client
+
+    def authorize(self, env, client, path, method):
+        if method.debug:
+            if not client.debug:
+                self.log.info("authorize: failed, client does not have debug enabled")
+                return None
+            return client
+
+        if method.read:
+            if not client.read:
+                self.log.info("authorize: failed, client does not have read enabled")
+                return None
+            return client
+
+        if method.write:
+            if not (client.write or client.test):
+                self.log.info("authorize: failed, client is not allowed to write or test")
+                return None
+
+        return client
+
+
+class X509Authenticator(PlainAuthenticator):
+
+    def get_cert_dns_names(self, pem):
+
+        cert = M2Crypto.X509.load_cert_string(pem)
+
+        subj = cert.get_subject()
+        commons = [n.get_data().as_text() for n in subj.get_entries_by_nid(subj.nid["CN"])]
+
+        try:
+            extstrs = cert.get_ext("subjectAltName").get_value().split(",")
+        except LookupError:
+            extstrs = []
+        extstrs = [val.strip() for val in extstrs]
+        altnames = [val[4:] for val in extstrs if val.startswith("DNS:")]
+
+        # bit of mangling to get rid of duplicates and leave commonname first
+        firstcommon = commons[0]
+        return [firstcommon] + list(set(altnames+commons) - set([firstcommon]))
+
+    def is_verified_by_apache(self, env, args):
+        # Allows correct work while SSLVerifyClient both "optional" and "required"
+        verify = env.get("SSL_CLIENT_VERIFY")
+        if verify == "SUCCESS":
+            return True
+        exception = self.req.error(
+            message="authenticate: certificate verification failed",
+            error=403, args=args, ssl_client_verify=verify, cert=env.get("SSL_CLIENT_CERT"))
+        exception.log(self.log)
+        return False
+
+    def authenticate(self, env, args):
+        if not self.is_verified_by_apache(env, args):
+            return None
+
+        try:
+            cert_names = self.get_cert_dns_names(env["SSL_CLIENT_CERT"])
+        except:
+            exception = self.req.error(
+                message="authenticate: cannot get or parse certificate from env",
+                error=403, exc=sys.exc_info(), env=env)
+            exception.log(self.log)
+            return None
+
+        return PlainAuthenticator.authenticate(self, env, args, hostnames=cert_names)
+
+
+class X509NameAuthenticator(X509Authenticator):
+
+    def authenticate(self, env, args):
+        if not self.is_verified_by_apache(env, args):
+            return None
+
+        try:
+            cert_name = env["SSL_CLIENT_S_DN_CN"]
+        except:
+            exception = self.req.error(
+                message="authenticate: cannot get or parse certificate from env",
+                error=403, exc=sys.exc_info(), env=env)
+            exception.log(self.log)
+            return None
+
+        if cert_name != args.setdefault("client", [cert_name])[0]:
+            exception = self.req.error(
+                message="authenticate: client name does not correspond with certificate",
+                error=403, cn=cert_name, args=args)
+            exception.log(self.log)
+            return None
+
+        return PlainAuthenticator.authenticate(self, env, args, check_secret=False)
+
+
+class X509MixMatchAuthenticator(X509Authenticator):
+
+    def __init__(self, req, log, db):
+        PlainAuthenticator.__init__(self, req, log, db)
+        self.hostname_auth = X509Authenticator(req, log, db)
+        self.name_auth = X509NameAuthenticator(req, log, db)
+
+    def authenticate(self, env, args):
+        if not self.is_verified_by_apache(env, args):
+            return None
+
+        try:
+            cert_name = env["SSL_CLIENT_S_DN_CN"]
+        except:
+            exception = self.req.error(
+                message="authenticate: cannot get or parse certificate from env",
+                error=403, exc=sys.exc_info(), env=env)
+            exception.log(self.log)
+            return None
+        name = args.get("client", [None])[0]
+        secret = args.get("secret", [None])[0]
+
+        # Client names are in reverse notation than DNS, client name should
+        # thus never be the same as machine hostname (if it is, client
+        # admin does something very amiss).
+
+        # So, if client sends the same name in query as in the certificate,
+        # or sends no name or secret (which is necessary for hostname auth),
+        # use X509NameAuthenticator. Otherwise (names are different and there
+        # is name and/or secret in query) use (hostname) X509Authenticator.
+
+        if name == cert_name or (name is None and secret is None):
+            auth = self.name_auth
+        else:
+            auth = self.hostname_auth
+
+        self.log.info("MixMatch is choosing %s (name: %s, cert_name: %s)" % (type(auth).__name__, name, cert_name))
+        return auth.authenticate(env, args)
+
+
+class NoValidator(ObjectBase):
+
+    def __init__(self, req, log):
+        ObjectBase.__init__(self, req, log)
+
+    def check(self, event):
+        return []
+
+
+class JSONSchemaValidator(NoValidator):
+
+    def __init__(self, req, log, filename=None):
+        NoValidator.__init__(self, req, log)
+        self.path = filename or path.join(path.dirname(__file__), "idea.schema")
+        with io.open(self.path, "r", encoding="utf-8") as f:
+            self.schema = json.load(f)
+        self.validator = Draft4Validator(self.schema)
+
+    def check(self, event):
+
+        def sortkey(k):
+            """ Treat keys as lowercase, prefer keys with less path segments """
+            return (len(k.path), "/".join(str(k.path)).lower())
+
+        res = []
+        for error in sorted(self.validator.iter_errors(event), key=sortkey):
+            res.append({
+                "error": 460,
+                "message": "Validation error: key \"%s\", value \"%s\"" % (
+                    "/".join(str(v) for v in error.path),
+                    error.instance
+                ),
+                "expected": error.schema.get('description', 'no additional info')
+            })
+
+        return res
+
+
+class MySQL(ObjectBase):
+
+    def __init__(
+            self, req, log, host, user, password, dbname, port, retry_count,
+            retry_pause, event_size_limit, catmap_filename, tagmap_filename):
+        ObjectBase.__init__(self, req, log)
+        self.host = host
+        self.user = user
+        self.password = password
+        self.dbname = dbname
+        self.port = port
+        self.retry_count = retry_count
+        self.retry_pause = retry_pause
+        self.retry_attempt = 0
+        self.event_size_limit = event_size_limit
+        self.catmap_filename = catmap_filename
+        self.tagmap_filename = tagmap_filename
+
+        with io.open(catmap_filename, "r", encoding="utf-8") as catmap_fd:
+            self.catmap = json.load(catmap_fd)
+            self.catmap_other = self.catmap["Other"]    # Catch error soon, avoid lookup later
+
+        with io.open(tagmap_filename, "r", encoding="utf-8") as tagmap_fd:
+            self.tagmap = json.load(tagmap_fd)
+            self.tagmap_other = self.tagmap["Other"]    # Catch error soon, avoid lookup later
+
+        self.con = None
+
+    def connect(self):
+        self.con = my.connect(
+            host=self.host, user=self.user, passwd=self.password,
+            db=self.dbname, port=self.port, cursorclass=mycursors.DictCursor)
+
+    def close(self):
+        try:
+            if self.con:
+                self.con.close()
+        except Exception:
+            pass
+        self.con = None
+
+    __del__ = close
+
+    def repeat(self):
+        """ Allows for graceful repeating of transactions self.retry_count
+            times. Unsuccessful attempts wait for self.retry_pause until
+            next attempt.
+
+            Meant for usage with context manager:
+
+            for attempt in self.repeat():
+                with attempt as db:
+                    crs = db.query(...)
+                    # do something with crs
+
+            Note that it's not reentrant (as is not underlying MySQL
+            connection), so avoid nesting on the same MySQL object.
+        """
+        self.retry_attempt = self.retry_count
+        while self.retry_attempt:
+            if self.retry_attempt != self.retry_count:
+                sleep(self.retry_pause)
+            self.retry_attempt -= 1
+            yield self
+
+    def __enter__(self):
+        """ Context manager protocol. Guarantees that transaction will
+            get either commited or rolled back in case of database
+            exception. Can be used with self.repeat(), or alone as:
+
+            with self as db:
+                crs = db.query(...)
+                # do something with crs
+
+            Note that it's not reentrant (as is not underlying MySQL
+            connection), so avoid nesting on the same MySQL object.
+        """
+        if not self.retry_attempt:
+            self.retry_attempt = 0
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """ Context manager protocol. If db exception is fired and
+            self.retry_attempt is not zero, it is only logged and
+            does not propagate, otherwise it propagates up. Also
+            open transaction is rolled back.
+            In case of no exception, transaction gets commited.
+        """
+        if not exc_type:
+            self.con.commit()
+            self.retry_attempt = 0
+        else:
+            try:
+                if self.con:
+                    self.con.rollback()
+            except my.Error:
+                pass
+            try:
+                self.close()
+            except my.Error:
+                pass
+            if self.retry_attempt:
+                self.log.info("Database error (%d attempts left): %s %s" % (self.retry_attempt, exc_type.__name__, exc_val))
+                return True
+
+    def query(self, *args, **kwargs):
+        if not self.con:
+            self.connect()
+        crs = self.con.cursor()
+        self.log.debug("execute: %s %s" % (args, kwargs))
+        crs.execute(*args, **kwargs)
+        return crs
+
+    def _get_comma_perc(self, l):
+        return ','.join(['%s'] * len(l))
+
+    def _get_not(self, b):
+        return "" if b else "NOT"
+
+    def get_client_by_name(self, cert_names=None, name=None, secret=None):
+        query = ["SELECT * FROM clients WHERE valid = 1"]
+        params = []
+        if name:
+            query.append(" AND name = %s")
+            params.append(name.lower())
+        if secret:
+            query.append(" AND secret = %s")
+            params.append(secret)
+        if cert_names:
+            query.append(" AND hostname IN (%s)" % self._get_comma_perc(cert_names))
+            params.extend(n.lower() for n in cert_names)
+
+        for attempt in self.repeat():
+            with attempt as db:
+                rows = db.query("".join(query), params).fetchall()
+                if len(rows) > 1:
+                    self.log.warning(
+                        "get_client_by_name: query returned more than one result (cert_names = %s, name = %s, secret = %s): %s" % (
+                            cert_names, name, secret, ", ".join([str(Client(**row)) for row in rows])))
+                    return None
+
+                return Client(**rows[0]) if rows else None
+
+    def get_clients(self, id=None):
+        query = ["SELECT * FROM clients"]
+        params = []
+        if id:
+            query.append("WHERE id = %s")
+            params.append(id)
+        query.append("ORDER BY id")
+        for attempt in self.repeat():
+            with attempt as db:
+                rows = db.query(" ".join(query), params).fetchall()
+                return [Client(**row) for row in rows]
+
+    def add_modify_client(self, id=None, **kwargs):
+        query = []
+        params = []
+        uquery = []
+        if id is None:
+            query.append("INSERT INTO clients SET")
+            uquery.append("registered = now()")
+        else:
+            query.append("UPDATE clients SET")
+        for attr in set(Client._fields) - set(["id", "registered"]):
+            val = kwargs.get(attr, None)
+            if val is not None:
+                if attr == "secret" and val == "":  # disable secret
+                    val = None
+                uquery.append("`%s` = %%s" % attr)
+                params.append(val)
+        if not uquery:
+            return id
+        query.append(", ".join(uquery))
+        if id is not None:
+            query.append("WHERE id = %s")
+            params.append(id)
+        for attempt in self.repeat():
+            with attempt as db:
+                crs = db.query(" ".join(query), params)
+                newid = crs.lastrowid if id is None else id
+                return newid
+
+    def get_debug(self):
+        for attempt in self.repeat():
+            with attempt as db:
+                rows = db.query("SELECT VERSION() AS VER").fetchall()
+                tablestat = db.query("SHOW TABLE STATUS").fetchall()
+                return {
+                    "db": "MySQL",
+                    "version": rows[0]["VER"],
+                    "tables": tablestat
+                }
+
+    def getMaps(self, section, variables):
+        maps = []
+        for v in variables:
+            try:
+                mapped = section[v]
+            except KeyError:
+                raise self.req.error(
+                    message="Wrong tag or category used in query.",
+                    error=422, exc=sys.exc_info(), key=v)
+            maps.append(mapped)
+        return set(maps)    # unique
+
+    def fetch_events(
+            self, client, id, count,
+            cat=None, nocat=None,
+            tag=None, notag=None,
+            group=None, nogroup=None):
+
+        if cat and nocat:
+            raise self.req.error(
+                message="Unrealizable conditions. Choose cat or nocat option.",
+                error=422, cat=cat, nocat=nocat)
+        if tag and notag:
+            raise self.req.error(
+                message="Unrealizable conditions. Choose tag or notag option.",
+                error=422, tag=tag, notag=notag)
+        if group and nogroup:
+            raise self.req.error(
+                message="Unrealizable conditions. Choose group or nogroup option.",
+                error=422, group=group, nogroup=nogroup)
+
+        query = ["SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s"]
+        params = [id or 0]
+
+        if cat or nocat:
+            cats = self.getMaps(self.catmap, (cat or nocat))
+            query.append(
+                " AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE category_id IN (%s))" % (
+                    self._get_not(cat), self._get_comma_perc(cats)))
+            params.extend(cats)
+
+        if tag or notag:
+            tags = self.getMaps(self.tagmap, (tag or notag))
+            query.append(
+                " AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE tag_id IN (%s))" % (
+                    self._get_not(tag), self._get_comma_perc(tags)))
+            params.extend(tags)
+
+        if group or nogroup:
+            subquery = []
+            for name in (group or nogroup):
+                escaped_name = name.replace('&', '&&').replace("_", "&_").replace("%", "&%")  # escape for LIKE
+                subquery.append("c.name = %s")                                # exact client
+                params.append(name)
+                subquery.append("c.name LIKE CONCAT(%s, '.%%') ESCAPE '&'")   # whole subtree
+                params.append(escaped_name)
+
+            query.append(" AND %s (%s)" % (self._get_not(group), " OR ".join(subquery)))
+
+        query.append(" AND e.valid = 1 LIMIT %s")
+        params.append(count)
+
+        query_string = "".join(query)
+
+        row = None
+        for attempt in self.repeat():
+            with attempt as db:
+                row = db.query(query_string, params).fetchall()
+
+        if row:
+            maxid = max(r['id'] for r in row)
+        else:
+            maxid = self.getLastEventId()
+
+        events = []
+        for r in row:
+            try:
+                e = json.loads(r["data"])
+            except Exception:
+                # Note that we use Error object just for proper formatting,
+                # but do not raise it; from client perspective invalid
+                # events get skipped silently.
+                err = self.req.error(
+                    message="Unable to deserialize JSON event from db, id=%s" % r["id"],
+                    error=500, exc=sys.exc_info(), id=r["id"])
+                err.log(self.log, prio=logging.WARNING)
+            events.append(e)
+
+        return {
+            "lastid": maxid,
+            "events": events
+        }
+
+    def store_events(self, client, events, events_raw):
+        try:
+            for attempt in self.repeat():
+                with attempt as db:
+                    for event, raw_event in zip(events, events_raw):
+                        lastid = db.query(
+                            "INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)",
+                            (client.id, raw_event)).lastrowid
+
+                        catlist = event.get('Category', ["Other"])
+                        cats = set(catlist) | set(cat.split(".", 1)[0] for cat in catlist)
+                        for cat in cats:
+                            cat_id = self.catmap.get(cat, self.catmap_other)
+                            db.query("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))
+
+                        nodes = event.get('Node', [])
+                        tags = []
+                        for node in nodes:
+                            tags.extend(node.get('Type', []))
+                        for tag in set(tags):
+                            tag_id = self.tagmap.get(tag, self.tagmap_other)
+                            db.query("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))
+                    return []
+        except Exception as e:
+            exception = self.req.error(message="DB error", error=500, exc=sys.exc_info(), env=self.req.env)
+            exception.log(self.log)
+            return [{"error": 500, "message": "DB error %s" % type(e).__name__}]
+
+    def insertLastReceivedId(self, client, id):
+        self.log.debug("insertLastReceivedId: id %i for client %i(%s)" % (id, client.id, client.hostname))
+        for attempt in self.repeat():
+            with attempt as db:
+                db.query("INSERT INTO last_events(client_id, event_id, timestamp) VALUES(%s, %s, NOW())", (client.id, id))
+
+    def getLastEventId(self):
+        for attempt in self.repeat():
+            with attempt as db:
+                row = db.query("SELECT MAX(id) as id FROM events").fetchall()[0]
+                return row['id'] or 1
+
+    def getLastReceivedId(self, client):
+        for attempt in self.repeat():
+            with attempt as db:
+                res = db.query(
+                    "SELECT event_id as id FROM last_events WHERE client_id = %s ORDER BY last_events.id DESC LIMIT 1",
+                    (client.id,)).fetchall()
+                try:
+                    row = res[0]
+                except IndexError:
+                    id = None
+                    self.log.debug("getLastReceivedId: probably first access, unable to get id for client %i(%s)" % (
+                        client.id, client.hostname))
+                else:
+                    id = row["id"]
+                    self.log.debug("getLastReceivedId: id %i for client %i(%s)" % (id, client.id, client.hostname))
+
+                return id
+
+    def load_maps(self):
+        with self as db:
+            db.query("DELETE FROM tags")
+            for tag, num in self.tagmap.items():
+                db.query("INSERT INTO tags(id, tag) VALUES (%s, %s)", (num, tag))
+            db.query("DELETE FROM categories")
+            for cat_subcat, num in self.catmap.items():
+                catsplit = cat_subcat.split(".", 1)
+                category = catsplit[0]
+                subcategory = catsplit[1] if len(catsplit) > 1 else None
+                db.query(
+                    "INSERT INTO categories(id, category, subcategory, cat_subcat) VALUES (%s, %s, %s, %s)",
+                    (num, category, subcategory, cat_subcat))
+
+    def purge_lastlog(self, days):
+        with self as db:
+            return db.query(
+                "DELETE FROM last_events "
+                " USING last_events LEFT JOIN ("
+                "    SELECT MAX(id) AS last FROM last_events"
+                "    GROUP BY client_id"
+                " ) AS maxids ON last=id"
+                " WHERE timestamp < DATE_SUB(CURDATE(), INTERVAL %s DAY) AND last IS NULL",
+                (days,)).rowcount
+
+    def purge_events(self, days):
+        with self as db:
+            affected = 0
+            id_ = db.query(
+                "SELECT MAX(id) as id"
+                "  FROM events"
+                "  WHERE received < DATE_SUB(CURDATE(), INTERVAL %s DAY)",
+                (days,)
+            ).fetchall()[0]["id"]
+            if id_ is None:
+                return 0
+            affected = db.query("DELETE FROM events WHERE id <= %s", (id_,)).rowcount
+            db.query("DELETE FROM event_category_mapping WHERE event_id <= %s", (id_,))
+            db.query("DELETE FROM event_tag_mapping WHERE event_id <= %s", (id_,))
+            return affected
+
+
+def expose(read=1, write=0, debug=0):
+
+    def expose_deco(meth):
+        meth.exposed = True
+        meth.read = read
+        meth.write = write
+        meth.debug = debug
+        if not hasattr(meth, "arguments"):
+            meth.arguments = get_method_params(meth)
+        return meth
+
+    return expose_deco
+
+
+class Server(ObjectBase):
+
+    def __init__(self, req, log, auth, handler):
+        ObjectBase.__init__(self, req, log)
+        self.auth = auth
+        self.handler = handler
+
+    def sanitize_args(self, path, func, args, exclude=["self", "post"]):
+        # silently remove internal args, these should never be used
+        # but if somebody does, we do not expose them by error message
+        intargs = set(args).intersection(exclude)
+        for a in intargs:
+            del args[a]
+        if intargs:
+            self.log.info("sanitize_args: Called with internal args: %s" % ", ".join(intargs))
+
+        # silently remove surplus arguments - potential forward
+        # compatibility (unknown args will get ignored)
+        badargs = set(args) - set(func.arguments)
+        for a in badargs:
+            del args[a]
+        if badargs:
+            self.log.info("sanitize_args: Called with superfluous args: %s" % ", ".join(badargs))
+
+        return args
+
+    def wsgi_app(self, environ, start_response, exc_info=None):
+        path = environ.get("PATH_INFO", "").lstrip("/")
+        self.req.reset(env=environ, path=path)
+        output = ""
+        status = "200 OK"
+        headers = [('Content-type', 'application/json')]
+        exception = None
+
+        try:
+            try:
+                method = getattr(self.handler, path)
+                method.exposed    # dummy access to trigger AttributeError
+            except Exception:
+                raise self.req.error(message="You've fallen off the cliff.", error=404)
+
+            self.req.args = args = parse_qs(environ.get('QUERY_STRING', ""))
+
+            self.req.client = client = self.auth.authenticate(environ, args)
+            if not client:
+                raise self.req.error(message="I'm watching. Authenticate.", error=403)
+
+            auth = self.auth.authorize(self.req.env, self.req.client, self.req.path, method)
+            if not auth:
+                raise self.req.error(message="I'm watching. Not authorized.", error=403, client=client.name)
+
+            args = self.sanitize_args(path, method, args)
+
+            # Based on RFC2616, section 4.4 we SHOULD respond with 400 (bad request) or 411
+            # (length required) if content length was not specified. We choose not to, to
+            # preserve compatibility with clients deployed in the wild, which use POST for
+            # all requests (even those without payload, with no specified content length).
+            # According to PEP3333, section "Input and Error Streams", the application SHOULD
+            # NOT attempt to read more data than specified by CONTENT_LENGTH. As stated in
+            # section "environ Variables", CONTENT_LENGTH may be empty (string) or absent.
+            try:
+                content_length = int(environ.get('CONTENT_LENGTH', 0))
+            except ValueError:
+                content_length = 0
+
+            try:
+                post_data = environ['wsgi.input'].read(content_length)
+            except:
+                raise self.req.error(message="Data read error.", error=408, exc=sys.exc_info())
+
+            headers, output = method(post_data, **args)
+
+        except Error as e:
+            exception = e
+        except Exception as e:
+            exception = self.req.error(message="Server exception", error=500, exc=sys.exc_info())
+
+        if exception:
+            status = "%d %s" % exception.get_http_err_msg()
+            output = json.dumps(exception.to_dict(), default=lambda v: str(v))
+            exception.log(self.log)
+
+        # Make sure everything is properly encoded - JSON and various function
+        # may spit out unicode instead of str and it gets propagated up (str
+        # + unicode = unicode).
+        # For Python2 the right thing would be to be unicode correct among whole
+        # source and always decode on input (json module does that for us) and
+        # on output here.
+        # For Python3 strings are internally unicode so no decoding on input is
+        # necessary. For output, "status" must be unicode string, "output" must
+        # be encoded bytes array, what is done here. Important: for Python 3 we
+        # define: unicode = str
+        if isinstance(status, unicode) and sys.version_info[0] < 3:
+            status = status.encode("utf-8")
+        if isinstance(output, unicode):
+            output = output.encode("utf-8")
+        headers.append(('Content-Length', str(len(output))))
+        start_response(status, headers)
+        self.req.reset()
+        return [output]
+
+    __call__ = wsgi_app
+
+
+def json_wrapper(method):
+    def meth_deco(self, post, **args):
+        if "events" in get_method_params(method):
+            try:
+                events = json.loads(post.decode('utf-8')) if post else None
+            except Exception as e:
+                raise self.req.error(
+                    message="Deserialization error.", error=400,
+                    exc=sys.exc_info(), args=post, parser=str(e))
+            if events:
+                args["events"] = events
+
+        result = method(self, **args)   # call requested method
+
+        try:
+            # 'default': takes care of non JSON serializable objects,
+            # which could (although shouldn't) appear in handler code
+            output = json.dumps(result, default=lambda v: str(v))
+        except Exception as e:
+            raise self.req.error(message="Serialization error", error=500, exc=sys.exc_info(), args=str(result))
+
+        return [('Content-type', 'application/json')], output
+
+    try:
+        meth_deco.arguments = method.arguments
+    except AttributeError:
+        meth_deco.arguments = get_method_params(method)
+    return meth_deco
+
+
+class WardenHandler(ObjectBase):
+
+    def __init__(
+            self, req, log, validator, db, auth,
+            send_events_limit=500, get_events_limit=1000,
+            description=None):
+
+        ObjectBase.__init__(self, req, log)
+        self.auth = auth
+        self.db = db
+        self.validator = validator
+        self.send_events_limit = send_events_limit
+        self.get_events_limit = get_events_limit
+        self.description = description
+
+    @expose(read=1, debug=1)
+    @json_wrapper
+    def getDebug(self):
+        return {
+            "environment": self.req.env,
+            "client": self.req.client._asdict(),
+            "database": self.db.get_debug(),
+            "system": {
+                "python": sys.version,
+                "uname": os.uname()
+            },
+            "process": {
+                "cwd": unicode(os.getcwd()),
+                "pid": os.getpid(),
+                "ppid": os.getppid(),
+                "pgrp": os.getpgrp(),
+                "uid": os.getuid(),
+                "gid": os.getgid(),
+                "euid": os.geteuid(),
+                "egid": os.getegid(),
+                "groups": os.getgroups()
+            }
+        }
+
+    @expose(read=1)
+    @json_wrapper
+    def getInfo(self):
+        info = {
+            "version": VERSION,
+            "send_events_limit": self.send_events_limit,
+            "get_events_limit": self.get_events_limit
+        }
+        if self.description:
+            info["description"] = self.description
+        return info
+
+    @expose(read=1)
+    @json_wrapper
+    def getEvents(
+            self, id=None, count=None,
+            cat=None, nocat=None,
+            tag=None, notag=None,
+            group=None, nogroup=None):
+
+        try:
+            id = int(id[0])
+        except (ValueError, TypeError, IndexError):
+            id = None
+
+        if id is None:
+            # If client was already here, fetch server notion of his last id
+            try:
+                id = self.db.getLastReceivedId(self.req.client)
+            except Exception as e:
+                self.log.info("cannot getLastReceivedId - " + type(e).__name__ + ": " + str(e))
+
+        if id is None:
+            # First access, remember the guy and get him last id
+            id = self.db.getLastEventId()
+            self.db.insertLastReceivedId(self.req.client, id)
+            return {
+                "lastid": id,
+                "events": []
+            }
+
+        if id <= 0:
+            # Client wants to get only last N events and reset server notion of last id
+            id += self.db.getLastEventId()
+            if id < 0: id = 0
+
+        try:
+            count = int(count[0])
+        except (ValueError, TypeError, IndexError):
+            count = self.get_events_limit
+
+        if self.get_events_limit:
+            count = min(count, self.get_events_limit)
+        count = max(0, count)
+
+        res = self.db.fetch_events(self.req.client, id, count, cat, nocat, tag, notag, group, nogroup)
+
+        self.db.insertLastReceivedId(self.req.client, res['lastid'])
+
+        self.log.info("sending %d events, lastid is %i" % (len(res["events"]), res["lastid"]))
+
+        return res
+
+    def check_node(self, event, name):
+        try:
+            ev_id = event['Node'][0]['Name'].lower()
+        except (KeyError, TypeError, IndexError):
+            # Event does not bear valid Node attribute
+            return [{"error": 422, "message": "Event does not bear valid Node attribute"}]
+        if ev_id != name:
+            return [{"error": 422, "message": "Node does not correspond with saving client"}]
+        return []
+
+    def add_event_nums(self, ilist, events, errlist):
+        for err in errlist:
+            err.setdefault("events", []).extend(ilist)
+            ev_ids = err.setdefault("events_id", [])
+            for i in ilist:
+                event = events[i]
+                try:
+                    id = event["ID"]
+                except (KeyError, TypeError, ValueError):
+                    id = None
+                ev_ids.append(id)
+        return errlist
+
+    @expose(write=1)
+    @json_wrapper
+    def sendEvents(self, events=[]):
+        if not isinstance(events, list):
+            raise self.req.error(message="List of events expected.", error=400)
+
+        errs = []
+        if len(events) > self.send_events_limit:
+            errs.extend(self.add_event_nums(range(self.send_events_limit, len(events)), events, [
+                {"error": 507, "message": "Too much events in one batch.", "send_events_limit": self.send_events_limit}]))
+
+        saved = 0
+        events_tosend = []
+        events_raw = []
+        events_nums = []
+        for i, event in enumerate(events[0:self.send_events_limit]):
+            v_errs = self.validator.check(event)
+            if v_errs:
+                errs.extend(self.add_event_nums([i], events, v_errs))
+                continue
+
+            node_errs = self.check_node(event, self.req.client.name)
+            if node_errs:
+                errs.extend(self.add_event_nums([i], events, node_errs))
+                continue
+
+            if self.req.client.test and 'Test' not in event.get('Category', []):
+                errs.extend(
+                    self.add_event_nums([i], events, [{
+                        "error": 422,
+                        "message": "You're allowed to send only messages, containing \"Test\" among categories.",
+                        "categories": event.get('Category', [])}]))
+                continue
+
+            raw_event = json.dumps(event)
+            if len(raw_event) >= self.db.event_size_limit:
+                errs.extend(
+                    self.add_event_nums([i], events, [
+                        {"error": 413, "message": "Event too long (>%i B)" % self.db.event_size_limit}
+                    ]))
+                continue
+
+            events_tosend.append(event)
+            events_raw.append(raw_event)
+            events_nums.append(i)
+
+        db_errs = self.db.store_events(self.req.client, events_tosend, events_raw)
+        if db_errs:
+            errs.extend(self.add_event_nums(events_nums, events_tosend, db_errs))
+            saved = 0
+        else:
+            saved = len(events_tosend)
+
+        self.log.info("Saved %i events" % saved)
+        if errs:
+            raise self.req.error(errors=errs)
+
+        return {"saved": saved}
+
+
+def read_ini(path):
+    c = ConfigParser.RawConfigParser()
+    res = c.read(path)
+    if not res or path not in res:
+        # We don't have loggin yet, hopefully this will go into webserver log
+        raise Error(message="Unable to read config: %s" % path)
+    data = {}
+    for sect in c.sections():
+        for opts in c.options(sect):
+            lsect = sect.lower()
+            if lsect not in data:
+                data[lsect] = {}
+            data[lsect][opts] = c.get(sect, opts)
+    return data
+
+
+def read_cfg(path):
+    with io.open(path, "r", encoding="utf-8") as f:
+        stripcomments = "\n".join((l for l in f if not l.lstrip().startswith(("#", "//"))))
+        conf = json.loads(stripcomments)
+
+    # Lowercase keys
+    conf = dict((
+        sect.lower(), dict(
+            (subkey.lower(), val) for subkey, val in subsect.items())
+    ) for sect, subsect in conf.items())
+
+    return conf
+
+
+def fallback_wsgi(environ, start_response, exc_info=None):
+
+    # If server does not start, set up simple server, returning
+    # Warden JSON compliant error message
+    error = 503
+    message = "Server not running due to initialization error"
+    headers = [('Content-type', 'application/json')]
+
+    logline = "Error(%d): %s" % (error, message)
+    status = "%d %s" % (error, message)
+    output = '{"errors": [{"error": %d, "message": "%s"}]}' % (
+        error, message)
+
+    logging.getLogger(__name__).critical(logline)
+    start_response(status, headers)
+    return [output]
+
+
+# Order in which the base objects must get initialized
+section_order = ("log", "db", "auth", "validator", "handler", "server")
+
+# List of sections and objects, configured by them
+# First object in each object list is the default one, otherwise
+# "type" keyword in section may be used to choose other
+section_def = {
+    "log": [FileLogger, SysLogger],
+    "db": [MySQL],
+    "auth": [X509NameAuthenticator, PlainAuthenticator, X509Authenticator, X509MixMatchAuthenticator],
+    "validator": [JSONSchemaValidator, NoValidator],
+    "handler": [WardenHandler],
+    "server": [Server]
+}
+
+# Object parameter conversions and defaults
+param_def = {
+    FileLogger: {
+        "req": {"type": "obj", "default": "req"},
+        "filename": {"type": "filepath", "default": path.join(path.dirname(__file__), path.splitext(path.split(__file__)[1])[0] + ".log")},
+        "level": {"type": "loglevel", "default": "info"},
+    },
+    SysLogger: {
+        "req": {"type": "obj", "default": "req"},
+        "socket": {"type": "filepath", "default": "/dev/log"},
+        "facility": {"type": "facility", "default": "daemon"},
+        "level": {"type": "loglevel", "default": "info"}
+    },
+    PlainAuthenticator: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "db": {"type": "obj", "default": "db"}
+    },
+    X509Authenticator: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "db": {"type": "obj", "default": "db"}
+    },
+    X509NameAuthenticator: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "db": {"type": "obj", "default": "db"}
+    },
+    X509MixMatchAuthenticator: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "db": {"type": "obj", "default": "db"}
+    },
+    NoValidator: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+    },
+    JSONSchemaValidator: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "filename": {"type": "filepath", "default": path.join(path.dirname(__file__), "idea.schema")}
+    },
+    MySQL: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "host": {"type": "str", "default": "localhost"},
+        "user": {"type": "str", "default": "warden"},
+        "password": {"type": "str", "default": ""},
+        "dbname": {"type": "str", "default": "warden3"},
+        "port": {"type": "natural", "default": 3306},
+        "retry_pause": {"type": "natural", "default": 3},
+        "retry_count": {"type": "natural", "default": 3},
+        "event_size_limit": {"type": "natural", "default": 5*1024*1024},
+        "catmap_filename": {"type": "filepath", "default": path.join(path.dirname(__file__), "catmap_mysql.json")},
+        "tagmap_filename": {"type": "filepath", "default": path.join(path.dirname(__file__), "tagmap_mysql.json")}
+    },
+    WardenHandler: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "validator": {"type": "obj", "default": "validator"},
+        "db": {"type": "obj", "default": "DB"},
+        "auth": {"type": "obj", "default": "auth"},
+        "send_events_limit": {"type": "natural", "default": 500},
+        "get_events_limit": {"type": "natural", "default": 1000},
+        "description": {"type": "str", "default": ""}
+    },
+    Server: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "auth": {"type": "obj", "default": "auth"},
+        "handler": {"type": "obj", "default": "handler"}
+    }
+}
+
+
+def build_server(conf, section_order=section_order, section_def=section_def, param_def=param_def):
+
+    objects = {}    # Already initialized objects
+
+    # Functions for validation and conversion of config values
+    def facility(name):
+        return int(getattr(logging.handlers.SysLogHandler, "LOG_" + name.upper()))
+
+    def loglevel(name):
+        return int(getattr(logging, name.upper()))
+
+    def natural(name):
+        num = int(name)
+        if num < 1:
+            raise ValueError("Not a natural number")
+        return num
+
+    def filepath(name):
+        # Make paths relative to dir of this script
+        return path.join(path.dirname(__file__), name)
+
+    def obj(name):
+        return objects[name.lower()]
+
+    # Typedef dictionary
+    conv_dict = {
+        "facility": facility,
+        "loglevel": loglevel,
+        "natural": natural,
+        "filepath": filepath,
+        "obj": obj,
+        "str": str
+    }
+
+    def init_obj(sect_name):
+        config = dict(conf.get(sect_name, {}))
+        sect_name = sect_name.lower()
+        sect_def = section_def[sect_name]
+
+        try:    # Object type defined?
+            objtype = config["type"]
+            del config["type"]
+        except KeyError:    # No, fetch default object type for this section
+            cls = sect_def[0]
+        else:  # Yes, get corresponding class/callable
+            names = [o.__name__ for o in sect_def]
+            try:
+                idx = names.index(objtype)
+            except ValueError:
+                raise KeyError("Unknown type %s in section %s" % (objtype, sect_name))
+            cls = sect_def[idx]
+
+        params = param_def[cls]
+
+        # No surplus parameters? Disallow also 'obj' attributes, these are only
+        # to provide default referenced section
+        for name in config:
+            if name not in params or (name in params and params[name]["type"] == "obj"):
+                raise KeyError("Unknown key %s in section %s" % (name, sect_name))
+
+        # Process parameters
+        kwargs = {}
+        for name, definition in params.items():
+            raw_val = config.get(name, definition["default"])
+            try:
+                type_callable = conv_dict[definition["type"]]
+                val = type_callable(raw_val)
+            except Exception:
+                raise KeyError("Bad value \"%s\" for %s in section %s" % (raw_val, name, sect_name))
+            kwargs[name] = val
+
+        try:
+            obj_inst = cls(**kwargs)         # run it
+        except Exception as e:
+            raise KeyError("Cannot initialize %s from section %s: %s" % (
+                cls.__name__, sect_name, str(e)))
+
+        objects[sect_name] = obj_inst
+        if isinstance(obj_inst, Object):
+            # Log only objects here, functions must take care of themselves
+            objects["log"].info("Initialized %s" % str(obj_inst))
+
+        return obj_inst
+
+    # Init logging with at least simple stderr StreamLogger
+    # Dunno if it's ok within wsgi, but we have no other choice, let's
+    # hope it at least ends up in webserver error log
+    objects["log"] = StreamLogger()
+
+    # Shared container for common data of ongoing WSGI request
+    objects["req"] = Request()
+
+    try:
+        # Now try to init required objects
+        for o in section_order:
+            init_obj(o)
+    except Exception as e:
+        objects["log"].critical(str(e))
+        objects["log"].debug("", exc_info=sys.exc_info())
+        return fallback_wsgi
+
+    objects["log"].info("Server ready")
+
+    return objects["server"]
+
+
+# Command line utilities
+
+def check_config():
+    # If we got so far, server object got set up fine
+    print("Looks clear.", file=sys.stderr)
+    return 0
+
+
+def list_clients(id=None):
+    clients = server.handler.db.get_clients(id)
+    lines = [[str(getattr(client, col)) for col in Client._fields] for client in clients]
+    col_width = [max(len(val) for val in col) for col in zip(*(lines+[Client._fields]))]
+    divider = ["-" * l for l in col_width]
+    for line in [Client._fields, divider] + lines:
+        print(" ".join([val.ljust(width) for val, width in zip(line, col_width)]))
+    return 0
+
+
+def register_client(**kwargs):
+    # argparse does _always_ return something, so we cannot rely on missing arguments
+    if kwargs["valid"] is None: kwargs["valid"] = 1
+    if kwargs["read"] is None: kwargs["read"] = 1
+    if kwargs["write"] is None: kwargs["write"] = 0
+    if kwargs["debug"] is None: kwargs["debug"] = 0
+    if kwargs["test"] is None: kwargs["test"] = 1
+    return modify_client(id=None, **kwargs)
+
+
+def modify_client(**kwargs):
+
+    def isValidHostname(hostname):
+        if len(hostname) > 255:
+            return False
+        if hostname.endswith("."):  # A single trailing dot is legal
+            hostname = hostname[:-1]  # strip exactly one dot from the right, if present
+        disallowed = re.compile(r"[^A-Z\d-]", re.IGNORECASE)
+        return all(  # Split by labels and verify individually
+            (label and len(label) <= 63  # length is within proper range
+             and not label.startswith("-") and not label.endswith("-")  # no bordering hyphens
+             and not disallowed.search(label))  # contains only legal characters
+            for label in hostname.split("."))
+
+    def isValidNSID(nsid):
+        allowed = re.compile(r"^(?:[a-zA-Z_][a-zA-Z0-9_]*\.)*[a-zA-Z_][a-zA-Z0-9_]*$")
+        return allowed.match(nsid)
+
+    def isValidEmail(mail):
+        allowed = re.compile(r"(^[a-zA-Z0-9_ .%!+-]*(?=<.*>))?(^|(<(?=.*(>))))[a-zA-Z0-9_.%!+-]+@[a-zA-Z0-9-.]+\4?$")   # just basic check
+        valid = (allowed.match(ms.strip())for ms in mail.split(','))
+        return all(valid)
+
+    def isValidID(id):
+        client = server.handler.db.get_clients(id)
+        return client and True or False
+
+    if kwargs["name"] is not None:
+        kwargs["name"] = kwargs["name"].lower()
+        if not isValidNSID(kwargs["name"]):
+            print("Invalid client name \"%s\"." % kwargs["name"], file=sys.stderr)
+            return 254
+
+    if kwargs["hostname"] is not None:
+        kwargs["hostname"] = kwargs["hostname"].lower()
+        if not isValidHostname(kwargs["hostname"]):
+            print("Invalid hostname \"%s\"." % kwargs["hostname"], file=sys.stderr)
+            return 253
+
+    if kwargs["requestor"] is not None and not isValidEmail(kwargs["requestor"]):
+        print("Invalid requestor email \"%s\"." % kwargs["requestor"], file=sys.stderr)
+        return 252
+
+    if kwargs["id"] is not None and not isValidID(kwargs["id"]):
+        print("Invalid id \"%s\"." % kwargs["id"], file=sys.stderr)
+        return 251
+
+    for c in server.handler.db.get_clients():
+        if kwargs["name"] is not None and kwargs["name"].lower() == c.name:
+            print("Clash with existing name: %s" % str(c), file=sys.stderr)
+            return 250
+        if kwargs["secret"] is not None and kwargs["secret"] == c.secret:
+            print("Clash with existing secret: %s" % str(c), file=sys.stderr)
+            return 249
+
+    newid = server.handler.db.add_modify_client(**kwargs)
+
+    return list_clients(id=newid)
+
+
+def load_maps():
+    server.handler.db.load_maps()
+    return 0
+
+
+def purge(days=30, lastlog=None, events=None):
+    if lastlog is None and events is None:
+        lastlog = events = True
+    if lastlog:
+        count = server.handler.db.purge_lastlog(days)
+        print("Purged %d lastlog entries." % count)
+    if events:
+        count = server.handler.db.purge_events(days)
+        print("Purged %d events." % count)
+    return 0
+
+
+def add_client_args(subargp, mod=False):
+    subargp.add_argument("--help", action="help", help="show this help message and exit")
+    if mod:
+        subargp.add_argument(
+            "-i", "--id", required=True, type=int,
+            help="client id")
+    subargp.add_argument(
+        "-n", "--name", required=not mod,
+        help="client name (in dotted reverse path notation)")
+    subargp.add_argument(
+        "-h", "--hostname", required=not mod,
+        help="client FQDN hostname")
+    subargp.add_argument(
+        "-r", "--requestor", required=not mod,
+        help="requestor email")
+    subargp.add_argument(
+        "-s", "--secret",
+        help="authentication token (use explicit empty string to disable)")
+    subargp.add_argument(
+        "--note",
+        help="client freetext description")
+
+    reg_valid = subargp.add_mutually_exclusive_group(required=False)
+    reg_valid.add_argument(
+        "--valid", action="store_const", const=1, default=None,
+        help="valid client (default)")
+    reg_valid.add_argument("--novalid", action="store_const", const=0, dest="valid", default=None)
+
+    reg_read = subargp.add_mutually_exclusive_group(required=False)
+    reg_read.add_argument(
+        "--read", action="store_const", const=1, default=None,
+        help="client is allowed to read (default)")
+    reg_read.add_argument("--noread", action="store_const", const=0, dest="read", default=None)
+
+    reg_write = subargp.add_mutually_exclusive_group(required=False)
+    reg_write.add_argument(
+        "--nowrite", action="store_const", const=0, dest="write", default=None,
+        help="client is allowed to send (default - no)")
+    reg_write.add_argument("--write", action="store_const", const=1, default=None)
+
+    reg_debug = subargp.add_mutually_exclusive_group(required=False)
+    reg_debug.add_argument(
+        "--nodebug", action="store_const", const=0, dest="debug", default=None,
+        help="client is allowed receive debug output (default - no)")
+    reg_debug.add_argument("--debug", action="store_const", const=1, default=None)
+
+    reg_test = subargp.add_mutually_exclusive_group(required=False)
+    reg_test.add_argument(
+        "--test", action="store_const", const=1, default=None,
+        help="client is yet in testing phase (default - yes)")
+    reg_test.add_argument("--notest", action="store_const", const=0, dest="test", default=None)
+
+
+def get_args():
+    import argparse
+    argp = argparse.ArgumentParser(
+        description="Warden server " + VERSION, add_help=False)
+    argp.add_argument(
+        "--help", action="help",
+        help="show this help message and exit")
+    argp.add_argument(
+        "-c", "--config",
+        help="path to configuration file")
+    subargp = argp.add_subparsers(title="commands", dest="command")
+    subargp.required = True
+
+    subargp_check = subargp.add_parser(
+        "check", add_help=False,
+        description="Try to setup server based on configuration file.",
+        help="check configuration")
+    subargp_check.set_defaults(command=check_config)
+    subargp_check.add_argument(
+        "--help", action="help",
+        help="show this help message and exit")
+
+    subargp_reg = subargp.add_parser(
+        "register", add_help=False,
+        description="Add new client registration entry.",
+        help="register new client")
+    subargp_reg.set_defaults(command=register_client)
+    add_client_args(subargp_reg)
+
+    subargp_mod = subargp.add_parser(
+        "modify", add_help=False,
+        description="Modify details of client registration entry.",
+        help="modify client registration")
+    subargp_mod.set_defaults(command=modify_client)
+    add_client_args(subargp_mod, mod=True)
+
+    subargp_list = subargp.add_parser(
+        "list", add_help=False,
+        description="List details of client registration entries.",
+        help="list registered clients")
+    subargp_list.set_defaults(command=list_clients)
+    subargp_list.add_argument(
+        "--help", action="help",
+        help="show this help message and exit")
+    subargp_list.add_argument(
+        "--id", action="store", type=int,
+        help="client id", default=None)
+
+    subargp_purge = subargp.add_parser(
+        "purge", add_help=False,
+        description=(
+            "Purge old events or lastlog records."
+            " Note that lastlog purge retains at least one newest record for each"
+            " client, even if it is more than number of 'days' old."),
+        help="purge old events or lastlog records")
+    subargp_purge.set_defaults(command=purge)
+    subargp_purge.add_argument(
+        "--help", action="help",
+        help="show this help message and exit")
+    subargp_purge.add_argument(
+        "-l", "--lastlog", action="store_true", dest="lastlog", default=None,
+        help="purge lastlog records")
+    subargp_purge.add_argument(
+        "-e", "--events", action="store_true", dest="events", default=None,
+        help="purge events")
+    subargp_purge.add_argument(
+        "-d", "--days", action="store", dest="days", type=int, default=30,
+        help="records older than 'days' back from today will get purged")
+
+    subargp_loadmaps = subargp.add_parser(
+        "loadmaps", add_help=False,
+        description=(
+            "Load 'categories' and 'tags' table from 'catmap_mysql.json' and 'tagmap_mysql.json'."
+            " Note that this is NOT needed for server at all, load them into db at will,"
+            " should you need to run your own specific SQL queries on data directly."
+            " Note also that previous content of both tables will be lost."),
+        help="load catmap and tagmap into db")
+    subargp_loadmaps.set_defaults(command=load_maps)
+    subargp_loadmaps.add_argument(
+        "--help", action="help",
+        help="show this help message and exit")
+
+    return argp.parse_args()
+
+
+if __name__ == "__main__":
+    args = get_args()
+    config = path.join(path.dirname(__file__), args.config or "warden_server.cfg")
+    server = build_server(read_cfg(config))
+    command = args.command
+    subargs = vars(args)
+    del subargs["command"]
+    del subargs["config"]
+    if not server or server is fallback_wsgi:
+        print("Failed initialization, check configured log targets for reasons.", file=sys.stderr)
+        sys.exit(255)
+    sys.exit(command(**subargs))

warden_server/warden_server.wsgi

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2011-2013 Cesnet z.s.p.o
+# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+
+from sys import path
+from os.path import dirname, join
+
+path.append(dirname(__file__))
+from warden_server import build_server
+
+## JSON configuration with line comments (trailing #)
+from warden_server import read_cfg
+application = build_server(read_cfg(join(dirname(__file__), "warden_server.cfg")))