warden_ra/README.ejbca

+EJBCA backend for Warden 3.# Registration Authority
+===================================================
+
+Introduction
+------------
+
+EJBCA_ is an open source CA management software. To use this backend
+with Warden RA, you need to have it already installed and running.
+Tested with EJBCA_ 3.9.
+
+.. _EJBCA: https://www.ejbca.org/
+
+
+Configuration
+-------------
+
+    Options for "Registry: EjbcaRegistry" section.
+
+    url: EJBCA API URL, for example "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl"
+    cert: certificate for authentication to EJBCA, defaults to "warden_ra.cert.pem"
+    key: key for authentication to EJBCA, defaults to "warden_ra.key.pem"
+    ca_name: name of the CA, dedicated for Warden, defaults to "Example CA"
+    certificate_profile_name: name of the EJBCA certificate profile, defaults to "Example"
+    end_entity_profile_name: name of the EJBCA entity profile, defaults to "Example EE"
+    subject_dn_template: template for the DN generation, defaults to "DC=cz,DC=example-ca,DC=warden,CN=%s"
+    username_suffix: suffix, which will be added to EJBCA entities, defaults to "@warden"
+
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

warden_ra/README.openssl

+OpenSSL local backed for Warden 3.# Registration Authority
+==========================================================
+
+Introduction
+------------
+
+This backend allows using basic `openssl ca`_ facility for certificate
+emission. Client information is kept as plain config files within "clients"
+subdirectory. Also, received CSRs and issued certificates are saved in "csr"
+and "newcerts" subdirectories, respectively. File "lock" is used to conduct
+concurrent access to running openssl binary.
+
+.. _openssl ca: https://www.openssl.org/docs/manmaster/man1/openssl-ca.html
+
+
+Installation
+------------
+
+Choose directory where OpenSSL CA structure will reside (for example
+"ca").
+
+    # mkdir ca
+    # cd ca/
+    /ca# mkdir certs crl newcerts private clients csr
+    /ca# chmod 700 private
+    /ca# touch index.txt
+    /ca# echo 1024 > serial
+
+Adjust permissions.
+
+   # s-bit, so newly created files receive permissions of parent
+   # directory, not of creator
+   ca# find . -type d | xargs chmod g+s
+   # owner - apache group (this is for Debian, adjust accordingly for
+   # different distribution)
+   ca# chgrp -R www-data .
+
+Generate CA root certificate.
+
+    ca# openssl genrsa -out private/ca.key.pem 4096
+    ca# openssl req -config openssl.cnf \
+                -key private/ca.key.pem \
+                -new -x509 -days 7300 -sha256 -extensions v3_ca \
+                -out certs/ca.cert.pem
+    ca# chmod 444 private/ca.key.pem certs/ca.cert.pem
+
+Create "openssl.cnf" in base directory. You can use "openssl.cnf.example" as
+a basis.
+
+
+Configuration
+-------------
+
+    Options for "Registry: OpenSSLRegistry" section.
+
+    base_dir: Base directory where OpenSSL CA environment is managed
+    subject_dn_template: Template for DN of issued certs, defaults to "DC=cz,DC=example-ca,DC=warden,CN=%s"
+    openssl_sign: OpenSSL command and arguments to run for signing, defaults to "openssl ca -config %(cnf)s -batch -extensions server_cert -days 375 -notext -md sha256 -in %(csr)s -subj '%(dn)s'"
+
+
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

warden_ra/apache22.conf

+SSLEngine on
+
+SSLVerifyClient optional
+SSLOptions +StdEnvVars +ExportCertData
+
+SSLCertificateFile      /opt/warden_server/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/key.pem
+SSLCACertificateFile    /opt/warden_server/chain_TERENA_SSL_CA_3.pem
+
+WSGIScriptAlias /warden_ra /opt/warden-ra/warden_ra.wsgi
+
+<Directory /opt/warden-ra/warden_ra.wsgi>
+    Order allow,deny
+    Allow from all
+</Directory>

warden_ra/apache24.conf

+SSLEngine on
+
+SSLVerifyClient optional
+SSLOptions +StdEnvVars +ExportCertData
+
+SSLCertificateFile      /opt/warden_server/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/key.pem
+SSLCACertificateFile    /opt/warden_server/chain_TERENA_SSL_CA_3.pem
+
+WSGIScriptAlias /warden_ra /opt/warden-ra/warden_ra.wsgi
+
+<Directory /opt/warden-ra/warden_ra.wsgi>
+    Require all granted
+</Directory>

warden_ra/ejbcaws.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2016, CESNET, z. s. p. o.
+# Use of this source is governed by an ISC license, see LICENSE file.
+
+import sys
+import socket
+import base64
+import suds.transport.http
+import suds.client
+import M2Crypto
+
+
+if sys.version_info[0] >= 3:
+    import urllib.request, urllib.error, urllib.parse
+    import http.client
+
+    def get_https_handler():
+        return urllib.request.HTTPSHandler
+else:
+    import urllib2
+    import httplib
+
+    def get_https_handler():
+        return urllib2.HTTPSHandler
+
+
+STATUS_FAILED = 11
+STATUS_GENERATED = 40
+STATUS_HISTORICAL = 60
+STATUS_INITIALIZED = 20
+STATUS_INPROCESS = 30
+STATUS_KEYRECOVERY = 70
+STATUS_NEW = 10
+STATUS_REVOKED = 50
+
+MATCH_TYPE_BEGINSWITH = 1
+MATCH_TYPE_CONTAINS = 2
+MATCH_TYPE_EQUALS = 0
+MATCH_WITH_CA = 5
+MATCH_WITH_CERTIFICATEPROFILE = 4
+MATCH_WITH_COMMONNAME = 101
+MATCH_WITH_COUNTRY = 112
+MATCH_WITH_DIRECTORYNAME = 204
+MATCH_WITH_DN = 7
+MATCH_WITH_DNSERIALNUMBER = 102
+MATCH_WITH_DNSNAME = 201
+MATCH_WITH_DOMAINCOMPONENT = 111
+MATCH_WITH_EDIPARTNAME = 205
+MATCH_WITH_EMAIL = 1
+MATCH_WITH_ENDENTITYPROFILE = 3
+MATCH_WITH_GIVENNAME = 103
+MATCH_WITH_GUID = 209
+MATCH_WITH_INITIALS = 104
+MATCH_WITH_IPADDRESS = 202
+MATCH_WITH_LOCALE = 109
+MATCH_WITH_ORGANIZATION = 108
+MATCH_WITH_ORGANIZATIONUNIT = 107
+MATCH_WITH_REGISTEREDID = 207
+MATCH_WITH_RFC822NAME = 200
+MATCH_WITH_STATE = 110
+MATCH_WITH_STATUS = 2
+MATCH_WITH_SURNAME = 105
+MATCH_WITH_TITLE = 106
+MATCH_WITH_TOKEN = 6
+MATCH_WITH_UID = 100
+MATCH_WITH_UPN = 208
+MATCH_WITH_URI = 206
+MATCH_WITH_USERNAME = 0
+MATCH_WITH_X400ADDRESS = 203
+
+TOKEN_TYPE_JKS = "JKS"
+TOKEN_TYPE_P12 = "P12"
+TOKEN_TYPE_PEM = "PEM"
+TOKEN_TYPE_USERGENERATED = "USERGENERATED"
+
+VIEW_RIGHTS = "/view_end_entity"
+EDIT_RIGHTS = "/edit_end_entity"
+CREATE_RIGHTS = "/create_end_entity"
+DELETE_RIGHTS = "/delete_end_entity"
+REVOKE_RIGHTS = "/revoke_end_entity"
+HISTORY_RIGHTS = "/view_end_entity_history"
+APPROVAL_RIGHTS = "/approve_end_entity"
+HARDTOKEN_RIGHTS = "/view_hardtoken"
+HARDTOKEN_PUKDATA_RIGHTS = "/view_hardtoken/puk_data"
+KEYRECOVERY_RIGHTS = "/keyrecovery"
+
+ENDENTITYPROFILEBASE = "/endentityprofilesrules"
+ENDENTITYPROFILEPREFIX = "/endentityprofilesrules/"
+USERDATASOURCEBASE = "/userdatasourcesrules"
+USERDATASOURCEPREFIX = "/userdatasourcesrules/"
+UDS_FETCH_RIGHTS = "/fetch_userdata"
+UDS_REMOVE_RIGHTS = "/remove_userdata"
+
+CABASE = "/ca"
+CAPREFIX = "/ca/"
+ROLE_PUBLICWEBUSER = "/public_web_user"
+ROLE_ADMINISTRATOR = "/administrator"
+ROLE_SUPERADMINISTRATOR = "/super_administrator"
+REGULAR_CAFUNCTIONALTY = "/ca_functionality"
+REGULAR_CABASICFUNCTIONS = "/ca_functionality/basic_functions"
+REGULAR_ACTIVATECA = "/ca_functionality/basic_functions/activate_ca"
+REGULAR_RENEWCA = "/ca_functionality/renew_ca"
+REGULAR_VIEWCERTIFICATE = "/ca_functionality/view_certificate"
+REGULAR_APPROVECAACTION = "/ca_functionality/approve_caaction"
+REGULAR_CREATECRL = "/ca_functionality/create_crl"
+REGULAR_EDITCERTIFICATEPROFILES = "/ca_functionality/edit_certificate_profiles"
+REGULAR_CREATECERTIFICATE = "/ca_functionality/create_certificate"
+REGULAR_STORECERTIFICATE = "/ca_functionality/store_certificate"
+REGULAR_RAFUNCTIONALITY = "/ra_functionality"
+REGULAR_EDITENDENTITYPROFILES = "/ra_functionality/edit_end_entity_profiles"
+REGULAR_EDITUSERDATASOURCES = "/ra_functionality/edit_user_data_sources"
+REGULAR_VIEWENDENTITY = "/ra_functionality/view_end_entity"
+REGULAR_CREATEENDENTITY = "/ra_functionality/create_end_entity"
+REGULAR_EDITENDENTITY = "/ra_functionality/edit_end_entity"
+REGULAR_DELETEENDENTITY = "/ra_functionality/delete_end_entity"
+REGULAR_REVOKEENDENTITY = "/ra_functionality/revoke_end_entity"
+REGULAR_VIEWENDENTITYHISTORY = "/ra_functionality/view_end_entity_history"
+REGULAR_APPROVEENDENTITY = "/ra_functionality/approve_end_entity"
+REGULAR_LOGFUNCTIONALITY = "/log_functionality"
+REGULAR_VIEWLOG = "/log_functionality/view_log"
+REGULAR_LOGCONFIGURATION = "/log_functionality/edit_log_configuration"
+REGULAR_LOG_CUSTOM_EVENTS = "/log_functionality/log_custom_events"
+REGULAR_SYSTEMFUNCTIONALITY = "/system_functionality"
+REGULAR_EDITADMINISTRATORPRIVILEDGES = "/system_functionality/edit_administrator_privileges"
+REGULAR_EDITSYSTEMCONFIGURATION = "/system_functionality/edit_systemconfiguration"
+REGULAR_VIEWHARDTOKENS = "/ra_functionality/view_hardtoken"
+REGULAR_VIEWPUKS = "/ra_functionality/view_hardtoken/puk_data"
+REGULAR_KEYRECOVERY = "/ra_functionality/keyrecovery"
+
+HARDTOKEN_HARDTOKENFUNCTIONALITY = "/hardtoken_functionality"
+HARDTOKEN_EDITHARDTOKENISSUERS = "/hardtoken_functionality/edit_hardtoken_issuers"
+HARDTOKEN_EDITHARDTOKENPROFILES = "/hardtoken_functionality/edit_hardtoken_profiles"
+HARDTOKEN_ISSUEHARDTOKENS = "/hardtoken_functionality/issue_hardtokens"
+HARDTOKEN_ISSUEHARDTOKENADMINISTRATORS = "/hardtoken_functionality/issue_hardtoken_administrators"
+
+RESPONSETYPE_CERTIFICATE = "CERTIFICATE"
+RESPONSETYPE_PKCS7 = "PKCS7"
+RESPONSETYPE_PKCS7WITHCHAIN = "PKCS7WITHCHAIN"
+
+NOT_REVOKED = -1
+REVOKATION_REASON_UNSPECIFIED = 0
+REVOKATION_REASON_KEYCOMPROMISE = 1
+REVOKATION_REASON_CACOMPROMISE = 2
+REVOKATION_REASON_AFFILIATIONCHANGED = 3
+REVOKATION_REASON_SUPERSEDED = 4
+REVOKATION_REASON_CESSATIONOFOPERATION = 5
+REVOKATION_REASON_CERTIFICATEHOLD = 6
+REVOKATION_REASON_REMOVEFROMCRL = 8
+REVOKATION_REASON_PRIVILEGESWITHDRAWN = 9
+REVOKATION_REASON_AACOMPROMISE = 10
+
+
+class HTTPSClientAuthHandler(get_https_handler()):
+
+    def __init__(self, key, cert):
+        get_https_handler().__init__(self)
+        self.key = key
+        self.cert = cert
+
+    def https_open(self, req):
+        return self.do_open(self.get_connection, req)
+
+    def get_connection(self, host, timeout=5):
+        if sys.version_info[0] >= 3:
+            return http.client.HTTPSConnection(host, key_file=self.key, cert_file=self.cert, timeout=timeout)
+        else:
+            return httplib.HTTPSConnection(host, key_file=self.key, cert_file=self.cert, timeout=timeout)
+
+
+class HTTPSClientCertTransport(suds.transport.http.HttpTransport):
+
+    def __init__(self, key, cert, *args, **kwargs):
+        suds.transport.http.HttpTransport.__init__(self, *args, **kwargs)
+        self.key = key
+        self.cert = cert
+
+    def u2open(self, u2request, timeout=None):
+        tm = timeout or self.options.timeout
+        if sys.version_info[0] >= 3:
+            url = urllib.request.build_opener(HTTPSClientAuthHandler(self.key, self.cert))
+        else:
+            url = urllib2.build_opener(HTTPSClientAuthHandler(self.key, self.cert))
+        if self.u2ver() < 2.6:
+            socket.setdefaulttimeout(tm)
+            return url.open(u2request)
+        else:
+            return url.open(u2request, timeout=tm)
+
+
+class Ejbca(object):
+
+    def __init__(self, url, cert=None, key=None):
+        self.url = url
+        self.cert = cert
+        self.key = key
+        self.transport = HTTPSClientCertTransport(self.key, self.cert) if self.cert else None
+        self.wsclient = suds.client.Client(self.url, transport=self.transport)
+
+
+    def get_version(self):
+        return self.wsclient.service.getEjbcaVersion()
+
+
+    def get_users(self):
+        return self.find_user(MATCH_WITH_DN, MATCH_TYPE_CONTAINS, "=")
+
+
+    def find_user(self, matchwith, matchtype, matchvalue):
+        usermatch = self.wsclient.factory.create('userMatch')
+        usermatch.matchwith = matchwith
+        usermatch.matchtype = matchtype
+        usermatch.matchvalue = matchvalue
+        return self.wsclient.service.findUser(usermatch)
+
+
+    def edit_user(self, user):
+        return self.wsclient.service.editUser(user)
+
+
+    def _decode_ejbca_cert(self, double_mess):
+        single_mess = base64.b64decode(double_mess)
+        cert_data = base64.b64decode(single_mess)
+        cert = M2Crypto.X509.load_cert_string(cert_data, M2Crypto.X509.FORMAT_DER)
+        return cert
+
+
+    def pkcs10_request(self, username, password, pkcs10, hardTokenSN, responseType):
+        res = self.wsclient.service.pkcs10Request(
+            arg0=username,
+            arg1=password,
+            arg2=pkcs10,
+            arg3=hardTokenSN,
+            arg4=responseType)
+        return self._decode_ejbca_cert(res["data"])
+
+
+    def find_certs(self, loginName, validOnly=False):
+        reslist = self.wsclient.service.findCerts(
+            arg0=loginName,
+            arg1=validOnly)
+        certs = []
+        for res in reslist:
+            double_mess = res["certificateData"]
+            if double_mess is not None:
+                cert = self._decode_ejbca_cert(double_mess)
+                cert.ejbca_status = res["type"]
+                certs.append(cert)
+        return certs

warden_ra/openssl.cnf.example

+# OpenSSL root CA configuration file.
+# Copy to `/root/ca/openssl.cnf`.
+
+[ ca ]
+# `man ca`
+default_ca = CA_default
+
+[ CA_default ]
+# Directory and file locations.
+dir               = /var/spool/example-ca
+certs             = $dir/certs
+crl_dir           = $dir/crl
+new_certs_dir     = $dir/newcerts
+database          = $dir/index.txt
+serial            = $dir/serial
+RANDFILE          = $dir/private/.rand
+unique_subject    = no
+
+# The root key and root certificate.
+private_key       = $dir/private/ca.key.pem
+certificate       = $dir/certs/ca.cert.pem
+
+# For certificate revocation lists.
+crlnumber         = $dir/crlnumber
+crl               = $dir/crl/ca.crl.pem
+crl_extensions    = crl_ext
+default_crl_days  = 30
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md        = sha256
+
+name_opt          = ca_default
+cert_opt          = ca_default
+default_days      = 375
+preserve          = no
+policy            = policy_loose
+
+[ policy_loose ]
+# Allow the CA to sign a more diverse range of certificates.
+# See the POLICY FORMAT section of the `ca` man page.
+countryName             = optional
+stateOrProvinceName     = optional
+localityName            = optional
+organizationName        = optional
+organizationalUnitName  = optional
+commonName              = supplied
+emailAddress            = optional
+
+[ req ]
+# Options for the `req` tool (`man req`).
+default_bits        = 2048
+distinguished_name  = req_distinguished_name
+string_mask         = utf8only
+
+# SHA-1 is deprecated, so use SHA-2 instead.
+default_md          = sha256
+
+# Extension to add when the -x509 option is used.
+x509_extensions     = v3_ca
+
+[ req_distinguished_name ]
+# See <https://en.wikipedia.org/wiki/Certificate_signing_request>.
+countryName                     = Country Name (2 letter code)
+stateOrProvinceName             = State or Province Name
+localityName                    = Locality Name
+0.organizationName              = Organization Name
+organizationalUnitName          = Organizational Unit Name
+commonName                      = Common Name
+emailAddress                    = Email Address
+
+# Optionally, specify some defaults.
+countryName_default             = CZ
+stateOrProvinceName_default     = Czech Republic
+localityName_default            =
+0.organizationName_default      = Example
+organizationalUnitName_default  =
+emailAddress_default            =
+
+[ v3_ca ]
+# Extensions for a typical CA (`man x509v3_config`).
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid:always,issuer
+basicConstraints = critical, CA:true
+keyUsage = critical, digitalSignature, cRLSign, keyCertSign
+
+[ server_cert ]
+# Extensions for server certificates (`man x509v3_config`).
+basicConstraints = CA:FALSE
+nsCertType = client
+nsComment = "OpenSSL Generate Client Certificate"
+subjectKeyIdentifier = hash
+authorityKeyIdentifier = keyid,issuer:always
+keyUsage = critical, digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth
+

warden_ra/warden_apply.sh

+#!/bin/bash
+
+key=key.pem
+csr=csr.pem
+cert=cert.pem
+result=${TMPDIR:-${TMP:-/tmp}}/cert.$$.$RANDOM
+config=${TMPDIR:-${TMP:-/tmp}}/conf.$$.$RANDOM
+if [ "$1" == "--cacert" ]; then
+  cacert="--cacert $2"
+  shift
+  shift
+fi
+url="$1"
+client="$2"
+password="$3"
+incert="$3"
+inkey="$4"
+
+trap 'rm -f "$config $result"' INT TERM HUP EXIT
+
+function flee { echo -e "$1"; exit $2; }
+
+[ -z "$client" -o -z "$password" ] && flee "Usage: ${0%.*} [--cacert CERT] url client.name password\n       ${0%.*} [--cacert CERT] url client.name cert_file key_file" 255
+
+url="${url%/}/getCert"
+
+for n in openssl curl; do
+    command -v "$n" 2>&1 >/dev/null || flee "Haven't found $n binary." 251
+done
+for n in "$csr" "$key" "$cert"; do
+    [ -e "$n" ] && flee "$n already exists, I won't overwrite, move them away first, please." 254
+done
+for n in "$result" "$config"; do
+    touch "$n" || flee "Error creating temporary file ($n)." 253
+done
+
+echo -e "default_bits=2048\ndistinguished_name=rdn\nprompt=no\n[rdn]\ncommonName=dummy" > "$config"
+
+openssl req -new -nodes -batch -keyout "$key" -out "$csr" -config "$config" || flee "Error generating key/certificate request." 252
+
+if [ -z "$inkey" ]; then
+    curl --progress-bar $cacert --request POST --data-binary '@-' "$url?name=$client&password=$password" < "$csr" > "$result"
+else
+    # local cert file name may be interpreted as a "nickname", add "./" to force interpretation as a file
+    if [[ ! "$incert" =~ "/" ]]; then
+        incert="./$incert"
+    fi
+    curl --progress-bar $cacert --request POST --data-binary '@-' --cert "$incert" --key "$inkey" "$url?name=$client" < "$csr" > "$result"
+fi
+
+case $(<$result) in '-----BEGIN CERTIFICATE-----'*)
+    mv "$result" "$cert"
+    flee "Succesfully generated key ($key) and obtained certificate ($cert)." 0
+esac
+
+flee "$(<$result)\n\nCertificate request failed. Please save all error messages for communication with registration authority representative." 252

warden_ra/warden_ra.cfg.dist

+{
+    "Log": {
+        "filename": "/var/log/warden_ra.log",
+        "level": "info"
+    },
+    "Registry": {
+
+// Example configuration for OpenSSL CA backend
+//        "type": "OpenSSLRegistry",
+//        "base_dir": "/var/spool/example-ca",
+//        "subject_dn_template": "DC=cz,DC=example-ca,DC=warden,CN=%s"
+
+// Example configuration for EJBCA backend
+//        "type": "EjbcaRegistry",
+//        "url": "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl",
+//        "cert": "warden_ra.cert.pem",
+//        "key": "warden_ra.key.pem",
+//        "ca_name": "Example CA",
+//        "certificate_profile_name": "Example",
+//        "end_entity_profile_name": "Example EE",
+//        "subject_dn_template": "DC=cz,DC=example-ca,DC=warden,CN=%s",
+//        "username_suffix": "@warden"
+
+    }
+}

warden_ra/warden_ra.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2016, CESNET, z. s. p. o.
+# Use of this source is governed by an ISC license, see LICENSE file.
+
+from __future__ import print_function
+
+import sys
+import os
+import time
+import fcntl
+import errno
+import string
+import random
+import struct
+import operator
+import argparse
+import json
+import logging
+import os.path as pth
+import subprocess
+import shlex
+import tempfile
+import M2Crypto
+
+import ejbcaws
+
+if sys.version_info[0] >= 3:
+    import configparser as ConfigParser
+else:
+    import ConfigParser
+
+# usual path to warden server
+sys.path.append(pth.join(pth.dirname(__file__), "..", "warden_server"))
+import warden_server
+from warden_server import Request, ObjectBase, FileLogger, SysLogger, Server, expose, read_cfg
+
+VERSION = "3.0-beta3"
+
+class ClientDisabledError(Exception): pass
+class ClientNotIssuableError(Exception): pass
+class AuthenticationError(Exception): pass
+class PopenError(Exception): pass
+
+
+class Client(object):
+
+    def __init__(self, name, admins=None, status=None, pwd=None, opaque=None):
+        self.name = name
+        self.admins = admins or []
+        self.status = status or "New"
+        self.pwd = pwd
+        self.opaque = opaque or {}
+
+    def update(self, admins=None, status=None, pwd=None):
+        if admins is not None:
+            self.admins = admins
+        if status:
+            if self.status == "Disabled" and status not in ("Passive", "Disabled"):
+                raise ClientDisabledError("This client is disabled")
+            self.status = status
+        self.pwd = pwd if status=="Issuable" and pwd else None
+
+    def __str__(self):
+        return (
+            "Client:   %s\n"
+            "Admins:   %s\n"
+            "Status:   %s\n"
+        ) % (self.name, ", ".join(self.admins), self.status)
+
+    def str(self, verbose=False):
+        return str(self) + (str(self.opaque) if self.opaque and verbose else "")
+
+
+class OpenSSLRegistry(object):
+
+    def __init__(self, log, base_dir,
+                 subject_dn_template, openssl_sign, lock_timeout):
+        self.base_dir = base_dir
+        self.cnf_file = pth.join(base_dir, "openssl.cnf")
+        self.client_dir = pth.join(base_dir, "clients")
+        self.serial_file = pth.join(base_dir, "serial")
+        self.newcerts_dir = pth.join(base_dir, "newcerts")
+        self.csr_dir = pth.join(base_dir, "csr")
+        self.lock_file = pth.join(base_dir, "lock")
+        self.lock_timeout = lock_timeout
+        self.log = log
+        self.subject_dn_template = subject_dn_template
+        self.openssl_sign = openssl_sign
+        os.umask(0o0002)    # read privilege for usual apache group
+
+    def get_clients(self):
+        return [self.get_client(c) for c in os.listdir(self.client_dir) if pth.isdir(pth.join(self.client_dir, c))]
+
+    def get_client(self, name):
+        config = ConfigParser.RawConfigParser()
+        try:
+            with open(pth.join(self.client_dir, name, "state")) as cf:
+                config.readfp(cf)
+        except IOError as e:
+            if e.errno == errno.ENOENT:
+                return None
+            raise
+        datum = dict(config.items("Client"))
+        return Client(name, admins=datum["admins"].split(","), status=datum["status"], pwd=datum.get("password"))
+
+    def new_client(self, name, admins=None):
+        user = self.get_client(name)
+        if user:
+            raise LookupError("Client %s already exists" % name)
+        return Client(name, admins)
+
+    def save_client(self, client):
+        config = ConfigParser.RawConfigParser()
+        config.add_section("Client")
+        config.set("Client", "admins", ",".join(client.admins))
+        config.set("Client", "status", client.status)
+        if client.pwd:
+            config.set("Client", "password", client.pwd)
+        client_path = pth.join(self.client_dir, client.name)
+        try:
+            os.makedirs(client_path)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        with tempfile.NamedTemporaryFile(dir=client_path, delete=False, mode="w") as cf:
+            config.write(cf)
+        os.chmod(cf.name, 0o660)    # read privilege for usual apache group
+        os.rename(cf.name, pth.join(client_path, "state")) # atomic + rewrite, so no need for locking
+
+    def get_certs(self, client):
+        files = [fname for fname in os.listdir(pth.join(self.client_dir, client.name)) if not fname.startswith(".") and fname.endswith(".pem")]
+        certs = [M2Crypto.X509.load_cert(pth.join(self.client_dir, client.name, fname)) for fname in files]
+        return certs
+
+    def __enter__(self):
+        self._lockfd = os.open(self.lock_file, os.O_CREAT)
+        start = time.time()
+        while True:
+            try:
+                fcntl.flock(self._lockfd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                return
+            except (OSError, IOError) as e:
+                if e.errno != errno.EAGAIN or time.time() > start + self.lock_timeout:
+                   raise
+            time.sleep(0.5)
+
+    def __exit__(self, type_, value, traceback):
+        fcntl.flock(self._lockfd, fcntl.LOCK_UN)
+        os.close(self._lockfd)
+        try:
+            os.unlink(self.lock_file)
+        except:
+            pass
+
+    def run_openssl(self, command, **kwargs):
+        cmdline = shlex.split(command % kwargs)
+        process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        res = process.communicate()
+        if process.returncode:
+            raise PopenError("Popen returned nonzero code", process.returncode, ' '.join(cmdline), res[0], res[1])
+        return res
+
+    def new_cert(self, client, csr, pwd):
+        if client.status != "Issuable" or not client.pwd:
+            raise ClientNotIssuableError("Client not allowed to issue request or password not set")
+        if client.pwd != pwd:
+            raise AuthenticationError("Wrong credentials")
+        dn = self.subject_dn_template.replace("/", "//").replace(",", "/") % client.name
+        if not dn.startswith("/"):
+            dn = "/" + dn
+        with tempfile.NamedTemporaryFile(dir=self.csr_dir, delete=False) as csr_file:
+            csr_file.write(csr)
+        with self:  # lock dance
+            with open(self.serial_file) as f:
+                serial = f.read().strip()
+            output = self.run_openssl(self.openssl_sign, cnf = self.cnf_file, csr = csr_file.name, dn = dn)
+        self.log.debug(output)
+        os.rename(csr_file.name, pth.join(self.csr_dir, serial + ".csr.pem"))
+        client_pem_name = pth.join(self.client_dir, client.name, serial + ".cert.pem")
+        os.symlink(pth.join(self.newcerts_dir, serial + ".pem"), client_pem_name)
+        with open(client_pem_name) as pem:
+            cert = M2Crypto.X509.load_cert_string(pem.read(), M2Crypto.X509.FORMAT_PEM)
+        client.update(status="Passive", pwd=None)
+        self.save_client(client)
+        return cert
+
+    def __str__(self):
+        return "%s<%s>" % (type(self).__name__, self.base_dir)
+
+
+class EjbcaRegistry(OpenSSLRegistry):
+
+    status_ejbca_to_str = {
+        ejbcaws.STATUS_NEW: "Issuable",
+        ejbcaws.STATUS_GENERATED: "Passive",
+        ejbcaws.STATUS_INITIALIZED: "New",
+        ejbcaws.STATUS_HISTORICAL: "Disabled"
+    }
+    status_str_to_ejbca = dict((v, k) for k, v in status_ejbca_to_str.items())
+
+    def __init__(self, log, url, cert=None, key=None,
+                 ca_name="", certificate_profile_name="", end_entity_profile_name="",
+                 subject_dn_template="%s", username_suffix=""):
+        self.log = log
+        self.ejbca = ejbcaws.Ejbca(url, cert, key)
+        self.ca_name = ca_name
+        self.certificate_profile_name = certificate_profile_name
+        self.end_entity_profile_name = end_entity_profile_name
+        self.subject_dn_template = subject_dn_template
+        self.username_suffix = username_suffix
+
+    def client_data(self, ejbca_data):
+        ejbca_username = ejbca_data["username"]
+        username = ejbca_username[:-len(self.username_suffix)] if ejbca_username.endswith(self.username_suffix) else ejbca_username
+        try:
+            alt_name = ejbca_data["subjectAltName"]
+        except KeyError:
+            alt_name = None
+        if alt_name:
+            admins = [u if not u.startswith("RFC822NAME") else u[11:] for u in alt_name.split(",")]
+        else:
+            admins = []
+        status = self.status_ejbca_to_str.get(ejbca_data["status"], "Other")
+        return username, admins, status, None, ejbca_data
+
+    def get_clients(self):
+        return [Client(*self.client_data(u)) for u in self.ejbca.get_users()]
+
+    def get_client(self, name):
+        users = self.ejbca.find_user(ejbcaws.MATCH_WITH_USERNAME, ejbcaws.MATCH_TYPE_EQUALS, name + self.username_suffix)
+        if len(users) > 1:
+            raise LookupError("%d users %s found (more than one?!)" % (len(users), name))
+        if not users:
+            return None
+        return Client(*self.client_data(users[0]))
+
+    def save_client(self, client):
+        edata = client.opaque or dict(
+            caName=self.ca_name,
+            certificateProfileName=self.certificate_profile_name,
+            endEntityProfileName=self.end_entity_profile_name,
+            keyRecoverable=False,
+            sendNotification=False,
+            tokenType=ejbcaws.TOKEN_TYPE_USERGENERATED,
+            password = "".join((random.choice(string.ascii_letters + string.digits) for dummy in range(16))),
+            clearPwd = True,
+            username = client.name + self.username_suffix,
+            subjectDN = self.subject_dn_template % client.name
+        )
+        edata["subjectAltName"] = ",".join(("RFC822NAME=%s" % a for a in client.admins))
+        try:
+            edata["status"] = self.status_str_to_ejbca.get(client.status)
+        except KeyError:
+            # Unknown status - either came from EJBCA and translated to
+            # "Other", or something wrong came in later. Let's just
+            # keep original EJBCA status unchanged.
+            pass
+        if client.pwd:
+            edata["password"] = client.pwd
+            edata["clearPwd"] = True
+        self.ejbca.edit_user(edata)
+
+    def get_certs(self, client):
+        return self.ejbca.find_certs(client.opaque["username"], validOnly=False)
+
+    def new_cert(self, client, csr, pwd):
+        cert = self.ejbca.pkcs10_request(
+            client.opaque["username"],
+            pwd, csr, 0, ejbcaws.RESPONSETYPE_CERTIFICATE)
+        return cert
+
+    def __str__(self):
+        return self.ejbca.get_version()
+
+
+def relaxed_ord(c):
+    # Compatibility wrapper for py2/py3
+    try:
+        return ord(c)
+    except TypeError:
+        return c
+
+
+def format_cert(cert):
+    return (
+        "Subject:     %s\n"
+        "Validity:    %s - %s\n"
+        "Serial:      %s\n"
+        "Fingerprint: md5:%s, sha1:%s\n"
+        "Issuer:      %s\n"
+    ) % (
+        cert.get_subject().as_text(),
+        cert.get_not_before().get_datetime().isoformat(),
+        cert.get_not_after().get_datetime().isoformat(),
+        ":".join(["%02x" % relaxed_ord(c) for c in struct.pack('!Q', cert.get_serial_number())]),
+        cert.get_fingerprint("md5"),
+        cert.get_fingerprint("sha1"),
+        cert.get_issuer().as_text()
+    )
+
+
+# Server side
+
+class OptionalAuthenticator(ObjectBase):
+
+    def __init__(self, req, log):
+        ObjectBase.__init__(self, req, log)
+
+
+    def __str__(self):
+        return "%s(req=%s)" % (type(self).__name__, type(self.req).__name__)
+
+
+    def authenticate(self, env, args):
+        cert_name = env.get("SSL_CLIENT_S_DN_CN")
+
+        if cert_name:
+            if cert_name != args.setdefault("name", [cert_name])[0]:
+                exception = self.req.error(message="authenticate: client name does not correspond with certificate", error=403, cn = cert_name, args = args)
+                exception.log(self.log)
+                return None
+
+            verify = env.get("SSL_CLIENT_VERIFY")
+            if verify != "SUCCESS":
+                exception = self.req.error(message="authenticate: certificate present but verification failed", error=403, cn = cert_name, args = args, verify=verify)
+                exception.log(self.log)
+                return None
+
+            return "cert"   # Ok, client authorized by valid certificate
+
+        else:
+            try:
+                args["password"][0]
+                return "pwd"    # Ok, pass on, but getCert will have to rely on certificate registry password
+            except (KeyError, IndexError):
+                exception = self.req.error(message="authenticate: no certificate nor password present", error=403, cn = cert_name, args = args)
+                exception.log(self.log)
+                return None
+
+
+    def authorize(self, env, client, path, method):
+        return True
+
+
+class CertHandler(ObjectBase):
+
+    def __init__(self, req, log, registry):
+        ObjectBase.__init__(self, req, log)
+        self.registry = registry
+
+    @expose(read=1, debug=1)
+    def getCert(self, csr_data=None, name=None, password=None):
+        if not (name and csr_data):
+            raise self.req.error(message="Wrong or missing arguments", error=400, name=name, password=password)
+        client = self.registry.get_client(name[0])
+        if not client:
+            raise self.req.error(message="Unknown client", error=403, name=name, password=password)
+        self.log.info("Client %s" % client)
+        if self.req.client == "cert":
+            # Correctly authenticated by cert, most probably not preactivated with password,
+            # so generate oneshot password and allow now
+            password = ["".join((random.choice(string.ascii_letters + string.digits) for dummy in range(16)))]
+            self.log.debug("Authorized by X509, enabling cert generation with password %s" % password)
+            try:
+                client.update(status="Issuable", pwd=password[0])
+                self.registry.save_client(client)
+            except ClientDisabledError as e:
+                raise self.req.error(message="Error enabling cert generation", error=403, exc=sys.exc_info())
+        if not password:
+            raise self.req.error(message="Missing password and certificate validation failed", error=403, name=name, password=password)
+        try:
+            newcert = self.registry.new_cert(client, csr_data.decode('latin1'), password[0])
+        except Exception as e:
+            raise self.req.error(message="Processing error", error=403, exc=sys.exc_info())
+        self.log.info("Generated.")
+        return [("Content-Type", "application/x-pem-file")], newcert.as_pem()
+
+
+# Order in which the base objects must get initialized
+section_order = ("log", "auth", "registry", "handler", "server")
+
+# List of sections and objects, configured by them
+# First object in each object list is the default one, otherwise
+# "type" keyword in section may be used to choose other
+section_def = {
+    "log": [FileLogger, SysLogger],
+    "auth": [OptionalAuthenticator],
+    "registry": [OpenSSLRegistry, EjbcaRegistry],
+    "handler": [CertHandler],
+    "server": [Server]
+}
+
+# Object parameter conversions and defaults
+param_def = {
+    FileLogger: warden_server.param_def[FileLogger],
+    SysLogger: warden_server.param_def[SysLogger],
+    Server: warden_server.param_def[Server],
+    OptionalAuthenticator: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"}
+    },
+    OpenSSLRegistry: {
+        "log":  {"type": "obj", "default": "log"},
+        "base_dir": {"type": "str", "default": pth.join(pth.dirname(__file__), "ca")},
+        "subject_dn_template": {"type": "str", "default": "DC=cz,DC=example-ca,DC=warden,CN=%s"},
+        "openssl_sign": {"type": "str", "default": "openssl ca -config %(cnf)s -batch -extensions server_cert -days 375 -notext -md sha256 -in %(csr)s -subj '%(dn)s'"},
+        "lock_timeout": {"type": "natural", "default": "3"}
+    },
+    EjbcaRegistry: {
+        "log":  {"type": "obj", "default": "log"},
+        "url": {"type": "str", "default": "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl"},
+        "cert": {"type": "filepath", "default": pth.join(pth.dirname(__file__), "warden_ra.cert.pem")},
+        "key": {"type": "filepath", "default": pth.join(pth.dirname(__file__), "warden_ra.key.pem")},
+        "ca_name": {"type": "str", "default": "Example CA"},
+        "certificate_profile_name": {"type": "str", "default": "Example"},
+        "end_entity_profile_name": {"type": "str", "default": "Example EE"},
+        "subject_dn_template": {"type": "str", "default": "DC=cz,DC=example-ca,DC=warden,CN=%s"},
+        "username_suffix": {"type": "str", "default": "@warden"}
+    },
+    CertHandler: {
+        "req": {"type": "obj", "default": "req"},
+        "log": {"type": "obj", "default": "log"},
+        "registry": {"type": "obj", "default": "registry"}
+    }
+}
+
+param_def[FileLogger]["filename"] = {"type": "filepath", "default": pth.join(pth.dirname(__file__), pth.splitext(pth.split(__file__)[1])[0] + ".log")}
+
+
+def build_server(conf):
+    return warden_server.build_server(conf, section_order, section_def, param_def)
+
+
+# Command line
+
+def list_clients(registry, name=None, verbose=False, show_cert=True):
+    if name is not None:
+        client = registry.get_client(name)
+        if client is None:
+            print("No such client.")
+            return
+        else:
+            print(client.str(verbose))
+            if show_cert:
+                for cert in sorted(registry.get_certs(client), key=lambda c: c.get_not_after().get_datetime()):
+                    print(format_cert(cert))
+                    if verbose:
+                        print(cert.as_text())
+    else:
+        clients = registry.get_clients()
+        for client in sorted(clients, key=operator.attrgetter("name")):
+            print(client.str(verbose))
+
+
+def register_client(registry, name, admins=None, verbose=False):
+    try:
+        client = registry.new_client(name, admins)
+    except LookupError as e:
+        print(e)
+        return
+    registry.save_client(client)
+    list_clients(registry, name, verbose, show_cert=False)
+
+
+def applicant(registry, name, password=None, verbose=False):
+    client = registry.get_client(name)
+    if not client:
+        print("No such client.")
+        return
+    if password is None:
+        password = "".join((random.choice(string.ascii_letters + string.digits) for dummy in range(16)))
+    try:
+        client.update(status="Issuable", pwd=password)
+    except ClientDisabledError:
+        print("This client is disabled. Use 'enable' first.")
+        return
+    registry.save_client(client)
+    list_clients(registry, name, verbose, show_cert=False)
+    print("Application password is: %s\n" % password)
+
+
+def enable(registry, name, verbose=False):
+    client = registry.get_client(name)
+    if not client:
+        print("No such client.")
+        return
+    client.update(status="Passive")
+    registry.save_client(client)
+    list_clients(registry, name, verbose, show_cert=False)
+
+
+def disable(registry, name, verbose=False):
+    client = registry.get_client(name)
+    if not client:
+        print("No such client.")
+        return
+    client.update(status="Disabled")
+    registry.save_client(client)
+    list_clients(registry, name, verbose, show_cert=False)
+
+
+def request(registry, key, csr, verbose=False):
+    openssl = subprocess.Popen(
+        [
+            "openssl", "req", "-new", "-nodes", "-batch",
+            "-keyout", key,
+            "-out", csr,
+            "-config", "/dev/stdin"
+        ], stdin=subprocess.PIPE
+    )
+    openssl.stdin.write(
+        "distinguished_name=req_distinguished_name\n"
+        "prompt=no\n"
+        "\n"
+        "[req_distinguished_name]\n"
+        "commonName=dummy".encode("ascii")
+    )
+    openssl.stdin.close()
+    openssl.wait()
+    if verbose:
+        with open(csr, "r") as f:
+            print(f.read())
+
+
+def gen_cert(registry, name, csr, cert, password, verbose=False):
+    with open(csr, "r") as f:
+        csr_data = f.read()
+    client = registry.get_client(name)
+    newcert = registry.new_cert(client, csr_data, password)
+    print(format_cert(newcert))
+    if verbose:
+        print(newcert.as_text())
+        print(newcert.as_pem())
+    with open(cert, "w") as f:
+        f.write(newcert.as_text())
+        f.write(newcert.as_pem().decode("ascii"))
+
+
+def get_args():
+    argp = argparse.ArgumentParser(
+        description="Warden server certificate registry", add_help=False)
+    argp.add_argument("--help", action="help",
+        help="show this help message and exit")
+    argp.add_argument("-c", "--config",
+        help="path to configuration file")
+    argp.add_argument("-v", "--verbose", action="store_true", default=False,
+        help="be more chatty")
+    subargp = argp.add_subparsers(title="commands")
+
+    subargp_list = subargp.add_parser("list", add_help=False,
+        description="List registered clients.",
+        help="list clients")
+    subargp_list.set_defaults(command=list_clients)
+    subargp_list.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_list.add_argument("--name", action="store", type=str,
+        help="client name")
+
+    subargp_reg = subargp.add_parser("register", add_help=False,
+        description="Add client registration entry.",
+        help="register client")
+    subargp_reg.set_defaults(command=register_client)
+    subargp_reg.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_reg.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+    subargp_reg.add_argument("--admins", action="store", type=str,
+        required=True, nargs="*", help="administrator list")
+
+    subargp_apply = subargp.add_parser("applicant", add_help=False,
+        description="Set client into certificate application mode and set its password",
+        help="allow for certificate application")
+    subargp_apply.set_defaults(command=applicant)
+    subargp_apply.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_apply.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+    subargp_apply.add_argument("--password", action="store", type=str,
+        help="password for application (will be autogenerated if not set)")
+
+    subargp_enable = subargp.add_parser("enable", add_help=False,
+        description="Enable this client",
+        help="enable this client")
+    subargp_enable.set_defaults(command=enable)
+    subargp_enable.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_enable.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+
+    subargp_disable = subargp.add_parser("disable", add_help=False,
+        description="Disable this client",
+        help="disable this client (no more applications until enabled again)")
+    subargp_disable.set_defaults(command=disable)
+    subargp_disable.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_disable.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+
+    subargp_req = subargp.add_parser("request", add_help=False,
+        description="Generate certificate request",
+        help="generate CSR")
+    subargp_req.set_defaults(command=request)
+    subargp_req.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_req.add_argument("--key", action="store", type=str,
+        required=True, help="file for saving the key")
+    subargp_req.add_argument("--csr", action="store", type=str,
+        required=True, help="file for saving the request")
+
+    subargp_cert = subargp.add_parser("gencert", add_help=False,
+        description="Request new certificate from registry",
+        help="get new certificate")
+    subargp_cert.set_defaults(command=gen_cert)
+    subargp_cert.add_argument("--help", action="help",
+        help="show this help message and exit")
+    subargp_cert.add_argument("--name", action="store", type=str,
+        required=True, help="client name")
+    subargp_cert.add_argument("--csr", action="store", type=str,
+        required=True, help="file for saving the request")
+    subargp_cert.add_argument("--cert", action="store", type=str,
+        required=True, help="file for saving the new certificate")
+    subargp_cert.add_argument("--password", action="store", type=str,
+        required=True, help="password for application")
+
+    return argp.parse_args()
+
+
+if __name__ == "__main__":
+    args = get_args()
+    config = pth.join(pth.dirname(__file__), args.config or "warden_ra.cfg")
+    server = build_server(read_cfg(config))
+    registry = server.handler.registry
+    if args.verbose:
+        print(registry)
+    command = args.command
+    subargs = vars(args)
+    del subargs["command"]
+    del subargs["config"]
+    sys.exit(command(registry, **subargs))

warden_ra/warden_ra.wsgi

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from sys import path
+from os.path import dirname, join
+
+path.append(dirname(__file__))
+from warden_ra import build_server
+
+## JSON configuration with line comments (trailing #)
+from warden_ra import read_cfg
+application = build_server(read_cfg(join(dirname(__file__), "warden_ra.cfg")))

warden3/warden_server/README → warden_server/README

-+-------------------+
-| Warden Server 3.0 |
-+-------------------+
++-------------------------+
+| Warden Server 3.0-beta3 |
++-------------------------+
 
 Content
 
@@ -13,8 +13,8 @@ Content
 ------------------------------------------------------------------------------
 A. Introduction
 
-   Warden is a system for efficient sharing information about detected events
-(threats). Warden Server is server-side part of the software, the
+   Warden is a system for efficient sharing of information about detected
+events (threats). Warden Server is server-side part of the software, the
 communication hub, allowing to publish detected events and download yet
 unprocessed ones.
 
@@ -28,7 +28,7 @@ of Warden Server administration.
    Warden Server is Python/WSGI based, written primarily with Apache mod_wsgi
 in mind. Other WSGI servers/frameworks are not yet tested, so your mileage
 may vary. Authentication is X509 certificate (for machine or client
-identification) + shared secret (for client icentification, where
+identification) + shared secret (for client identification, where
 certificate does not suffice).
 
 ------------------------------------------------------------------------------
@@ -37,7 +37,7 @@ B. Dependencies
  1. Platform
 
     Python 2.7+
-	Apache 2.2
+	Apache 2.2/2.4
 	mod_wsgi 3.3+
 
  2. Python modules
@@ -46,6 +46,10 @@ B. Dependencies
     python-m2crypto 0.20+
     jsonschema 2.4+
 
+ 3. Database
+
+    MySQL | MariaDB >= 5.5
+
 ------------------------------------------------------------------------------
 C. Installation
 
@@ -54,36 +58,45 @@ C. Installation
 
    # cd /opt
    # tar xjf warden_server_3.0.tar.bz2
-   # ls
-   warden_server_3.0
+   # mv warden_server_3.0 warden_server
 
  * Create database and desired database users
    (We're using db "warden3" and user "warden@localhost" as an example.)
 
    # mysql -p
 
-   mysql> CREATE DATABASE warden3;
-   mysql> GRANT ALL ON warden3.* TO `warden`@`localhost`;
-   mysql> SET PASSWORD FOR 'warden'@'localhost' = PASSWORD('example');
-   mysql> FLUSH PRIVILEDGES;
+   > CREATE DATABASE warden3;
+   > CREATE USER 'warden'@'localhost' IDENTIFIED BY 'example';
+   > GRANT ALL ON warden3.* TO `warden`@`localhost`;
+   > FLUSH PRIVILEGES;
 
  * Create necessary table structure
 
-   mysql -p -u warden warden3 < warden3.0.sql
+   mysql -p -u warden warden3 < warden_3.0.sql
+
+ * Get up to date Idea schema
+
+   wget -O warden_server/idea.schema https://idea.cesnet.cz/_media/en/idea0.schema
 
  * Enable mod_wsgi, mod_ssl, include Warden configuration
 
    This depends heavily on your distribution and Apache configuration.
    Basically you need to create and include apache.conf:
 
-      Include /opt/warden_server_3.0/apache.conf
+      Include /opt/warden_server/apache.conf
 
    or paste the contents into whichever Directory, Location or VirtualHost
-   you dedicate for Warden. You can use apache.conf.dist as an example.
+   you dedicate for Warden.  You can use apache22.conf.dist or
+   apache24.conf.dist (for Apache version 2.2 or 2.4, respectively) as an
+   example.
 
    You may need to change paths to certificate/key/ca material, path to
    warden_server.wsgi and web path alias.
 
+   Also note that warden_server itself is NOT thread safe, so included
+   configuration expects Apache with mpm-prefork worker, or you will have
+   to configure mod_wsgi as separate process with threads=1.
+
    Also, for warden_server.wsgi, you can use warden_server.wsgi.dist as
    a template. You will possibly need to change at least configuration
    file path.
@@ -117,8 +130,9 @@ particular implementation object of the aspect, for example type of logger
 
 		Log: FileLogger, SysLogger
 		DB: MySQL
-		Auth: X509Authenticator, NoAuthenticator
-		Validator: JSONSchemaValidator, "NoValidator
+		Auth: X509Authenticator, X509NameAuthenticator,
+              X509MixMatchAuthenticator,PlainAuthenticator
+		Validator: JSONSchemaValidator, NoValidator
 		Handler: WardenHandler
 
 	"type" keyword is not mandatory, if not specified, first implementation
@@ -136,17 +150,31 @@ object from particular section list is used ("FileLogger" for example).
       facility: syslog facility, defaults to "daemon"
       level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
 
-   NoAuthenticator: forego authentication, for debug purposes
-
    X509Authenticator: authenticate based on certificate chain validation,
        hostname corresponding with certificate CN or SubjectAltName and
-       optionally shared secret
+       optionally shared secret (note that more clients on one machine
+       will have to have the certificate with the same hostname, clients
+       than can be differentiated by separate secrets).
+       This method is OBSOLETE.
+
+    X509NameAuthenticator: authenticate based on certificate chain validation,
+        certificate CN must correspond with client _name_, NOT hostname.
+
+    X509MixMatchAuthenticator: automatically choose X509Authenticator or
+        X509NameAuthenticator based on existence of 'secret' in query. Allows
+        for seamless transition of clients between two authentication methods.
+
+   PlainAuthenticator: authenticate based on client name or shared secret, usable
+      over plain HTTP connection or HTTPS without client certificate - note that
+      this pretty much spoils security, and is meant only for testing and
+      debugging purposes, NOT for production servers
 
    NoValidator: forego event JSON validation, for debug purposes
 
    JSONSchemaValidator: validate incoming events based on JSON schema file
       filename: path to schema file, defaults to "idea.schema" at
-                installation directory
+                installation directory, for information on obtaining current
+                IDEA schema file, refer to https://idea.cesnet.cz/en/schema
 
    MySQL: database storage backend
       host: database server host, default "localhost"
@@ -204,7 +232,8 @@ warden_server.py register [--help] -n NAME -h HOSTNAME -r REQUESTOR
      -r REQUESTOR, --requestor REQUESTOR
                            requestor email
      -s SECRET, --secret SECRET
-                           authentication token
+                           authentication token (use explicit empty string to
+                           disable)
      --note NOTE           client freetext description
      --valid               valid client (default)
      --novalid

warden_server/README.test

++----------------------------------+
+| Warden3 Server Test Suite README |
++----------------------------------+
+
+Content
+
+  A. Introduction
+  B. Compatibility
+  C. Dependencies
+  D. Usage
+
+-------------------------------------------------------------------------------
+A. Introduction
+
+The Warden Server Test Suite is a collection of high-level functional tests
+(black-box testing), covering the most important interfaces of the Warden
+Server.
+
+-------------------------------------------------------------------------------
+B. Compatibility
+
+* The test suite, just like the Warden Server, is compatible with both Python2
+  (tested on 2.7) and Python3 (tested on 3.6).
+* Just like Warden Server, the test suite requires a local MySQL installation.
+* It is safe to run the test suite on a production system. For testing,
+  a database distinct from the default production one is used. Also, the user
+  account used for accessing the testing database is set for local login only.
+  To be extra safe, make sure not to name the production database `w3test`.
+
+-------------------------------------------------------------------------------
+C. Dependencies
+
+In addition to the regular Warden Server dependencies, package `unittest2` is
+required to run the test suite. It can be installed by running:
+	pip install unittest2
+or `pip3 install unittest2` for Python3 version
+	or on Debian:
+apt-get install python-unittest2
+	or alternatively:
+apt-get install python3-unittest2
+for Python3 version.
+
+An optional dependency is a code coverage measurement tool `Coverage.py`,
+which can be installed by:
+	pip install coverage
+or `pip3 install coverage` for Python3 version
+or on Debian:
+	apt-get install python-coverage
+or alternatively:
+	apt-get install python3-coverage
+for Python3 version.
+
+-------------------------------------------------------------------------------
+D. Usage
+
+Before running the tests (for the first time), a DB user with required rights
+must be created. An easy way to do it is:
+	./test_warden_server.py --init
+This will prompt for MySQL root password.
+
+Standard usage for testing:
+	./test_warden_server.py
+
+Advanced usage:
+
+	./test_warden_server.py --help
+	usage: test_warden_server.py [-h] [-i] [-n]
+
+	Warden3 Server Test Suite
+
+	optional arguments:
+	  -h, --help     show this help message and exit
+	  -i, --init     Set up an user with rights to CREATE/DROP the
+			 test database
+	  -n, --nopurge  Skip the database purge after running the tests
+
+
+Option -n (--nopurge) is meant for debugging purposes and test development, it
+keeps the test database around for inspection after running the tests.
+
+-------------------------------------------------------------------------------

warden3/warden_server/apache.conf.dist → warden_server/apache22.conf.dist

 SSLEngine on
 
-SSLVerifyClient require
+SSLVerifyClient optional
 SSLVerifyDepth 4
 SSLOptions +StdEnvVars +ExportCertData
 
 #SSLCipherSuite ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP:+eNULL
 
-SSLCertificateFile      /opt/warden_server_3/etc/cert.pem
-SSLCertificateKeyFile   /opt/warden_server_3/etc/key.pem
-SSLCACertificateFile    /opt/warden_server_3/etc/tcs-ca-bundle.pem
+SSLCertificateFile      /opt/warden_server/etc/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/etc/key.pem
+SSLCACertificateFile    /opt/warden_server/etc/tcs-ca-bundle.pem
 
-WSGIScriptAlias /warden3 /opt/warden_server_3/warden_server.wsgi
+WSGIScriptAlias /warden3 /opt/warden_server/warden_server.wsgi
 
-<Directory /opt/warden_server_3/warden_server.wsgi>
+<Directory /opt/warden_server/warden_server.wsgi>
     Order allow,deny
     Allow from all
 </Directory>

warden_server/apache24.conf.dist

+SSLEngine on
+
+SSLVerifyClient optional
+SSLVerifyDepth 4
+SSLOptions +StdEnvVars +ExportCertData
+
+#SSLCipherSuite ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP:+eNULL
+
+SSLCertificateFile      /opt/warden_server/etc/cert.pem
+SSLCertificateKeyFile   /opt/warden_server/etc/key.pem
+SSLCACertificateFile    /opt/warden_server/etc/tcs-ca-bundle.pem
+
+WSGIScriptAlias /warden3 /opt/warden_server/warden_server.wsgi
+
+<DirectoryMatch /opt/warden_server/warden_server.wsgi>
+	Require all granted
+</DirectoryMatch>

warden_server/test_warden_server.py

+#!/usr/bin/python
+"""Warden3 Server Test Suite"""
+
+from __future__ import print_function
+import argparse
+import getpass
+import sys
+import warnings
+from os import path
+from copy import deepcopy
+import MySQLdb as my
+from warden_server import build_server
+import warden_server
+
+if sys.version_info >= (3, 10):
+    import unittest
+else:
+    import unittest2 as unittest
+
+if sys.version_info[0] >= 3:
+    from io import StringIO
+else:
+    from StringIO import StringIO
+
+USER = 'warden3test'
+PASSWORD = 'h7w*D>4B)3omcvLM$oJp'
+DB = 'w3test'
+
+
+def setUpModule():      # pylint: disable = locally-disabled, invalid-name
+    """Initialize the test database"""
+    print(__doc__)
+    conn = None
+    try:
+        conn = my.connect(user=USER, passwd=PASSWORD)
+        cur = conn.cursor()
+        with warnings.catch_warnings():                         # The database is not supposed to exist
+            warnings.simplefilter("ignore")
+            cur.execute("DROP DATABASE IF EXISTS %s" % (DB,))   # NOT SECURE
+            cur.execute("CREATE DATABASE %s" % (DB,))           # NOT SECURE
+            cur.execute("USE %s" % (DB,))                       # NOT SECURE
+            with open(path.join(path.dirname(__file__), 'warden_3.0.sql')) as script:
+                statements = ''.join([line.replace('\n', '') for line in script if line[0:2] != '--']).split(';')[:-1]
+                for statement in statements:
+                    cur.execute(statement)
+            cur.execute("INSERT INTO clients VALUES(NULL, NOW(), 'warden-info@cesnet.cz', 'test.server.warden.cesnet.cz', NULL, 1, 'cz.cesnet.warden3test', 'abc', 1, 1, 1, 0)")
+            conn.commit()
+    except my.OperationalError as ex:
+        if conn:
+            conn.rollback()
+            conn.close()
+            conn = None
+        print('Setup failed, have you tried --init ? Original exception: %s' % (str(ex),))
+        exit()
+    finally:
+        if conn:
+            conn.close()
+
+
+NO_PURGE = False
+
+
+def tearDownModule():  # pylint: disable = locally-disabled, invalid-name
+    """Clean up by purging the test database"""
+    if not NO_PURGE:
+        conn = my.connect(user=USER, passwd=PASSWORD)
+        cur = conn.cursor()
+        cur.execute("DROP DATABASE IF EXISTS %s" % (DB,))       # NOT SECURE
+        conn.commit()
+        conn.close()
+
+
+class ReadableSTR(str):
+    """Mission: To boldly quack like a buffer, like no str has quacked before"""
+    def read(self, content_length=None):
+        """Return own content"""
+        return ReadableSTR(self[0:content_length])
+
+    if getattr(str, 'decode', None) is None:
+        def decode(self, encoding="UTF-8", errors="strict"):    # pylint: disable = locally-disabled, unused-argument
+            """For Py3 return own content, no decoding necessary"""
+            return self
+
+
+class Request(object):
+    """Abstraction layer to perform an WSGI request"""
+    def __init__(self, app, uri, payload=""):
+        env = self.get_environ(uri, payload)
+        self.status = None
+        self.headers = None
+        raw_out = app(env, self.start_response)
+        self.out = [item.decode('ascii') for item in raw_out]
+
+    def __call__(self):
+        return self.status, self.headers, self.out
+
+    @staticmethod
+    def get_environ(uri, payload):
+        """Prepares an (partial) environ for WSGI app, almost like an WSGI server would"""
+        try:
+            full_path, query_string = uri.split('?')
+        except ValueError:
+            full_path = uri
+            query_string = ''
+        path_info = '/' + full_path.split('/')[-1]
+        env = {
+            "REQUEST_URI": uri,
+            "PATH_INFO": path_info,
+            "QUERY_STRING": query_string,
+            "SSL_CLIENT_VERIFY": "SUCCESS",
+            "SSL_CLIENT_S_DN_CN": "cz.cesnet.warden3test",
+            "SSL_CLIENT_CERT": "-----BEGIN CERTIFICATE-----\nMIIDgDCCAmgCCQDEG431XDXZjDANBgkqhkiG9w0BAQsFADCBgTELMAkGA1UEBhMCQ1oxFzAVBgNVBAoMDkNFU05FVCwgYS5sLmUuMQwwCgYDVQQLDAM3MDkxJTAjBgNVBAMMHHRlc3Quc2VydmVyLndhcmRlbi5jZXNuZXQuY3oxJDAiBgkqhkiG9w0BCQEWFXdhcmRlbi1pbmZvQGNlc25ldC5jejAeFw0xODA3MjMxMzMyMjFaFw0xODA4MjIxMzMyMjFaMIGBMQswCQYDVQQGEwJDWjEXMBUGA1UECgwOQ0VTTkVULCBhLmwuZS4xDDAKBgNVBAsMAzcwOTElMCMGA1UEAwwcdGVzdC5zZXJ2ZXIud2FyZGVuLmNlc25ldC5jejEkMCIGCSqGSIb3DQEJARYVd2FyZGVuLWluZm9AY2VzbmV0LmN6MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvgwOv1bv44hyWF7UDAPGdm+PqcbITi/6SVEfCENbMx6DAT+M3ZJlg7aOZyiZ16CRNxrjWizXYYY1H+NhOvlPZwsBcHFvnaBrcBciURMW6AQ+OiIHUONDUV7zqTcyiZ6NDMoNy472UpfNBMYXMtaUjPO33aRYwtl+QjoivU8bhzcSxyr/4P6WnZ7rW2nuHWfUNcGWGVxsRw7E2r4OY3Yr6M4SjKEDTEalByApoOYj2s3oEmeiNPjxKhN0wgD4h38+HcnpmKGZLNFbOEdT/7luA6IwzJ7l0p4ktjgCl/x3/Y6ZBrIZuFCNxjYrdfciD27LmcA5A6nEJ083fa4d+O/H8QIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBc6EtV6FYnFBd735h4zwe2SIaFs2bu1d6COsOsaWe5loInI+oEATThaBlA9QiVamikkug3t2wgro8YcYhp0CMPN1gMxR6GstrBrKafprWp/Dv3+IP8RY+Z2lJ0ivw1MTMipqsCMiB+Lvs2wRVV3xBIXslgI3dbceZXos2bj6CPf3Frho7Z7oRaHetI+1a0T9QqZSug7dUSmYNCd9ZXQ8kFzU3eCFP0JKMqOy75KHIE00xowarDDFjTyyPoHmZviIOsY8ByKGNRDQz/WnZWzghAQjb+7tTFm2deOQua0XIyO7GSIU2xdGbTje4wA3/YiWhkpF8HWpCEAN8G6sMTDEXF\n-----END CERTIFICATE-----",      # pylint: disable = locally-disabled, line-too-long
+            "wsgi.input": ReadableSTR(payload),
+            "CONTENT_LENGTH": len(payload)
+            }
+        return env
+
+    def start_response(self, status, headers):
+        """Mocked start_response to record returned status and headers"""
+        self.status = status
+        self.headers = headers
+
+
+class Warden3ServerTest(unittest.TestCase):
+    """High level Warden3 Server tests"""
+    config = {'log': {'level': 'debug'}, 'validator': {'type': 'NoValidator'}, 'auth': {'type': 'PlainAuthenticator'},
+              'db': {'user': USER, 'password': PASSWORD, 'dbname': DB}, 'handler': {'description': 'Warden Test Server'}}
+
+    getInfo_interface_tests_specific = [
+        ("/getInfo", "403 I'm watching. Authenticate."),
+        ("/getInfo?client=", "403 I'm watching. Authenticate."),
+        ("/getInfo?client=cz.cesnet.warden3test", "403 I'm watching. Authenticate."),
+        ("/getInfo?client=cz.cesnet.warden3test&secret=123", "403 I'm watching. Authenticate."),
+        ("/getInfo?secret=123", "403 I'm watching. Authenticate."),
+    ]
+    getEvents_interface_tests_specific = [
+        ("/getEvents", "403 I'm watching. Authenticate.", None),
+        ("/getEvents?secret=123", "403 I'm watching. Authenticate.", None),
+    ]
+
+    @classmethod
+    def setUpClass(cls):
+        """Pre-test cleanup"""
+        cls.clean_lastid()
+        cls.app = build_server(cls.config)
+
+    @classmethod
+    def clean_lastid(cls):
+        """Cleans the lastid information for all clients"""
+        conn = my.connect(user=USER, passwd=PASSWORD, db=DB)
+        cur = conn.cursor()
+        cur.execute("DELETE FROM events")
+        cur.execute("DELETE FROM last_events")
+        cur.close()
+        conn.commit()
+        conn.close()
+
+    def test_getInfo_interface(self):       # pylint: disable = locally-disabled, invalid-name
+        """Tests the getInfo method invocation"""
+        tests_common = [
+            ("/getInfo?secret=abc", "200 OK"),
+            ("/getInfo?secret=abc&evil=false", "200 OK"),       # RFC3514
+            ("/getInfo?client=cz.cesnet.warden3test&secret=abc", "200 OK"),
+            ("/getInfo?client=asdf.blefub", "403 I'm watching. Authenticate."),
+            ("/getInfo?client=asdf.blefub&secret=abc", "403 I'm watching. Authenticate."),
+            ("/getInfo?secret=abc&self=test", "200 OK"),        # Internal parameter
+        ]
+        for query, expected_status in tests_common + self.getInfo_interface_tests_specific:
+            with self.subTest(query=query, expected_status=expected_status):
+                status, _, _ = Request(self.app, query)()
+                self.assertEqual(status, expected_status)
+
+    def test_getEvents_interface(self):     # pylint: disable = locally-disabled, invalid-name
+        """Tests the getEvents method invocation"""
+        tests_common = [
+            ("/getEvents?secret=abc", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?client=foo", "403 I'm watching. Authenticate.", None),
+            ("/getEvents?secret=abc&foo=bar", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?secret=abc&lastid=1", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?secret=abc&lastid=0", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?secret=abc&lastid=9", "200 OK", ['{"lastid": 1, "events": []}']),
+            ("/getEvents?secret=abc&cat=bflm", "422 Wrong tag or category used in query.", None),
+            ("/getEvents?secret=abc&cat=Other", "200 OK", None),
+            ("/getEvents?secret=abc&tag=Other", "200 OK", None),
+            ("/getEvents?secret=abc&group=Other", "200 OK", None),
+            ("/getEvents?secret=abc&cat=Other&nocat=Test", "422 Unrealizable conditions. Choose cat or nocat option.", None),
+            ("/getEvents?secret=abc&tag=Other&notag=Test", "422 Unrealizable conditions. Choose tag or notag option.", None),
+            ("/getEvents?secret=abc&group=Other&nogroup=Test", "422 Unrealizable conditions. Choose group or nogroup option.", None),
+            ("/getEvents?client=cz.cesnet.warden3test&secret=abc&count=3&id=10", "200 OK", None)
+        ]
+        for query, expected_status, expected_response in tests_common + self.getEvents_interface_tests_specific:
+            with self.subTest(query=query, expected_status=expected_status, expected_response=expected_response):
+                status, _, out = Request(self.app, query)()
+                self.assertEqual(status, expected_status)
+                if expected_response is not None:
+                    self.assertEqual(out, expected_response)
+
+    def test_getDebug_interface(self):      # pylint: disable = locally-disabled, invalid-name
+        """Tests the getDebug method invocation"""
+        tests = [
+            ("/getDebug?secret=abc", "200 OK"),
+            ("/getDebug?client=cz.cesnet.warden3test&secret=abc", "200 OK"),
+            ("/getDebug?secret=abc&self=test", "200 OK"),
+        ]
+        for query, expected_status in tests:
+            with self.subTest(query=query, expected_status=expected_status):
+                status, _, _ = Request(self.app, query)()
+                self.assertEqual(status, expected_status)
+
+    def test_methods(self):
+        """Tests application behaviour in method parsing"""
+        tests = [
+            ("", "404 You've fallen off the cliff."),
+            ("/blefub?client=client&secret=secret", "404 You've fallen off the cliff."),
+            ("/?client=client&secret=secret", "404 You've fallen off the cliff."),
+        ]
+        for query, expected_status in tests:
+            with self.subTest(query=query, expected_status=expected_status):
+                status, _, _ = Request(self.app, query)()
+                self.assertEqual(status, expected_status)
+
+    def test_payload(self):
+        """Tests parsing of transported data"""
+        tests = [
+            ("/getInfo?secret=abc", "", "200 OK", None),
+            ("/getInfo?secret=abc", "[1]", "200 OK", None),
+            ("/getInfo?secret=abc", "{#$%^", "200 OK", None),
+            ("/sendEvents?secret=abc", "", "200 OK", ['{"saved": 0}']),
+            ("/sendEvents?secret=abc", "{'test': 'true'}", "400 Deserialization error.", None),
+            ("/sendEvents?secret=abc", '{"test": "true"}', "400 List of events expected.", None),
+            ("/sendEvents?secret=abc", '[{"test": "true"}]', "422 Event does not bear valid Node attribute", None),
+            ("/sendEvents?secret=abc", '[{"Node": ["test", "test2"]}]', "422 Event does not bear valid Node attribute", None),
+            ("/sendEvents?secret=abc", '[{"Node": ["Name", "test"]}]', "422 Event does not bear valid Node attribute", None),
+            ("/sendEvents?secret=abc", '[{"Node": [{"Name"}]}]', "400 Deserialization error.", None),
+            ("/sendEvents?secret=abc", '[{"Node": [{"Name": "test"}]}]', "422 Node does not correspond with saving client", None),
+            ("/sendEvents?secret=abc", '[{"Node": [{"Name": "cz.cesnet.warden3test"}]}]', "200 OK", ['{"saved": 1}']),
+        ]
+        for query, payload, expected_status, expected_response in tests:
+            with self.subTest(query=query, payload=payload, expected_status=expected_status, expected_response=expected_response):
+                status, _, out = Request(self.app, query, payload)()
+                self.assertEqual(status, expected_status)
+                if expected_response is not None:
+                    self.assertEqual(out, expected_response)
+
+
+class X509AuthenticatorTest(Warden3ServerTest):
+    """Performs the basic test suite using the X509Authenticator"""
+    config = deepcopy(Warden3ServerTest.config)
+    config['auth']['type'] = 'X509Authenticator'
+
+
+class X509NameAuthenticatorTest(Warden3ServerTest):
+    """Performs the basic test suite using the X509NameAuthenticator"""
+    config = deepcopy(Warden3ServerTest.config)
+    config['auth']['type'] = 'X509NameAuthenticator'
+
+    getInfo_interface_tests_specific = [
+        ("/getInfo", "200 OK"),
+        ("/getInfo?client=", "200 OK"),
+        ("/getInfo?client=cz.cesnet.warden3test", "200 OK"),
+        ("/getInfo?client=cz.cesnet.warden3test&secret=123", "200 OK"),
+        ("/getInfo?secret=123", "200 OK"),
+    ]
+    getEvents_interface_tests_specific = [
+        ("/getEvents", "200 OK", None),
+        ("/getEvents?secret=123", "200 OK", None),
+    ]
+
+
+class WScliTest(unittest.TestCase):
+    """Tester of the Warden Server command line interface"""
+    @classmethod
+    def setUpClass(cls):
+        cls.config = {'log': {'level': 'debug'}, 'validator': {'type': 'NoValidator'}, 'auth': {'type': 'PlainAuthenticator'},
+                      'db': {'user': USER, 'password': PASSWORD, 'dbname': DB}, 'handler': {'description': 'Warden Test Server'}}
+        warden_server.server = build_server(cls.config)
+
+    @staticmethod
+    def do_cli(command_line):
+        """Performs the command line action requested by argv and presents the results"""
+        argv_backup = sys.argv
+        sys.argv = command_line
+        out = StringIO()
+        err = StringIO()
+        sys.stdout = out
+        sys.stderr = err
+        try:
+            args = warden_server.get_args()
+            command = args.command
+            subargs = vars(args)
+            del subargs["command"]
+            del subargs["config"]
+            ret = command(**subargs)
+        except SystemExit as sys_exit:
+            ret = sys_exit.code
+        sys.stdout = sys.__stdout__
+        sys.stderr = sys.__stderr__
+        sys.argv = argv_backup
+        return ret, out.getvalue(), err.getvalue()
+
+    @staticmethod
+    def do_sql_select(query, params):
+        """Reads data from database"""
+        conn = my.connect(user=USER, passwd=PASSWORD, db=DB)
+        cur = conn.cursor()
+        cur.execute(query, params)
+        result = cur.fetchall()
+        cur.close()
+        conn.close()
+        return result
+
+    def test_list(self):
+        """Tests the list command line option"""
+        tests = [
+            (['list'], 0, 4),
+            (['list', '--id=1'], 0, 4),
+            (['list', '--id=1000'], 0, 3),
+            (['list', '--id', '1'], 0, 4),
+            (['list', '--id', '1000'], 0, 3),
+        ]
+        for supplied_arguments, expected_return, output_lines in tests:
+            with self.subTest(supplied_arguments=supplied_arguments, expected_return=expected_return, output_lines=output_lines):
+                ret, out, _ = self.do_cli(['./warden_server.py'] + supplied_arguments)
+                self.assertEqual(ret, expected_return)
+                self.assertEqual(len(out.split('\n')), output_lines)
+
+    def test_register_modify(self):
+        """Tests the client registration and its modification"""
+        tests = [
+            (['register', '-n', 'cz.cesnet.warden.test2', '-h', 'test2.warden.cesnet.cz', '-r', 'warden-info@cesnet.cz'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--novalid'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 0, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--valid'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--valid', '--novalid'], 2,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--read'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 0, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--noread', '--write'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 0, 0, 1, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--debug', '--read'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 1, 1, 1, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--notest', '--nodebug'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '--notest', '--nodebug'], 2,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', '1000', '--notest', '--nodebug'], 251,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test2', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-n', 'cz.cesnet.warden.test3'], 0,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-n', '..'], 254,
+             (('warden-info@cesnet.cz', 'test2.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-h', 'test3.warden.cesnet.cz'], 0,
+             (('warden-info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-h', ''.zfill(256)], 253,
+             (('warden-info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-h', '..'], 253,
+             (('warden-info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'warden-info@cesnet.cz, info@cesnet.cz'], 0,
+             (('warden-info@cesnet.cz, info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'warden-info@cesnet.cz ,info@cesnet.cz'], 0,
+             (('warden-info@cesnet.cz ,info@cesnet.cz', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'Warden Info <warden-info@cesnet.cz>'], 0,
+             (('Warden Info <warden-info@cesnet.cz>', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'Other Info <other-info@x.cz'], 252,
+             (('Warden Info <warden-info@cesnet.cz>', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'Other other@x.cz'], 252,
+             (('Warden Info <warden-info@cesnet.cz>', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'a@b, '], 252,
+             (('Warden Info <warden-info@cesnet.cz>', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'a@b'], 0,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', '@'], 252,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'abc'], 252,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-r', 'a@b@c'], 252,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-n', 'cz.cesnet.warden.test3'], 250,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-s', 'abc'], 249,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', None, 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-s', 'top_secret'], 0,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '-s', 'top_secret'], 249,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, None),)),
+            (['modify', '-i', 'CLIENT_ID', '--note', ''.zfill(1024)], 0,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, ''.zfill(1024)),)),
+            (['modify', '-i', 'CLIENT_ID', '--note', 'Valid until: 18.01.2038'], 0,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, 'Valid until: 18.01.2038'),)),
+            (['modify', '-i', 'CLIENT_ID', '--note', 'Valid until:', '20.1.2038'], 2,
+             (('a@b', 'test3.warden.cesnet.cz', 'cz.cesnet.warden.test3', 'top_secret', 1, 1, 0, 1, 0, 'Valid until: 18.01.2038'),)),
+        ]
+        test_sql = "SELECT requestor, hostname, name, secret, valid, clients.read, debug, clients.write, test, note FROM clients WHERE id = %s"
+        client_id = None
+        for supplied_arguments, expected_return, expected_sql_result in tests:
+            with self.subTest(supplied_arguments=supplied_arguments, expected_return=expected_return, expected_sql_result=expected_sql_result):
+                supplied_arguments = [entry.replace('CLIENT_ID', str(client_id)) for entry in supplied_arguments]
+                ret, out, _ = self.do_cli(['./warden_server.py'] + supplied_arguments)
+                self.assertEqual(ret, expected_return)
+                try:
+                    client_id = int(out.split('\n')[-2].split(' ')[0])
+                except IndexError:      # No modification was performed, keep the previous client_id
+                    pass
+                result = self.do_sql_select(test_sql, (client_id,))
+                self.assertEqual(result, expected_sql_result)
+
+
+def init_user():
+    """DB user rights setup"""
+    conn = None
+    try:
+        conn = my.connect(user='root', passwd=getpass.getpass('Enter MySQL Root password:'))
+        with conn.cursor() as cur:
+            cur.execute("CREATE USER IF NOT EXISTS %s@'localhost' IDENTIFIED BY %s", (USER, PASSWORD))
+            cur.execute("GRANT SELECT, INSERT, UPDATE, CREATE, DELETE, DROP ON *.* TO %s@'localhost'", (USER,))
+        conn.commit()
+        print("DB User set up successfuly")
+    except my.OperationalError as ex:
+        if conn:
+            conn.rollback()
+            conn.close()
+            conn = None
+        print('Connection unsuccessful, bad password? Original exception: %s' % (str(ex)))
+        exit()
+    except KeyboardInterrupt:
+        print("\nCancelled!")
+        exit()
+    finally:
+        if conn:
+            conn.close()
+
+
+def main():
+    """Parses arguments and acts accordingly"""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('-i', '--init', action='store_true', help='Set up an user with rights to CREATE/DROP the test database')
+    parser.add_argument('-n', '--nopurge', action='store_true', help='Skip the database purge after running the tests')
+
+    args = parser.parse_args()
+    if args.init:
+        init_user()
+    else:
+        if args.nopurge:
+            global NO_PURGE    # pylint: disable = locally-disabled, global-statement
+            NO_PURGE = True
+        sys.argv = [sys.argv[0]]
+        unittest.main()
+
+
+if __name__ == "__main__":
+    main()

warden3/warden_server/warden_3.0.sql → warden_server/warden_3.0.sql

@@ -27,7 +27,7 @@ SET time_zone = "+00:00";
 --
 
 CREATE TABLE IF NOT EXISTS `categories` (
-  `id` int(11) NOT NULL,
+  `id` int UNSIGNED NOT NULL,
   `category` varchar(64) NOT NULL,
   `subcategory` varchar(64) DEFAULT NULL,
   `cat_subcat` varchar(129) NOT NULL,
@@ -41,19 +41,21 @@ CREATE TABLE IF NOT EXISTS `categories` (
 --
 
 CREATE TABLE IF NOT EXISTS `clients` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `id` int UNSIGNED NOT NULL AUTO_INCREMENT,
   `registered` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00',
   `requestor` varchar(256) NOT NULL,
   `hostname` varchar(256) NOT NULL,
   `note` text NULL,
-  `valid` tinyint(1) NOT NULL DEFAULT '1',
+  `valid` tinyint UNSIGNED NOT NULL DEFAULT '1',
   `name` varchar(64) NOT NULL,
   `secret` varchar(16) NULL,
-  `read` tinyint(1) NOT NULL DEFAULT '1',
-  `debug` tinyint(1) NOT NULL DEFAULT '0',
-  `write` tinyint(1) NOT NULL DEFAULT '0',
-  `test` int(11) NOT NULL DEFAULT '0',
-  PRIMARY KEY (`id`)
+  `read` tinyint UNSIGNED NOT NULL DEFAULT '1',
+  `debug` tinyint UNSIGNED NOT NULL DEFAULT '0',
+  `write` tinyint UNSIGNED NOT NULL DEFAULT '0',
+  `test` tinyint UNSIGNED NOT NULL DEFAULT '0',
+  PRIMARY KEY (`id`),
+  KEY `clients_1` (`valid`, `secret`, `hostname`),
+  KEY `clients_2` (`valid`, `name`)
 ) ENGINE=InnoDB  DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci AUTO_INCREMENT=1 ;
 
 -- --------------------------------------------------------
@@ -63,14 +65,15 @@ CREATE TABLE IF NOT EXISTS `clients` (
 --
 
 CREATE TABLE IF NOT EXISTS `events` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
+  `id` bigint UNSIGNED NOT NULL AUTO_INCREMENT,
   `received` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  `client_id` int(11) NOT NULL,
+  `client_id` int UNSIGNED NOT NULL,
   `data` longtext NOT NULL,
-  `valid` tinyint(1) NOT NULL DEFAULT '1',
+  `valid` tinyint UNSIGNED NOT NULL DEFAULT '1',
   PRIMARY KEY (`id`),
-  KEY `id` (`id`,`client_id`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8  DEFAULT COLLATE utf8_unicode_ci AUTO_INCREMENT=1 ;
+  KEY `id` (`id`,`client_id`),
+  KEY `received` (`received`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8  DEFAULT COLLATE utf8_unicode_ci AUTO_INCREMENT=2 ;
 
 -- --------------------------------------------------------
 
@@ -79,8 +82,8 @@ CREATE TABLE IF NOT EXISTS `events` (
 --
 
 CREATE TABLE IF NOT EXISTS `event_category_mapping` (
-  `event_id` int(11) NOT NULL,
-  `category_id` int(11) NOT NULL,
+  `event_id` bigint UNSIGNED NOT NULL,
+  `category_id` int UNSIGNED NOT NULL,
   KEY `event_id_2` (`event_id`,`category_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci;
 
@@ -91,8 +94,8 @@ CREATE TABLE IF NOT EXISTS `event_category_mapping` (
 --
 
 CREATE TABLE IF NOT EXISTS `event_tag_mapping` (
-  `event_id` int(11) NOT NULL,
-  `tag_id` int(11) NOT NULL,
+  `event_id` bigint UNSIGNED NOT NULL,
+  `tag_id` int UNSIGNED NOT NULL,
   KEY `event_id_2` (`event_id`,`tag_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8 DEFAULT COLLATE utf8_unicode_ci;
 
@@ -103,9 +106,9 @@ CREATE TABLE IF NOT EXISTS `event_tag_mapping` (
 --
 
 CREATE TABLE IF NOT EXISTS `last_events` (
-  `id` int(11) NOT NULL AUTO_INCREMENT,
-  `client_id` int(11) NOT NULL,
-  `event_id` int(11) NOT NULL,
+  `id` int UNSIGNED NOT NULL AUTO_INCREMENT,
+  `client_id` int UNSIGNED NOT NULL,
+  `event_id` bigint UNSIGNED NOT NULL,
   `timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
   KEY `client_id` (`client_id`,`event_id`)
@@ -118,7 +121,7 @@ CREATE TABLE IF NOT EXISTS `last_events` (
 --
 
 CREATE TABLE IF NOT EXISTS `tags` (
-  `id` int(11) NOT NULL,
+  `id` int UNSIGNED NOT NULL,
   `tag` varchar(64) NOT NULL,
   KEY `id_tag_name` (`id`,`tag`),
   KEY `tag_name` (`tag`)

warden3/warden_server/warden_server.cfg.dist → warden_server/warden_server.cfg.dist

@@ -7,5 +7,10 @@
         "send_events_limit": 500,
         "get_events_limit": 1000,
         "description": "Warden 3 distribution config"
+    },
+    "DB": {
+        "user": "warden",
+        "password": "EXAMPLE",
+        "dbname": "warden3"
     }
 }
\ No newline at end of file