Merge branch 'warden-3' of homeproj.cesnet.cz:warden into warden-3

warden3/contrib/connectors/hp-labrea/labrea-idea.py

@@ -259,13 +259,15 @@ class IdeaGen(object):
                 "category": ["Recon.Scanning"],
                 "description": "TCP connections/scan",
                 "template": "labrea-001",
-                "note": "Connections from remote host to never assigned IP"
+                "note": "Connections from remote host to never assigned IP",
+                "proto": ["tcp"]
             },
             "ping": {
                 "category": ["Recon.Scanning"],
                 "description": "Ping scan",
                 "template": "labrea-002",
-                "note": "Ping requests from remote host to never assigned IP"
+                "note": "Ping requests from remote host to never assigned IP",
+                "proto": ["icmp"]
             },
             "synack": {
                 "category": ["Availability.DoS"],
@@ -273,7 +275,8 @@ class IdeaGen(object):
                 "template": "labrea-003",
                 "note": "Unsolicited SYN/ACK packet received from remote host to never assigned IP",
                 "source_type": ["Backscatter"],
-                "source_to_target": True
+                "source_to_target": True,
+                "proto": ["tcp"]
             }
         }
 
@@ -284,7 +287,7 @@ class IdeaGen(object):
         tmpl = self.template[template]
         isource = {
             "IP6" if ":" in src else "IP4": [src],
-            "Proto": ["tcp"]
+            "Proto": tmpl["proto"]
         }
         if "source_type" in tmpl:
             isource["Type"] = tmpl["source_type"]
@@ -296,7 +299,7 @@ class IdeaGen(object):
             folded_tgt.setdefault(frozenset(ports), []).append(tgt)
         itargets = []
         for ports, tgt in folded_tgt.items():
-            itarget = {"Proto": ["tcp"]}
+            itarget = {"Proto": tmpl["proto"]}
             tgts4 = [ip for ip in tgt if ":" not in ip]
             tgts6 = [ip for ip in tgt if ":" in ip]
             if tgts4:
@@ -314,7 +317,7 @@ class IdeaGen(object):
         idea = {
             "Format": "IDEA0",
             "ID": str(uuid.uuid4()),
-            "Category": tmpl["category"] + ["Test"] if self.test else [],
+            "Category": tmpl["category"] + (["Test"] if self.test else []),
             "Description": tmpl["description"],
             "DetectTime": self.format_timestamp(detect_time),
             "EventTime": self.format_timestamp(event_time),

warden3/contrib/warden_filer/README

@@ -135,11 +135,11 @@ directories must obey simple protocols, which use atomic "rename" to avoid
 locking issues.
    Also, your directory (and its structure) _must_ reside on the same
 filesystem to keep "rename" atomic. _Never_ try to mount some of the
-subdirectories ("temp", "incoming", "errors") from other filesystem.
+subdirectories ("tmp", "incoming", "errors") from other filesystem.
 
  1. Inserting file
 
-   * The file you want to create _must_ be created in the "temp" subdirectory
+   * The file you want to create _must_ be created in the "tmp" subdirectory
      first, _not_ "incoming". Filename is arbitrary, but must be unique among
      all subdirectories.
 
@@ -148,7 +148,7 @@ subdirectories ("temp", "incoming", "errors") from other filesystem.
      or complete.
 
    For simple usage (bash scripts, etc.), just creating sufficiently random
-   filename in "temp" and then moving into "incoming" may be enough.
+   filename in "tmp" and then moving into "incoming" may be enough.
    Concatenating $RANDOM couple of times will do. :)
    
    For advanced or potentially concurrent usage inserting enough of unique
@@ -159,7 +159,7 @@ subdirectories ("temp", "incoming", "errors") from other filesystem.
 
  2. Picking up file
 
-   * Rename the file to work with into "temp" directory.
+   * Rename the file to work with into "tmp" directory.
 
    * Do whatever you want with contents, and when finished, rename file back
      into "incoming", or remove, or move somewhere else, or move into "errors"

warden3/contrib/warden_filer/warden_filer.py

@@ -65,9 +65,9 @@ class NamedFile(object):
 
 class SafeDir(object):
     """ Maildir like directory for safe file exchange.
-        - Producers are expected to drop files into "temp" under globally unique
+        - Producers are expected to drop files into "tmp" under globally unique
           filename and rename it into "incoming" atomically (newfile method)
-        - Workers pick files in "incoming", rename them into "temp",
+        - Workers pick files in "incoming", rename them into "tmp",
           do whatever they want, and either discard them or move into
           "errors" directory
     """
@@ -76,7 +76,7 @@ class SafeDir(object):
         self.path = self._ensure_path(p)
         self.incoming = self._ensure_path(path.join(self.path, "incoming"))
         self.errors = self._ensure_path(path.join(self.path, "errors"))
-        self.temp = self._ensure_path(path.join(self.path, "temp"))
+        self.temp = self._ensure_path(path.join(self.path, "tmp"))
         self.hostname = socket.gethostname()
         self.pid = os.getpid()
 
@@ -116,7 +116,7 @@ class SafeDir(object):
         #       which checked uniqueness among all directories by atomic
         #       links.
 
-        # First find and open name unique within temp
+        # First find and open name unique within tmp
         tmpname = None
         while not tmpname:
             tmpname = self._get_new_name()

warden3/contrib/warden_ra/README

+Warden Registration Authority for Warden 3.X
+============================================
+
+Introduction
+------------
+
+Warden RA is a certificate registration authority for Warden_ server.
+It is meant to support the client registration process and simplification
+of the credential transport.
+
+As Warden clients are authenticated by X509 certificate, the usual certificate
+generation process can be used - local key and certificate request gets
+generated, the request is submitted to registration authority, and after
+review, certificate is issued and delivered back.
+
+However in centralised Warden setup, it is useful to be able to preallow
+certificate for particular client during negotiation phase (thus removing
+another round-trip).
+
+This is done by issuing 'token' by Warden registration officer during client
+registration, which is just a oneshot password, allowing sending the request
+and getting new certificate in one step through web API.
+
+Password is meant only for completely new clients or unusual situations,
+however RA also allows prolongation - generating new certificate by using
+old certificate (which must be still valid, of course) instead of password.
+
+The application script, which can be distributed to newly registered clients,
+is also included.
+
+Dependencies
+------------
+
+ 1. Platform
+
+    Python 2.7+
+    Apache 2.2+
+    mod_wsgi 3.3+
+    EJBCA_ 3.9+
+
+
+Registration process
+--------------------
+
+New client credentials
+``````````````````````
+
+After succesful negotiation of new Warden client parameters, the registration
+officer enables new certificate generation by issuing (on the server side):
+
+    warden_ra.py applicant --name org.example.warden.client
+
+The tool generates and sets one time password on the registration authority
+side, and this password can be sent (preferably through the secured channel)
+to the new client administrator along with other setup information.
+
+The client administrator runs the application script with application
+password:
+
+    warden_apply.sh org.example.warden.client P4SSW0RD
+
+The script creates new X509 key, CSR certificate request and makes call to
+the Warden RA web service, where it obtains the new complete certificate.
+
+
+Prolonging existing client credentials
+``````````````````````````````````````
+
+The client administrator runs the application script with his existing valid
+Warden credentials, which he needs to prolong:
+
+    warden_apply.sh org.example.warden.client cert.pem key.pem
+
+The script creates new X509 key, CSR certificate request and makes call to
+the Warden RA web service, where it obtains the new complete certificate.
+
+
+Installation
+------------
+
+As for now, correctly configured and running EJBCA_ PKI is necessary. PKI part
+of the RA is however pluggable, so simple openssl backend is also planned.
+
+
+This depends heavily on your distribution and Apache configuration.
+Basically you need to create and include apache.conf:
+
+  Include /opt/warden_ra/apache22.conf
+
+or paste the contents into whichever Directory, Location or VirtualHost
+you dedicate for Warden RA. Note that you have to use different host than
+the one for Warden server, as Warden RA needs different Apache options
+for SSL authentication.
+
+You may need to change paths to certificate/key/ca material, path to
+warden_server.wsgi and web path alias.
+
+Note that warden_ra itself is NOT thread safe, so included configuration
+expects Apache with mpm-prefork worker, or you will have to configure
+mod_wsgi as separate process with threads=1.
+
+Also, for warden_server.wsgi, you can use warden_server.wsgi.dist as
+a template. You will possibly need to change at least configuration
+file path.
+
+ * Configure Warden RA (see next chapter)
+
+ * Reload Apache
+
+
+Configuration
+-------------
+
+   Configuration is JSON object in file (warden_server.cfg by default),
+however, lines starting with "#" or "//" are allowed and will be ignored as
+comments. File must contain valid JSON object, containing configuration. See
+also warden_server.cfg.dist as example.
+
+   Top level JSON object contains subsections, which configure particular
+aspect of the server. Each subsection can contain "type" key, which chooses
+particular implementation object of the aspect, for example type of logger
+(file, syslog), such as:
+
+   {
+      "Log": {
+         "type": "SysLogger",
+         ...
+      },
+      "DB": { ... }
+   }
+
+Sections and their "type" objects can be:
+
+    Log: FileLogger, SysLogger
+    Auth: OptionalAuthenticator
+    Registry: EjbcaRegistry
+    Handler: CertHandler
+
+"type" keyword is not mandatory, if not specified, first implementation
+object from particular section list is used ("FileLogger" for example).
+
+Object function and configuration keys are as follows:
+
+   FileLogger: logging into file on the filesystem
+      filename: name of the file, defaults to "warden_ra.log" at
+                installation directory
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+   SysLogger: logging into unix syslog
+      socket: path to syslog socket, defaults to "/dev/log"
+      facility: syslog facility, defaults to "daemon"
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+    OptionalAuthenticator: authenticate based on X509 certificate, or
+        signal the password auth for the registry
+
+    CertHandler: the main certificate requestor implementation
+
+    EjbcaRegistry: EJBCA connector configuration
+        url: EJBCA API URL, for example "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl"
+        cert: certificate for authentication to EJBCA, defaults to "warden_ra.cert.pem"
+        key: key for authentication to EJBCA, defaults to "warden_ra.key.pem"
+        ca_name: name of the CA, dedicated for Warden, defaults to "Example CA"
+        certificate_profile_name: name of the EJBCA certificate profile, defaults to "Example"
+        end_entity_profile_name: name of the EJBCA entity profile, defaults to "Example EE"
+        subject_dn_template: template for the DN generation, defaults to "DC=cz,DC=example-ca,DC=warden,CN=%s"
+        username_suffix: suffix, which will be added to EJBCA entities, defaults to "@warden"
+
+
+Command line
+------------
+
+Whe run from the command line, RA allows for client and request management.
+
+
+    warden_ra.py [--help] [-c CONFIG] [-v]
+                        {list,register,applicant,request,gencert} ...
+
+    Warden server certificate registry
+    
+    arguments:
+      --help                show this help message and exit
+      -c CONFIG, --config CONFIG
+                            path to configuration file
+      -v, --verbose         be more chatty
+    
+    commands:
+      {list,register,applicant,request,gencert}
+        list                list clients
+        register            register client
+        applicant           allow for certificate application
+        request             generate CSR
+        gencert             get new certificate
+
+
+    warden_ra.py list [--help] [--name NAME]
+    
+    List registered clients.
+    
+    arguments:
+      --help       show this help message and exit
+      --name NAME  client name
+
+
+    warden_ra.py register [--help] --name NAME --admins
+                                 [ADMINS [ADMINS ...]]
+    
+    Add client registration entry.
+    
+    arguments:
+      --help                show this help message and exit
+      --name NAME           client name
+      --admins [ADMINS [ADMINS ...]]
+                            administrator list
+
+
+    warden_ra.py applicant [--help] --name NAME [--password PASSWORD]
+    
+    Set client into certificate application mode and set its password
+    
+    optional arguments:
+      --help               show this help message and exit
+      --name NAME          client name
+      --password PASSWORD  password for application (will be autogenerated if not
+                           set)
+
+    
+.. _Warden: https://warden.cesnet.cz/
+.. _EJBCA: https://www.ejbca.org/
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

warden3/contrib/warden_ra/apache22.conf

 SSLEngine on
 
-SSLVerifyClient none
+SSLVerifyClient optional
 SSLOptions +StdEnvVars +ExportCertData
 
 SSLCertificateFile      /opt/warden_server_3/cert.pem

warden3/contrib/warden_ra/warden_apply.sh

 #!/bin/bash
 
-url='https://warden-ra.cesnet.cz/warden-ra/getCert'
+url='https://warden-hub.cesnet.cz/warden-ra/getCert'
 key=key.pem
 csr=csr.pem
 cert=cert.pem
@@ -8,12 +8,14 @@ result=${TMPDIR:-${TMP:-/tmp}}/cert.$$.$RANDOM
 config=${TMPDIR:-${TMP:-/tmp}}/conf.$$.$RANDOM
 client="$1"
 password="$2"
+incert="$2"
+inkey="$3"
 
 trap 'rm -f "$config $result"' INT TERM HUP EXIT
 
 function flee { echo -e "$1"; exit $2; }
 
-[ -z "$client" -o -z "$password" ] && flee "Usage: ${0%.*} client.name password" 255
+[ -z "$client" -o -z "$password" ] && flee "Usage: ${0%.*} client.name password\n       ${0%.*} client.name cert_file key_file" 255
 
 for n in openssl curl; do
     command -v "$n" 2>&1 >/dev/null || flee "Haven't found $n binary." 251
@@ -29,7 +31,11 @@ echo -e "default_bits=2048\ndistinguished_name=rdn\nprompt=no\n[rdn]\ncommonName
 
 openssl req -new -nodes -batch -keyout "$key" -out "$csr" -config "$config" || flee "Error generating key/certificate request." 252
 
-curl --progress-bar --request POST --data-binary '@-' "$url?name=$client&password=$password" < "$csr" > "$result"
+if [ -z "$inkey" ]; then
+	curl --progress-bar --request POST --data-binary '@-' "$url?name=$client&password=$password" < "$csr" > "$result"
+else
+	curl --progress-bar --request POST --data-binary '@-' --cert "$incert" --key "$inkey" "$url?name=$client" < "$csr" > "$result"
+fi
 
 case $(<$result) in '-----BEGIN CERTIFICATE-----'*)
     mv "$result" "$cert"

warden3/contrib/warden_ra/warden_ra.cfg.dist

@@ -4,6 +4,7 @@
         "level": "info"
     },
     "Registry": {
+        "type": "EjbcaRegistry",
         "url": "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl",
         "cert": "warden_ra.cert.pem",
         "key": "warden_ra.key.pem",

warden3/warden_client/warden_client.py

@@ -13,7 +13,7 @@ from operator import itemgetter
 
 fix_logging_filename = str if version_info<(2, 7) else lambda x: x
 
-if version_info > (3, 0):
+if version_info[0] >= 3:
     import http.client as httplib
     from urllib.parse import urlparse
     from urllib.parse import urlencode

warden3/warden_server/README

@@ -123,7 +123,8 @@ particular implementation object of the aspect, for example type of logger
 
 		Log: FileLogger, SysLogger
 		DB: MySQL
-		Auth: X509Authenticator, PlainAuthenticator
+		Auth: X509Authenticator, X509NameAuthenticator,
+              X509MixMatchAuthenticator,PlainAuthenticator
 		Validator: JSONSchemaValidator, NoValidator
 		Handler: WardenHandler
 
@@ -144,7 +145,17 @@ object from particular section list is used ("FileLogger" for example).
 
    X509Authenticator: authenticate based on certificate chain validation,
        hostname corresponding with certificate CN or SubjectAltName and
-       optionally shared secret
+       optionally shared secret (note that more clients on one machine
+       will have to have the certificate with the same hostname, clients
+       than can be differentiated by separate secrets).
+       This method is OBSOLETE.
+
+    X509NameAuthenticator: authenticate based on certificate chain validation,
+        certificate CN must correspond with client _name_, NOT hostname.
+
+    X509MixMatchAuthenticator: automatically choose X509Authenticator or
+        X509NameAuthenticator based on existence of 'secret' in query. Allows
+        for seamless transition of clients between two authentication methods.
 
    PlainAuthenticator: authenticate based on client name or shared secret, usable
       over plain HTTP connection or HTTPS without client certificate - note that

warden3/warden_server/apache22.conf.dist

 SSLEngine on
 
-SSLVerifyClient require
+SSLVerifyClient optional
 SSLVerifyDepth 4
 SSLOptions +StdEnvVars +ExportCertData
 

warden3/warden_server/apache24.conf.dist

 SSLEngine on
 
-SSLVerifyClient require
+SSLVerifyClient optional
 SSLVerifyDepth 4
 SSLOptions +StdEnvVars +ExportCertData
 

warden3/warden_server/warden_3.0.sql

@@ -73,7 +73,7 @@ CREATE TABLE IF NOT EXISTS `events` (
   PRIMARY KEY (`id`),
   KEY `id` (`id`,`client_id`),
   KEY `received` (`received`)
-  ) ENGINE=InnoDB DEFAULT CHARSET=utf8  DEFAULT COLLATE utf8_unicode_ci AUTO_INCREMENT=1 ;
+) ENGINE=InnoDB DEFAULT CHARSET=utf8  DEFAULT COLLATE utf8_unicode_ci AUTO_INCREMENT=2 ;
 
 -- --------------------------------------------------------