warden_server.py

    },
    MySQL: {
        "req": {"type": "obj", "default": "req"},
        "log": {"type": "obj", "default": "log"},
        "host": {"type": "str", "default": "localhost"},
        "user": {"type": "str", "default": "warden"},
        "password": {"type": "str", "default": ""},
        "dbname": {"type": "str", "default": "warden3"},
        "port": {"type": "natural", "default": 3306},
        "retry_pause": {"type": "natural", "default": 3},
        "retry_count": {"type": "natural", "default": 3},
        "event_size_limit": {"type": "natural", "default": 5*1024*1024},
        "catmap_filename": {"type": "filepath", "default": path.join(path.dirname(__file__), "catmap_db.json")},
        "tagmap_filename": {"type": "filepath", "default": path.join(path.dirname(__file__), "tagmap_db.json")}
    },
    PostgreSQL: {
        "req": {"type": "obj", "default": "req"},
        "log": {"type": "obj", "default": "log"},
        "host": {"type": "str", "default": "localhost"},
        "user": {"type": "str", "default": "warden"},
        "password": {"type": "str", "default": ""},
        "dbname": {"type": "str", "default": "warden3"},
        "port": {"type": "natural", "default": 5432},
        "retry_pause": {"type": "natural", "default": 3},
        "retry_count": {"type": "natural", "default": 3},
        "event_size_limit": {"type": "natural", "default": 5*1024*1024},
        "catmap_filename": {"type": "filepath", "default": path.join(path.dirname(__file__), "catmap_db.json")},
        "tagmap_filename": {"type": "filepath", "default": path.join(path.dirname(__file__), "tagmap_db.json")}
    },
    WardenHandler: {
        "req": {"type": "obj", "default": "req"},
        "log": {"type": "obj", "default": "log"},
        "validator": {"type": "obj", "default": "validator"},
        "db": {"type": "obj", "default": "DB"},
        "auth": {"type": "obj", "default": "auth"},
        "send_events_limit": {"type": "natural", "default": 500},
        "get_events_limit": {"type": "natural", "default": 1000},
        "description": {"type": "str", "default": ""}
    },
    Server: {
        "req": {"type": "obj", "default": "req"},
        "log": {"type": "obj", "default": "log"},
        "auth": {"type": "obj", "default": "auth"},
        "handler": {"type": "obj", "default": "handler"}
    }
}


def build_server(conf, section_order=section_order, section_def=section_def, param_def=param_def):

    objects = {}    # Already initialized objects

    # Functions for validation and conversion of config values
    def facility(name):
        return int(getattr(logging.handlers.SysLogHandler, "LOG_" + name.upper()))

    def loglevel(name):
        return int(getattr(logging, name.upper()))

    def natural(name):
        num = int(name)
        if num < 1:
            raise ValueError("Not a natural number")
        return num

    def filepath(name):
        # Make paths relative to dir of this script
        return path.join(path.dirname(__file__), name)

    def obj(name):
        return objects[name.lower()]

    # Typedef dictionary
    conv_dict = {
        "facility": facility,
        "loglevel": loglevel,
        "natural": natural,
        "filepath": filepath,
        "obj": obj,
        "str": str
    }

    def init_obj(sect_name):
        config = dict(conf.get(sect_name, {}))
        sect_name = sect_name.lower()
        sect_def = section_def[sect_name]

        try:    # Object type defined?
            objtype = config["type"]
            del config["type"]
        except KeyError:    # No, fetch default object type for this section
            cls = sect_def[0]
        else:  # Yes, get corresponding class/callable
            names = [o.__name__ for o in sect_def]
            try:
                idx = names.index(objtype)
            except ValueError:
                raise KeyError("Unknown type %s in section %s" % (objtype, sect_name))
            cls = sect_def[idx]

        params = param_def[cls]

        # No surplus parameters? Disallow also 'obj' attributes, these are only
        # to provide default referenced section
        for name in config:
            if name not in params or (name in params and params[name]["type"] == "obj"):
                raise KeyError("Unknown key %s in section %s" % (name, sect_name))

        # Process parameters
        kwargs = {}
        for name, definition in params.items():
            raw_val = config.get(name, definition["default"])
            try:
                type_callable = conv_dict[definition["type"]]
                val = type_callable(raw_val)
            except Exception:
                raise KeyError("Bad value \"%s\" for %s in section %s" % (raw_val, name, sect_name))
            kwargs[name] = val

        try:
            obj_inst = cls(**kwargs)         # run it
        except Exception as e:
            raise KeyError("Cannot initialize %s from section %s: %s" % (
                cls.__name__, sect_name, str(e)))

        objects[sect_name] = obj_inst
        if isinstance(obj_inst, Object):
            # Log only objects here, functions must take care of themselves
            objects["log"].info("Initialized %s" % str(obj_inst))

        return obj_inst

    # Init logging with at least simple stderr StreamLogger
    # Dunno if it's ok within wsgi, but we have no other choice, let's
    # hope it at least ends up in webserver error log
    objects["log"] = StreamLogger()

    # Shared container for common data of ongoing WSGI request
    objects["req"] = Request()

    try:
        # Now try to init required objects
        for o in section_order:
            init_obj(o)
    except Exception as e:
        objects["log"].critical(str(e))
        objects["log"].debug("", exc_info=sys.exc_info())
        return fallback_wsgi

    objects["log"].info("Server ready")

    return objects["server"]


# Command line utilities

def check_config():
    # If we got so far, server object got set up fine
    print("Looks clear.", file=sys.stderr)
    return 0


def list_clients(id=None):
    clients = server.handler.db.get_clients(id)
    lines = [[str(getattr(client, col)) for col in Client._fields] for client in clients]
    col_width = [max(len(val) for val in col) for col in zip(*(lines+[Client._fields]))]
    divider = ["-" * l for l in col_width]
    for line in [Client._fields, divider] + lines:
        print(" ".join([val.ljust(width) for val, width in zip(line, col_width)]))
    return 0


def register_client(**kwargs):
    # argparse does _always_ return something, so we cannot rely on missing arguments
    if kwargs["valid"] is None: kwargs["valid"] = True
    if kwargs["read"] is None: kwargs["read"] = True
    if kwargs["write"] is None: kwargs["write"] = False
    if kwargs["debug"] is None: kwargs["debug"] = False
    if kwargs["test"] is None: kwargs["test"] = True
    return modify_client(id=None, **kwargs)


def modify_client(**kwargs):

    def isValidHostname(hostname):
        if len(hostname) > 255:
            return False
        if hostname.endswith("."):  # A single trailing dot is legal
            hostname = hostname[:-1]  # strip exactly one dot from the right, if present
        disallowed = re.compile(r"[^A-Z\d-]", re.IGNORECASE)
        return all(  # Split by labels and verify individually
            (label and len(label) <= 63  # length is within proper range
             and not label.startswith("-") and not label.endswith("-")  # no bordering hyphens
             and not disallowed.search(label))  # contains only legal characters
            for label in hostname.split("."))

    def isValidNSID(nsid):
        allowed = re.compile(r"^(?:[a-zA-Z_][a-zA-Z0-9_]*\.)*[a-zA-Z_][a-zA-Z0-9_]*$")
        return allowed.match(nsid)

    def isValidEmail(mail):
        allowed = re.compile(r"(^[a-zA-Z0-9_ .%!+-]*(?=<.*>))?(^|(<(?=.*(>))))[a-zA-Z0-9_.%!+-]+@[a-zA-Z0-9-.]+\4?$")   # just basic check
        valid = (allowed.match(ms.strip())for ms in mail.split(','))
        return all(valid)

    def isValidID(id):
        client = server.handler.db.get_clients(id)
        return client and True or False

    if kwargs["name"] is not None:
        kwargs["name"] = kwargs["name"].lower()
        if not isValidNSID(kwargs["name"]):
            print("Invalid client name \"%s\"." % kwargs["name"], file=sys.stderr)
            return 254

    if kwargs["hostname"] is not None:
        kwargs["hostname"] = kwargs["hostname"].lower()
        if not isValidHostname(kwargs["hostname"]):
            print("Invalid hostname \"%s\"." % kwargs["hostname"], file=sys.stderr)
            return 253

    if kwargs["requestor"] is not None and not isValidEmail(kwargs["requestor"]):
        print("Invalid requestor email \"%s\"." % kwargs["requestor"], file=sys.stderr)
        return 252

    if kwargs["id"] is not None and not isValidID(kwargs["id"]):
        print("Invalid id \"%s\"." % kwargs["id"], file=sys.stderr)
        return 251

    for c in server.handler.db.get_clients():
        if kwargs["name"] is not None and kwargs["name"].lower() == c.name:
            print("Clash with existing name: %s" % str(c), file=sys.stderr)
            return 250
        if kwargs["secret"] is not None and kwargs["secret"] == c.secret:
            print("Clash with existing secret: %s" % str(c), file=sys.stderr)
            return 249

    newid = server.handler.db.add_modify_client(**kwargs)

    return list_clients(id=newid)


def load_maps():
    server.handler.db.load_maps()
    return 0


def purge(days=30, lastlog=None, events=None):
    if lastlog is None and events is None:
        lastlog = events = True
    if lastlog:
        count = server.handler.db.purge_lastlog(days)
        print("Purged %d lastlog entries." % count)
    if events:
        count = server.handler.db.purge_events(days)
        print("Purged %d events." % count)
    return 0


def add_client_args(subargp, mod=False):
    subargp.add_argument("--help", action="help", help="show this help message and exit")
    if mod:
        subargp.add_argument(
            "-i", "--id", required=True, type=int,
            help="client id")
    subargp.add_argument(
        "-n", "--name", required=not mod,
        help="client name (in dotted reverse path notation)")
    subargp.add_argument(
        "-h", "--hostname", required=not mod,
        help="client FQDN hostname")
    subargp.add_argument(
        "-r", "--requestor", required=not mod,
        help="requestor email")
    subargp.add_argument(
        "-s", "--secret",
        help="authentication token (use explicit empty string to disable)")
    subargp.add_argument(
        "--note",
        help="client freetext description")

    reg_valid = subargp.add_mutually_exclusive_group(required=False)
    reg_valid.add_argument(
        "--valid", action="store_const", const=True, default=None,
        help="valid client (default)")
    reg_valid.add_argument("--novalid", action="store_const", const=False, dest="valid", default=None)

    reg_read = subargp.add_mutually_exclusive_group(required=False)
    reg_read.add_argument(
        "--read", action="store_const", const=True, default=None,
        help="client is allowed to read (default)")
    reg_read.add_argument("--noread", action="store_const", const=False, dest="read", default=None)

    reg_write = subargp.add_mutually_exclusive_group(required=False)
    reg_write.add_argument(
        "--nowrite", action="store_const", const=False, dest="write", default=None,
        help="client is allowed to send (default - no)")
    reg_write.add_argument("--write", action="store_const", const=True, default=None)

    reg_debug = subargp.add_mutually_exclusive_group(required=False)
    reg_debug.add_argument(
        "--nodebug", action="store_const", const=False, dest="debug", default=None,
        help="client is allowed receive debug output (default - no)")
    reg_debug.add_argument("--debug", action="store_const", const=True, default=None)

    reg_test = subargp.add_mutually_exclusive_group(required=False)
    reg_test.add_argument(
        "--test", action="store_const", const=True, default=None,
        help="client is yet in testing phase (default - yes)")
    reg_test.add_argument("--notest", action="store_const", const=False, dest="test", default=None)


def get_args():
    import argparse
    argp = argparse.ArgumentParser(
        description="Warden server " + VERSION, add_help=False)
    argp.add_argument(
        "--help", action="help",
        help="show this help message and exit")
    argp.add_argument(
        "-c", "--config",
        help="path to configuration file")
    subargp = argp.add_subparsers(title="commands", dest="command")
    subargp.required = True

    subargp_check = subargp.add_parser(
        "check", add_help=False,
        description="Try to setup server based on configuration file.",
        help="check configuration")
    subargp_check.set_defaults(command=check_config)
    subargp_check.add_argument(
        "--help", action="help",
        help="show this help message and exit")

    subargp_reg = subargp.add_parser(
        "register", add_help=False,
        description="Add new client registration entry.",
        help="register new client")
    subargp_reg.set_defaults(command=register_client)
    add_client_args(subargp_reg)

    subargp_mod = subargp.add_parser(
        "modify", add_help=False,
        description="Modify details of client registration entry.",
        help="modify client registration")
    subargp_mod.set_defaults(command=modify_client)
    add_client_args(subargp_mod, mod=True)

    subargp_list = subargp.add_parser(
        "list", add_help=False,
        description="List details of client registration entries.",
        help="list registered clients")
    subargp_list.set_defaults(command=list_clients)
    subargp_list.add_argument(
        "--help", action="help",
        help="show this help message and exit")
    subargp_list.add_argument(
        "--id", action="store", type=int,
        help="client id", default=None)

    subargp_purge = subargp.add_parser(
        "purge", add_help=False,
        description=(
            "Purge old events or lastlog records."
            " Note that lastlog purge retains at least one newest record for each"
            " client, even if it is more than number of 'days' old."),
        help="purge old events or lastlog records")
    subargp_purge.set_defaults(command=purge)
    subargp_purge.add_argument(
        "--help", action="help",
        help="show this help message and exit")
    subargp_purge.add_argument(
        "-l", "--lastlog", action="store_true", dest="lastlog", default=None,
        help="purge lastlog records")
    subargp_purge.add_argument(
        "-e", "--events", action="store_true", dest="events", default=None,
        help="purge events")
    subargp_purge.add_argument(
        "-d", "--days", action="store", dest="days", type=int, default=30,
        help="records older than 'days' back from today will get purged")

    subargp_loadmaps = subargp.add_parser(
        "loadmaps", add_help=False,
        description=(
            "Load 'categories' and 'tags' table from 'catmap_db.json' and 'tagmap_db.json'."
            " Note also that previous content of both tables will be lost."),
        help="load catmap and tagmap into db")
    subargp_loadmaps.set_defaults(command=load_maps)
    subargp_loadmaps.add_argument(
        "--help", action="help",
        help="show this help message and exit")

    return argp.parse_args()


if __name__ == "__main__":
    args = get_args()
    config = path.join(path.dirname(__file__), args.config or "warden_server.cfg")
    server = build_server(read_cfg(config))
    command = args.command
    subargs = vars(args)
    del subargs["command"]
    del subargs["config"]
    if not server or server is fallback_wsgi:
        print("Failed initialization, check configured log targets for reasons.", file=sys.stderr)
        sys.exit(255)
    sys.exit(command(**subargs))