warden_server.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2011-2015 Cesnet z.s.p.o
# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.

import sys
import os
import logging
import logging.handlers
import ConfigParser
from traceback import format_tb
import M2Crypto.X509
import json
import MySQLdb as my
import MySQLdb.cursors as mycursors
import re
import email.utils
from collections import namedtuple
from time import sleep
from urlparse import parse_qs
from os import path
from random import randint
from contextlib import closing

# for local version of up to date jsonschema
sys.path.append(path.join(path.dirname(__file__), "..", "lib"))

from jsonschema import Draft4Validator


VERSION = "3.0-beta2"

class Error(Exception):

    def __init__(self, method=None, req_id=None, errors=None, **kwargs):
        self.method = method
        self.req_id = req_id
        self.errors = [kwargs] if kwargs else []
        if errors:
            self.errors.extend(errors)


    def append(self, _events=None, **kwargs):
        self.errors.append(kwargs)


    def get_http_err_msg(self):
        try:
            err = self.errors[0]["error"]
            msg = self.errors[0]["message"].replace("\n", " ")
        except (IndexError, KeyError):
            err = 500
            msg = "There's NO self-destruction button! Ah, you've just found it..."
        for e in self.errors:
            next_err = e.get("error", 500)
            if err != next_err:
                # errors not same, round to basic err code (400, 500)
                # and use the highest one
                err = max(err//100, next_err//100)*100
            next_msg = e.get("message", "Unknown error").replace("\n", " ")
            if msg != next_msg:
                msg = "Multiple errors"
        return err, msg


    def __str__(self):
        return "\n".join(self.str_err(e) for e in self.errors)


    def log(self, logger, prio=logging.ERROR):
        for e in self.errors:
            logger.log(prio, self.str_err(e))
            info = self.str_info(e)
            if info:
                logger.info(info)
            debug = self.str_debug(e)
            if debug:
                logger.debug(debug)


    def str_err(self, e):
        out = []
        out.append("Error(%s) %s " % (e.get("error", 0), e.get("message", "Unknown error")))
        if "exc" in e and e["exc"]:
            out.append("(cause was %s: %s)" % (e["exc"][0].__name__, str(e["exc"][1])))
        return "".join(out)


    def str_info(self, e):
        ecopy = dict(e)    # shallow copy
        ecopy.pop("req_id", None)
        ecopy.pop("method", None)
        ecopy.pop("error", None)
        ecopy.pop("message", None)
        ecopy.pop("exc", None)
        if ecopy:
            out = "Detail: %s" % (json.dumps(ecopy, default=lambda v: str(v)))
        else:
            out = ""
        return out


    def str_debug(self, e):
        out = []
        if not "exc" in e or not e["exc"]:
            return ""
        exc_tb = e["exc"][2]
        if exc_tb:
            out.append("Traceback:\n")
            out.extend(format_tb(exc_tb))
        return "".join(out)


    def to_dict(self):
        errlist = []
        for e in self.errors:
            ecopy = dict(e)
            ecopy.pop("exc", None)
            errlist.append(ecopy)
        d = {
            "method": self.method,
            "req_id": self.req_id,
            "errors": errlist
        }
        return d



def get_clean_root_logger(level=logging.INFO):
    """ Attempts to get logging module into clean slate state """

    # We want to be able to set up at least stderr logger before any
    # configuration is read, and then later get rid of it and set up
    # whatever administrator requires.
    # However, there can exist only one logger, but we want to get a clean
    # slate everytime we initialize StreamLogger or FileLogger... which
    # is not exactly supported by logging module.
    # So, we look directly inside logger class and clean up handlers/filters
    # manually.
    logger = logging.getLogger(__name__)
    logger.setLevel(level)
    while logger.handlers:
        logger.removeHandler(logger.handlers[0])
    while logger.filters:
        logger.removeFilter(logger.filters[0])
    logger.propagate = False
    return logger



def StreamLogger(stream=sys.stderr, level=logging.DEBUG):
    """ Fallback handler just for setup, not meant to be used from
        configuration file because during wsgi query stdout/stderr
        is forbidden.
    """

    fhand = logging.StreamHandler(stream)
    fform = logging.Formatter('%(asctime)s %(filename)s[%(process)d]: (%(levelname)s) %(message)s')
    fhand.setFormatter(fform)
    logger = get_clean_root_logger(level)
    logger.addHandler(fhand)
    return logger



class LogRequestFilter(logging.Filter):
    """ Filter class, instance of which is added to logger class to add
        info about request automatically into every logline, no matter
        how it came into existence.
    """

    def __init__(self, req):
        logging.Filter.__init__(self)
        self.req = req


    def filter(self, record):
        if self.req.env:
            record.req_preamble = "%08x/%s: " % (self.req.req_id or 0, self.req.path)
        else:
            record.req_preamble = ""
        return True



def FileLogger(req, filename, level=logging.INFO):

    fhand = logging.FileHandler(filename)
    fform = logging.Formatter('%(asctime)s %(filename)s[%(process)d]: (%(levelname)s) %(req_preamble)s%(message)s')
    fhand.setFormatter(fform)
    ffilt = LogRequestFilter(req)
    logger = get_clean_root_logger(level)
    logger.addFilter(ffilt)
    logger.addHandler(fhand)
    logger.info("Initialized FileLogger(req=%s, filename=\"%s\", level=\"%d\")" % (type(req).__name__, filename, level))
    return logger



def SysLogger(req, socket="/dev/log", facility=logging.handlers.SysLogHandler.LOG_DAEMON, level=logging.INFO):

    fhand = logging.handlers.SysLogHandler(address=socket, facility=facility)
    fform = logging.Formatter('%(filename)s[%(process)d]: (%(levelname)s) %(req_preamble)s%(message)s')
    fhand.setFormatter(fform)
    ffilt = LogRequestFilter(req)
    logger = get_clean_root_logger(level)
    logger.addFilter(ffilt)
    logger.addHandler(fhand)
    logger.info("Initialized SysLogger(req=%s, socket=\"%s\", facility=\"%d\", level=\"%d\")" % (type(req).__name__, socket, facility, level))
    return logger



Client = namedtuple("Client",
    ["id", "registered", "requestor", "hostname", "name",
    "secret", "valid", "read", "debug", "write", "test", "note"])


class Object(object):

    def __str__(self):
        return "%s()" % type(self).__name__



class Request(Object):
    """ Simple container for info about ongoing request.
        One instance gets created before server startup, and all other
        configured objects get it as parameter during instantiation.

        Server then takes care of populating this instance on the start
        of wsgi request (and resetting at the end). All other objects
        then can find this actual request info in their own self.req.

        However, only Server.wsgi_app, handler (WardenHandler) exposed
        methods and logging related objects should use self.req directly.
        All other objects should use self.req only as source of data for
        error/exception handling/logging, and should take/return
        necessary data as arguments/return values for clarity on
        which data their main codepaths work with.
    """

    def __init__(self):
        Object.__init__(self)
        self.reset()


    def __str__(self):
        return "%s()" % (type(self).__name__, str(self.env), str(self.client))


    def reset(self, env=None, client=None, path=None, req_id=None):
        self.env = env
        self.client = client
        self.path = path or ""
        if req_id is not None:
            self.req_id = req_id
        else:
            self.req_id = 0 if env is None else randint(0x00000000, 0xFFFFFFFF)


    def error(self, **kwargs):
        return Error(self.path, self.req_id, **kwargs)



class ObjectBase(Object):

    def __init__(self, req, log):
        Object.__init__(self)
        self.req = req
        self.log = log


    def __str__(self):
        return "%s(req=%s)" % (type(self).__name__, type(self.req).__name__)



class PlainAuthenticator(ObjectBase):

    def __init__(self, req, log, db):
        ObjectBase.__init__(self, req, log)
        self.db = db


    def __str__(self):
        return "%s(req=%s, db=%s)" % (type(self).__name__, type(self.req).__name__, type(self.db).__name__)


    def authenticate(self, env, args, hostnames = None, check_secret = True):
        name = args.get("client", [None])[0]
        secret = args.get("secret", [None])[0] if check_secret else None

        client = self.db.get_client_by_name(hostnames, name, secret)

        if not client:
            self.log.info("authenticate: client not found by name: \"%s\", secret: %s, hostnames: %s" % (
                name, secret, str(hostnames)))
            return None

        # Clients with 'secret' set must get authenticated by it.
        # No secret turns secret auth off for this particular client.
        if client.secret is not None and secret is None and check_secret:
            self.log.info("authenticate: missing secret argument")
            return None

        self.log.info("authenticate: %s" % str(client))

        # These args are not for handler
        args.pop("client", None)
        args.pop("secret", None)

        return client


    def authorize(self, env, client, path, method):
        if method.debug:
            if not client.debug:
                self.log.info("authorize: failed, client does not have debug enabled")
                return None
            return client

        if method.read:
            if not client.read:
                self.log.info("authorize: failed, client does not have read enabled")
                return None
            return client

        if method.write:
            if not (client.write or client.test):
                self.log.info("authorize: failed, client is not allowed to write or test")
                return None

        return client


class X509Authenticator(PlainAuthenticator):

    def get_cert_dns_names(self, pem):

        cert = M2Crypto.X509.load_cert_string(pem)

        subj = cert.get_subject()
        commons = [n.get_data().as_text() for n in subj.get_entries_by_nid(subj.nid["CN"])]

        try:
           extstrs = cert.get_ext("subjectAltName").get_value().split(",")
        except LookupError:
           extstrs = []
        extstrs = [val.strip() for val in extstrs]
        altnames = [val[4:] for val in extstrs if val.startswith("DNS:")]

        # bit of mangling to get rid of duplicates and leave commonname first
        firstcommon = commons[0]
        return [firstcommon] + list(set(altnames+commons) - set([firstcommon]))


    def authenticate(self, env, args):
        try:
            cert_names = self.get_cert_dns_names(env["SSL_CLIENT_CERT"])
        except:
            exception = self.req.error(message="authenticate: cannot get or parse certificate from env", error=403, exc=sys.exc_info(), env=env)
            exception.log(self.log)
            return None

        return PlainAuthenticator.authenticate(self, env, args, hostnames = cert_names)
        

class X509NameAuthenticator(PlainAuthenticator):

    def authenticate(self, env, args):
        try:
            cert_name = env["SSL_CLIENT_S_DN_CN"]
        except:
            exception = self.req.error(message="authenticate: cannot get or parse certificate from env", error=403, exc=sys.exc_info(), env=env)
            exception.log(self.log)
            return None

        if cert_name != args.setdefault("client", [cert_name])[0]:
            exception = self.req.error(message="authenticate: client name does not correspond with certificate", error=403, cn = cert_name, args = args)
            exception.log(self.log)
            return None

        return PlainAuthenticator.authenticate(self, env, args, check_secret = False)


class X509MixMatchAuthenticator(PlainAuthenticator):

    def __init__(self, req, log, db):
        PlainAuthenticator.__init__(self, req, log, db)
        self.hostname_auth = X509Authenticator(req, log, db)
        self.name_auth = X509NameAuthenticator(req, log, db)


    def authenticate(self, env, args):
        try:
            cert_name = env["SSL_CLIENT_S_DN_CN"]
        except:
            exception = self.req.error(message="authenticate: cannot get or parse certificate from env", error=403, exc=sys.exc_info(), env=env)
            exception.log(self.log)
            return None
        name = args.get("client", [None])[0]
        secret =  args.get("secret", [None])[0]

        # Client names are in reverse notation than DNS, client name should
        # thus never be the same as machine hostname (if it is, client
        # admin does something very amiss).

        # So, if client sends the same name in query as in the certificate,
        # or sends no name or secret (which is necessary for hostname auth),
        # use X509NameAuthenticator. Otherwise (names are different and there
        # is name and/or secret in query) use (hostname) X509Authenticator.

        if name == cert_name or (name is None and secret is None):
            auth = self.name_auth
        else:
            auth = self.hostname_auth

        self.log.info("MixMatch is choosing %s (name: %s, cert_name: %s)" % (type(auth).__name__, name, cert_name))
        return auth.authenticate(env, args)


class NoValidator(ObjectBase):

    def __init__(self, req, log):
        ObjectBase.__init__(self, req, log)


    def __str__(self):
        return "%s(req=%s)" % (type(self).__name__, type(self.req).__name__)


    def check(self, event):
        return []


class JSONSchemaValidator(NoValidator):

    def __init__(self, req, log, filename=None):
        NoValidator.__init__(self, req, log)
        self.path = filename or path.join(path.dirname(__file__), "idea.schema")
        with open(self.path) as f:
            self.schema = json.load(f)
        self.validator = Draft4Validator(self.schema)


    def __str__(self):
        return "%s(req=%s, filename=\"%s\")" % (type(self).__name__, type(self.req).__name__, self.path)


    def check(self, event):

        def sortkey(k):
            """ Treat keys as lowercase, prefer keys with less path segments """
            return (len(k.path), "/".join(str(k.path)).lower())

        res = []
        for error in sorted(self.validator.iter_errors(event), key=sortkey):
            res.append({
                "error": 460,
                "message": "Validation error: key \"%s\", value \"%s\"" % (
                    "/".join(str(v) for v in error.path),
                    error.instance
                ),
                "expected": error.schema.get('description', 'no additional info')
            })

        return res



class MySQL(ObjectBase):

    def __init__(self, req, log, host, user, password, dbname, port, retry_count,
            retry_pause, event_size_limit, catmap_filename, tagmap_filename):
        ObjectBase.__init__(self, req, log)
        self.host = host
        self.user = user
        self.password = password
        self.dbname = dbname
        self.port = port
        self.retry_count = retry_count
        self.retry_pause = retry_pause
        self.retry_attempt = 0
        self.event_size_limit = event_size_limit
        self.catmap_filename = catmap_filename
        self.tagmap_filename = tagmap_filename

        with open(catmap_filename, "r") as catmap_fd:
            self.catmap = json.load(catmap_fd)
            self.catmap_other = self.catmap["Other"]    # Catch error soon, avoid lookup later

        with open(tagmap_filename, "r") as tagmap_fd:
            self.tagmap = json.load(tagmap_fd)
            self.tagmap_other = self.catmap["Other"]    # Catch error soon, avoid lookup later

        self.con = None


    def __str__(self):
        return "%s(req=%s, host='%s', user='%s', dbname='%s', port=%d, retry_count=%d, retry_pause=%d, catmap_filename=\"%s\", tagmap_filename=\"%s\")" % (
            type(self).__name__, type(self.req).__name__, self.host, self.user, self.dbname, self.port, self.retry_count, self.retry_pause, self.catmap_filename, self.tagmap_filename)


    def connect(self):
        self.con = my.connect(host=self.host, user=self.user, passwd=self.password,
            db=self.dbname, port=self.port, cursorclass=mycursors.DictCursor)


    def close(self):
        try:
            if self.con:
                self.con.close()
        except Exception:
            pass
        self.con = None


    __del__ = close
    
    
    def repeat(self):
        """ Allows for graceful repeating of transactions self.retry_count
            times. Unsuccessful attempts wait for self.retry_pause until
            next attempt.

            Meant for usage with context manager:

            for attempt in self.repeat():
                with attempt as db:
                    crs = db.query(...)
                    # do something with crs

            Note that it's not reentrant (as is not underlying MySQL
            connection), so avoid nesting on the same MySQL object.
        """
        self.retry_attempt = self.retry_count
        while self.retry_attempt:
            if self.retry_attempt != self.retry_count:
                sleep(self.retry_pause)
            self.retry_attempt -= 1
            yield self


    def __enter__(self):
        """ Context manager protocol. Guarantees that transaction will
            get either commited or rolled back in case of database
            exception. Can be used with self.repeat(), or alone as:

            with self as db:
                crs = db.query(...)
                # do something with crs

            Note that it's not reentrant (as is not underlying MySQL
            connection), so avoid nesting on the same MySQL object.
        """
        if not self.retry_attempt:
            self.retry_attempt = 0
        return self


    def __exit__(self, exc_type, exc_val, exc_tb):
        """ Context manager protocol. If db exception is fired and
            self.retry_attempt is not zero, it is only logged and
            does not propagate, otherwise it propagates up. Also
            open transaction is rolled back.
            In case of no exception, transaction gets commited.
        """
        if not exc_type:
            self.con.commit()
            self.retry_attempt = 0
        else:
            try:
                if self.con:
                    self.con.rollback()
            except my.Error:
                pass
            try:
                self.close()
            except my.Error:
                pass
            if self.retry_attempt:
                self.log.info("Database error (%d attempts left): %s %s" % (self.retry_attempt, exc_type.__name__, exc_val))
                return True


    def query(self, *args, **kwargs):
        if not self.con:
            self.connect()
        crs = self.con.cursor()
        self.log.debug("execute: %s %s" % (args, kwargs))
        crs.execute(*args, **kwargs)
        return crs


    def _get_comma_perc(self, l):
        return ','.join(['%s'] * len(l))


    def _get_not(self, b):
        return "" if b else "NOT"


    def get_client_by_name(self, cert_names=None, name=None, secret=None):
        query = ["SELECT * FROM clients WHERE valid = 1"]
        params = []
        if name:
            query.append(" AND name = %s")
            params.append(name.lower())
        if secret:
            query.append(" AND secret = %s")
            params.append(secret)
        if cert_names:
            query.append(" AND hostname IN (%s)" % self._get_comma_perc(cert_names))
            params.extend(n.lower() for n in cert_names)

        for attempt in self.repeat():
            with attempt as db:
                rows = db.query("".join(query), params).fetchall()

                if len(rows)>1:
                    self.log.warn("get_client_by_name: query returned more than one result (cert_names = %s, name = %s, secret = %s): %s" % (cert_names, name, secret, ", ".join([str(Client(**row)) for row in rows])))
                    return None

                return Client(**rows[0]) if rows else None



    def get_clients(self, id=None):
        query = ["SELECT * FROM clients"]
        params = []
        if id:
            query.append("WHERE id = %s")
            params.append(id)
        query.append("ORDER BY id")
        for attempt in self.repeat():
            with attempt as db:
                rows = db.query(" ".join(query), params).fetchall()
                return [Client(**row) for row in rows]


    def add_modify_client(self, id=None, **kwargs):
        query = []
        params = []
        uquery = []
        if id is None:
            query.append("INSERT INTO clients SET")
            uquery.append("registered = now()")
        else:
            query.append("UPDATE clients SET")
        for attr in set(Client._fields) - set(["id", "registered"]):
            val = kwargs.get(attr, None)
            if val is not None:
                if attr == "secret" and val == "":  # disable secret
                    val = None
                uquery.append("`%s` = %%s" % attr)
                params.append(val)
        if not uquery:
            return id
        query.append(", ".join(uquery))
        if id is not None:
            query.append("WHERE id = %s")
            params.append(id)
        for attempt in self.repeat():
            with attempt as db:
                crs = db.query(" ".join(query), params)
                newid = crs.lastrowid if id is None else id
                return newid


    def get_debug(self):
        for attempt in self.repeat():
            with attempt as db:
                rows = db.query("SELECT VERSION() AS VER").fetchall()
                tablestat = db.query("SHOW TABLE STATUS").fetchall()
                return {
                    "db": "MySQL",
                    "version": rows[0]["VER"],
                    "tables": tablestat
                }


    def getMaps(self, section, variables):
        maps = []
        for v in variables:
            try:
                mapped = section[v]
            except KeyError:
                raise self.req.error(message="Wrong tag or category used in query.", error=422,
                    exc=sys.exc_info(), key=v)
            maps.append(mapped)
        return set(maps)    # unique


    def fetch_events(self, client, id, count,
            cat=None, nocat=None,
            tag=None, notag=None,
            group=None, nogroup=None):
       
        if cat and nocat:
            raise self.req.error(message="Unrealizable conditions. Choose cat or nocat option.", error=422,
                        cat=cat, nocat=nocat)
        if tag and notag:
            raise self.req.error(message="Unrealizable conditions. Choose tag or notag option.", error=422,
                        tag=tag, notag=notag)
        if group and nogroup:
            raise self.req.error(message="Unrealizable conditions. Choose group or nogroup option.", error=422,
                        group=group, nogroup=nogroup)

        query = ["SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s"]
        params = [id or 0]

        if cat or nocat:
            cats = self.getMaps(self.catmap, (cat or nocat))
            query.append(
                " AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE category_id IN (%s))" % (
                    self._get_not(cat), self._get_comma_perc(cats)))
            params.extend(cats)

        if tag or notag:
            tags = self.getMaps(self.tagmap, (tag or notag))
            query.append(
                " AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE tag_id IN (%s))" % (
                    self._get_not(tag), self._get_comma_perc(tags)))
            params.extend(tags)

        if group or nogroup:
            subquery = []
            for name in (group or nogroup):
                subquery.append("c.name = %s")      # exact client
                params.append(name)
                subquery.append("c.name LIKE %s")   # whole subtree
                params.append(name + ".%")

            query.append(" AND %s (%s)" % (self._get_not(group), " OR ".join(subquery)))

        query.append(" AND e.valid = 1 LIMIT %s")
        params.append(count)

        query_string = "".join(query)

        row = None
        for attempt in self.repeat():
            with attempt as db:
                row = db.query(query_string, params).fetchall()

        if row:
            maxid = max(r['id'] for r in row)
        else:
            maxid = self.getLastEventId()

        events = []
        for r in row:
            try:
                e = json.loads(r["data"])
            except Exception:
                # Note that we use Error object just for proper formatting,
                # but do not raise it; from client perspective invalid
                # events get skipped silently.
                err = self.req.error(message="Unable to deserialize JSON event from db, id=%s" % r["id"], error=500,
                    exc=sys.exc_info(), id=r["id"])
                err.log(self.log, prio=logging.WARNING)
            events.append(e)

        return {
            "lastid": maxid,
            "events": events
        }


    def store_events(self, client, events, events_raw):
        try:
            for attempt in self.repeat():
                with attempt as db:
                    for event, raw_event in zip(events, events_raw):
                        lastid = db.query("INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)",
                            (client.id, raw_event)).lastrowid

                        catlist = event.get('Category', ["Other"])
                        cats = set(catlist) | set(cat.split(".", 1)[0] for cat in catlist)
                        for cat in cats:
                            cat_id = self.catmap.get(cat, self.catmap_other)
                            db.query("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))

                        nodes = event.get('Node', [])
                        tags = []
                        for node in nodes:
                            tags.extend(node.get('Type', []))
                        for tag in set(tags):
                            tag_id = self.tagmap.get(tag, self.tagmap_other)
                            db.query("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))
                    return []
        except Exception as e:
            exception = self.req.error(message="DB error", error=500, exc=sys.exc_info(), env=env)
            exception.log(self.log)
            return [{"error": 500, "message": "DB error %s" % type(e).__name__}]


    def insertLastReceivedId(self, client, id):
        self.log.debug("insertLastReceivedId: id %i for client %i(%s)" % (id, client.id, client.hostname))
        for attempt in self.repeat():
            with attempt as db:
                db.query("INSERT INTO last_events(client_id, event_id, timestamp) VALUES(%s, %s, NOW())", (client.id, id))


    def getLastEventId(self):
        for attempt in self.repeat():
            with attempt as db:
                row = db.query("SELECT MAX(id) as id FROM events").fetchall()[0]
                return row['id'] or 1


    def getLastReceivedId(self, client):
        for attempt in self.repeat():
            with attempt as db:
                res = db.query("SELECT event_id as id FROM last_events WHERE client_id = %s ORDER BY last_events.id DESC LIMIT 1", (client.id,)).fetchall()
                try:
                    row = res[0]
                except IndexError:
                    id = None
                    self.log.debug("getLastReceivedId: probably first access, unable to get id for client %i(%s)" % (client.id, client.hostname))
                else:
                    id = row["id"]
                    self.log.debug("getLastReceivedId: id %i for client %i(%s)" % (id, client.id, client.hostname))

                return id


    def load_maps(self):
        with self as db:
            db.query("DELETE FROM tags")
            for tag, num in self.tagmap.iteritems():
                db.query("INSERT INTO tags(id, tag) VALUES (%s, %s)", (num, tag))
            db.query("DELETE FROM categories")
            for cat_subcat, num in self.catmap.iteritems():
                catsplit = cat_subcat.split(".", 1)
                category = catsplit[0]
                subcategory = catsplit[1] if len(catsplit)>1 else None
                db.query("INSERT INTO categories(id, category, subcategory, cat_subcat) VALUES (%s, %s, %s, %s)",
                    (num, category, subcategory, cat_subcat))


    def purge_lastlog(self, days):
        with self as db:
            return db.query(
                "DELETE FROM last_events "
                " USING last_events LEFT JOIN ("
                "    SELECT MAX(id) AS last FROM last_events"
                "    GROUP BY client_id"
                " ) AS maxids ON last=id"
                " WHERE timestamp < DATE_SUB(CURDATE(), INTERVAL %s DAY) AND last IS NULL",
                (days,)).rowcount


    def purge_events(self, days):
        with self as db:
            affected = 0
            id_ = db.query(
                "SELECT MAX(id) as id"
                "  FROM events"
                "  WHERE received < DATE_SUB(CURDATE(), INTERVAL %s DAY)",
                (days,)
            ).fetchall()[0]["id"]
            if id_ is None:
                return 0
            affected = db.query("DELETE FROM events WHERE id <= %s", (id_,)).rowcount
            db.query("DELETE FROM event_category_mapping WHERE event_id <= %s", (id_,))
            db.query("DELETE FROM event_tag_mapping WHERE event_id <= %s", (id_,))
            return affected



def expose(read=1, write=0, debug=0):

    def expose_deco(meth):
        meth.exposed = True
        meth.read = read
        meth.write = write
        meth.debug = debug
        if not hasattr(meth, "arguments"):
            meth.arguments = meth.func_code.co_varnames[:meth.func_code.co_argcount]
        return meth

    return expose_deco


class Server(ObjectBase):

    def __init__(self, req, log, auth, handler):
        ObjectBase.__init__(self, req, log)
        self.auth = auth
        self.handler = handler


    def __str__(self):
        return "%s(req=%s, auth=%s, handler=%s)" % (type(self).__name__, type(self.req).__name__, type(self.auth).__name__, type(self.handler).__name__)


    def sanitize_args(self, path, func, args, exclude=["self", "post"]):
        # silently remove internal args, these should never be used
        # but if somebody does, we do not expose them by error message
        intargs = set(args).intersection(exclude)
        for a in intargs:
            del args[a]
        if intargs:
            self.log.info("sanitize_args: Called with internal args: %s" % ", ".join(intargs))

        # silently remove surplus arguments - potential forward
        # compatibility (unknown args will get ignored)
        badargs = set(args) - set(func.arguments)
        for a in badargs:
            del args[a]
        if badargs:
            self.log.info("sanitize_args: Called with superfluous args: %s" % ", ".join(badargs))

        return args


    def wsgi_app(self, environ, start_response, exc_info=None):
        path = environ.get("PATH_INFO", "").lstrip("/")
        self.req.reset(env=environ, path=path)
        output = ""
        status = "200 OK"
        headers = [('Content-type', 'application/json')]
        exception = None

        try:
            try:
                method = getattr(self.handler, path)
                method.exposed    # dummy access to trigger AttributeError
            except Exception:
                raise self.req.error(message="You've fallen off the cliff.", error=404)

            self.req.args = args = parse_qs(environ.get('QUERY_STRING', ""))

            self.req.client = client = self.auth.authenticate(environ, args)
            if not client:
                raise self.req.error(message="I'm watching. Authenticate.", error=403)

            auth = self.auth.authorize(self.req.env, self.req.client, self.req.path, method)
            if not auth:
                raise self.req.error(message="I'm watching. Not authorized.", error=403, client=client.name)

            args = self.sanitize_args(path, method, args)

            try:
                post_data = environ['wsgi.input'].read()
            except:
                raise self.req.error(message="Data read error.", error=408, exc=sys.exc_info())

            headers, output = method(post_data, **args)

        except Error as e:
            exception = e
        except Exception as e:
            exception = self.req.error(message="Server exception", error=500, exc=sys.exc_info())

        if exception:
            status = "%d %s" % exception.get_http_err_msg()
            output = json.dumps(exception.to_dict(), default=lambda v: str(v))
            exception.log(self.log)

        # Make sure everything is properly encoded - JSON and various function
        # may spit out unicode instead of str and it gets propagated up (str
        # + unicode = unicode). However, the right thing would be to be unicode
        # correct among whole source and always decode on input (json module
        # does that for us) and on output here.
        if isinstance(status, unicode):
            status = status.encode("utf-8")
        if isinstance(output, unicode):
            output = output.encode("utf-8")
        headers.append(('Content-Length', str(len(output))))
        start_response(status, headers)
        self.req.reset()
        return [output]


    __call__ = wsgi_app



def json_wrapper(method):

    def meth_deco(self, post, **args):
        if "events" in method.func_code.co_varnames[0:method.func_code.co_argcount]:
            try:
                events = json.loads(post) if post else None
            except Exception as e:
                raise self.req.error(message="Deserialization error.", error=400,
                    exc=sys.exc_info(), args=post, parser=str(e))
            if events:
                args["events"] = events

        result = method(self, **args)   # call requested method

        try:
            # 'default': takes care of non JSON serializable objects,
            # which could (although shouldn't) appear in handler code