warden_server.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2011-2015 Cesnet z.s.p.o
# Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.

import sys
import os
import logging
import logging.handlers
import ConfigParser
from traceback import format_tb
import M2Crypto.X509
import json
import MySQLdb as my
import MySQLdb.cursors as mycursors
import re
import email.utils
from collections import namedtuple
from time import sleep
from urlparse import parse_qs
from os import path
from random import randint

# for local version of up to date jsonschema
sys.path.append(path.join(path.dirname(__file__), "..", "lib"))

from jsonschema import Draft4Validator


VERSION = "3.0-beta2"

class Error(Exception):

    def __init__(self, method=None, req_id=None, errors=None, **kwargs):
        self.method = method
        self.req_id = req_id
        self.errors = [kwargs] if kwargs else []
        if errors:
            self.errors.extend(errors)


    def append(self, _events=None, **kwargs):
        self.errors.append(kwargs)


    def get_http_err_msg(self):
        try:
            err = self.errors[0]["error"]
            msg = self.errors[0]["message"].replace("\n", " ")
        except (IndexError, KeyError):
            err = 500
            msg = "There's NO self-destruction button! Ah, you've just found it..."
        for e in self.errors:
            next_err = e.get("error", 500)
            if err != next_err:
                # errors not same, round to basic err code (400, 500)
                # and use the highest one
                err = max(err//100, next_err//100)*100
            next_msg = e.get("message", "Unknown error").replace("\n", " ")
            if msg != next_msg:
                msg = "Multiple errors"
        return err, msg


    def __str__(self):
        return "\n".join(self.str_err(e) for e in self.errors)


    def log(self, logger, prio=logging.ERROR):
        for e in self.errors:
            logger.log(prio, self.str_err(e))
            info = self.str_info(e)
            if info:
                logger.info(info)
            debug = self.str_debug(e)
            if debug:
                logger.debug(debug)


    def str_err(self, e):
        out = []
        out.append("Error(%s) %s " % (e.get("error", 0), e.get("message", "Unknown error")))
        if "exc" in e and e["exc"]:
            out.append("(cause was %s: %s)" % (e["exc"][0].__name__, str(e["exc"][1])))
        return "".join(out)


    def str_info(self, e):
        ecopy = dict(e)    # shallow copy
        ecopy.pop("req_id", None)
        ecopy.pop("method", None)
        ecopy.pop("error", None)
        ecopy.pop("message", None)
        ecopy.pop("exc", None)
        if ecopy:
            out = "Detail: %s" % (json.dumps(ecopy, default=lambda v: str(v)))
        else:
            out = ""
        return out


    def str_debug(self, e):
        out = []
        if not "exc" in e or not e["exc"]:
            return ""
        exc_tb = e["exc"][2]
        if exc_tb:
            out.append("Traceback:\n")
            out.extend(format_tb(exc_tb))
        return "".join(out)


    def to_dict(self):
        errlist = []
        for e in self.errors:
            ecopy = dict(e)
            ecopy.pop("exc", None)
            errlist.append(ecopy)
        d = {
            "method": self.method,
            "req_id": self.req_id,
            "errors": errlist
        }
        return d



def get_clean_root_logger(level=logging.INFO):
    """ Attempts to get logging module into clean slate state """

    # We want to be able to set up at least stderr logger before any
    # configuration is read, and then later get rid of it and set up
    # whatever administrator requires.
    # However, there can exist only one logger, but we want to get a clean
    # slate everytime we initialize StreamLogger or FileLogger... which
    # is not exactly supported by logging module.
    # So, we look directly inside logger class and clean up handlers/filters
    # manually.
    logger = logging.getLogger(__name__)
    logger.setLevel(level)
    while logger.handlers:
        logger.removeHandler(logger.handlers[0])
    while logger.filters:
        logger.removeFilter(logger.filters[0])
    logger.propagate = False
    return logger



def StreamLogger(stream=sys.stderr, level=logging.INFO):
    """ Fallback handler just for setup, not meant to be used from
        configuration file because during wsgi query stdout/stderr
        is forbidden.
    """

    fhand = logging.StreamHandler(stream)
    fform = logging.Formatter('%(asctime)s %(filename)s[%(process)d]: (%(levelname)s) %(message)s')
    fhand.setFormatter(fform)
    logger = get_clean_root_logger(level)
    logger.addHandler(fhand)



class LogRequestFilter(logging.Filter):
    """ Filter class, instance of which is added to logger class to add
        info about request automatically into every logline, no matter
        how it came into existence.
    """

    def __init__(self, req):
        logging.Filter.__init__(self)
        self.req = req


    def filter(self, record):
        if self.req.env:
            record.req_preamble = "%08x/%s: " % (self.req.req_id or 0, self.req.path)
        else:
            record.req_preamble = ""
        return True



def FileLogger(req, filename, level=logging.INFO):

    fhand = logging.FileHandler(filename)
    fform = logging.Formatter('%(asctime)s %(filename)s[%(process)d]: (%(levelname)s) %(req_preamble)s%(message)s')
    fhand.setFormatter(fform)
    ffilt = LogRequestFilter(req)
    logger = get_clean_root_logger(level)
    logger.addFilter(ffilt)
    logger.addHandler(fhand)
    logging.info("Initialized FileLogger(req=%s, filename=\"%s\", level=\"%d\")" % (type(req).__name__, filename, level))



def SysLogger(req, socket="/dev/log", facility=logging.handlers.SysLogHandler.LOG_DAEMON, level=logging.INFO):

    fhand = logging.handlers.SysLogHandler(address=socket, facility=facility)
    fform = logging.Formatter('%(filename)s[%(process)d]: (%(levelname)s) %(req_preamble)s%(message)s')
    fhand.setFormatter(fform)
    ffilt = LogRequestFilter(req)
    logger = get_clean_root_logger(level)
    logger.addFilter(ffilt)
    logger.addHandler(fhand)
    logging.info("Initialized SysLogger(req=%s, socket=\"%s\", facility=\"%d\", level=\"%d\")" % (type(req).__name__, socket, facility, level))



Client = namedtuple("Client",
    ["id", "registered", "requestor", "hostname", "name",
    "secret", "valid", "read", "debug", "write", "test", "note"])


class Object(object):

    def __str__(self):
        return "%s()" % type(self).__name__



class Request(Object):
    """ Simple container for info about ongoing request.
        One instance gets created before server startup, and all other
        configured objects get it as parameter during instantiation.

        Server then takes care of populating this instance on the start
        of wsgi request (and resetting at the end). All other objects
        then can find this actual request info in their own self.req.

        However, only Server.wsgi_app, handler (WardenHandler) exposed
        methods and logging related objects should use self.req directly.
        All other objects should use self.req only as source of data for
        error/exception handling/logging, and should take/return
        necessary data as arguments/return values for clarity on
        which data their main codepaths work with.
    """

    def __init__(self):
        Object.__init__(self)
        self.reset()


    def __str__(self):
        return "%s()" % (type(self).__name__, str(self.env), str(self.client))


    def reset(self, env=None, client=None, path=None, req_id=None):
        self.env = env
        self.client = client
        self.path = path or ""
        if req_id is not None:
            self.req_id = req_id
        else:
            self.req_id = 0 if env is None else randint(0x00000000, 0xFFFFFFFF)


    def error(self, **kwargs):
        return Error(self.path, self.req_id, **kwargs)



class ObjectReq(Object):

    def __init__(self, req):
        Object.__init__(self)
        self.req = req


    def __str__(self):
        return "%s(req=%s)" % (type(self).__name__, type(self.req).__name__)



class PlainAuthenticator(ObjectReq):

    def __init__(self, req, db):
        ObjectReq.__init__(self, req)
        self.db = db


    def __str__(self):
        return "%s(req=%s, db=%s)" % (type(self).__name__, type(self.req).__name__, type(self.db).__name__)


    def authenticate(self, env, args):
        name = args.get("client", [None])[0]
        secret =  args.get("secret", [None])[0]
        hostnames = args.get("hostnames", [None])[0]

        client = self.db.get_client_by_name(hostnames, name, secret)

        if not client:
            logging.getLogger(__name__).info("authenticate: client not found by name: \"%s\", secret: %s, hostnames: %s" % (
                name, secret, str(hostnames)))
            return None

        # Clients with 'secret' set must get authenticated by it.
        # No secret turns secret auth off for this particular client.
        if client.secret is not None and secret is None:
            logging.getLogger(__name__).info("authenticate: missing secret argument")
            return None

        logging.getLogger(__name__).info("authenticate: %s" % str(client))

        # These args are not for handler
        args.pop("client", None)
        args.pop("secret", None)
        args.pop("hostnames", None)

        return client


    def authorize(self, env, client, path, method):
        if method.debug:
            if not client.debug:
                logging.getLogger(__name__).info("authorize: failed, client does not have debug enabled")
                return None
            return client

        if method.read:
            if not client.read:
                logging.getLogger(__name__).info("authorize: failed, client does not have read enabled")
                return None
            return client

        if method.write:
            if not (client.write or client.test):
                logging.getLogger(__name__).info("authorize: failed, client is not allowed to write or test")
                return None

        return client


class X509Authenticator(PlainAuthenticator):

    def get_cert_dns_names(self, pem):

        cert = M2Crypto.X509.load_cert_string(pem)

        subj = cert.get_subject()
        commons = [n.get_data().as_text() for n in subj.get_entries_by_nid(subj.nid["CN"])]

        try:
           extstrs = cert.get_ext("subjectAltName").get_value().split(",")
        except LookupError:
           extstrs = []
        extstrs = [val.strip() for val in extstrs]
        altnames = [val[4:] for val in extstrs if val.startswith("DNS:")]

        # bit of mangling to get rid of duplicates and leave commonname first
        firstcommon = commons[0]
        return [firstcommon] + list(set(altnames+commons) - set([firstcommon]))


    def authenticate(self, env, args):
        try:
            cert_names = self.get_cert_dns_names(env["SSL_CLIENT_CERT"])
        except:
            exception = self.req.error(message="authenticate: cannot get or parse certificate from env", error=403, exc=sys.exc_info(), env=env)
            exception.log(logging.getLogger(__name__))
            return None

        args["hostnames"] = [cert_names]

        return PlainAuthenticator.authenticate(self, env, args)
        

class X509NameAuthenticator(PlainAuthenticator):

    def authenticate(self, env, args):
        try:
            cert_name = env["SSL_CLIENT_S_DN_CN"]
        except:
            exception = self.req.error(message="authenticate: cannot get or parse certificate from env", error=403, exc=sys.exc_info(), env=env)
            exception.log(logging.getLogger(__name__))
            return None

        if cert_name != args.setdefault("client", [cert_name])[0]:
            exception = self.req.error(message="authenticate: client name does not correspond with certificate", error=403, cn = cert_name, args = args)
            exception.log(logging.getLogger(__name__))
            return None

        return PlainAuthenticator.authenticate(self, env, args)


class X509MixMatchAuthenticator(PlainAuthenticator):

    def __init__(self, req, db):
        PlainAuthenticator.__init__(self, req, db)
        self.hostname_auth = X509Authenticator(req, db)
        self.name_auth = X509Authenticator(req, db)


    def authenticate(self, env, args):
        try:
            cert_name = env["SSL_CLIENT_S_DN_CN"]
        except:
            exception = self.req.error(message="authenticate: cannot get or parse certificate from env", error=403, exc=sys.exc_info(), env=env)
            exception.log(logging.getLogger(__name__))
            return None
        name = args.get("client", [None])[0]
        secret =  args.get("secret", [None])[0]

        # Client names are in reverse notation than DNS, client name should
        # thus never be the same as machine hostname (if it is, client
        # admin does something very amiss).

        # So, if client sends the same name in query as in the certificate,
        # or sends no name or secret (which is necessary for hostname auth),
        # use X509NameAuthenticator. Otherwise (names are different and there
        # is name and/or secret in query) use (hostname) X509Authenticator.

        if name == cert_name or (name is None and secret is None):
            auth = self.name_auth
        else:
            auth = self.hostname_auth

        return auth.authenticate(self, env, args)


class NoValidator(ObjectReq):

    def __init__(self, req):
        ObjectReq.__init__(self, req)


    def __str__(self):
        return "%s(req=%s)" % (type(self).__name__, type(self.req).__name__)


    def check(self, event):
        return []


class JSONSchemaValidator(NoValidator):

    def __init__(self, req, filename=None):
        NoValidator.__init__(self, req)
        self.path = filename or path.join(path.dirname(__file__), "idea.schema")
        with open(self.path) as f:
            self.schema = json.load(f)
        self.validator = Draft4Validator(self.schema)


    def __str__(self):
        return "%s(req=%s, filename=\"%s\")" % (type(self).__name__, type(self.req).__name__, self.path)


    def check(self, event):

        def sortkey(k):
            """ Treat keys as lowercase, prefer keys with less path segments """
            return (len(k.path), "/".join(str(k.path)).lower())

        res = []
        for error in sorted(self.validator.iter_errors(event), key=sortkey):
            res.append({
                "error": 460,
                "message": "Validation error: key \"%s\", value \"%s\"" % (
                    "/".join(str(v) for v in error.path),
                    error.instance
                ),
                "expected": error.schema.get('description', 'no additional info')
            })

        return res



class MySQL(ObjectReq):

    def __init__(self, req, host, user, password, dbname, port, retry_count,
            retry_pause, event_size_limit, catmap_filename, tagmap_filename):
        ObjectReq.__init__(self, req)
        self.host = host
        self.user = user
        self.password = password
        self.dbname = dbname
        self.port = port
        self.retry_count = retry_count
        self.retry_pause = retry_pause
        self.event_size_limit = event_size_limit
        self.catmap_filename = catmap_filename
        self.tagmap_filename = tagmap_filename

        with open(catmap_filename, "r") as catmap_fd:
            self.catmap = json.load(catmap_fd)
            self.catmap_other = self.catmap["Other"]    # Catch error soon, avoid lookup later

        with open(tagmap_filename, "r") as tagmap_fd:
            self.tagmap = json.load(tagmap_fd)
            self.tagmap_other = self.catmap["Other"]    # Catch error soon, avoid lookup later

        self.con = None

        self.connect()


    def __str__(self):
        return "%s(req=%s, host='%s', user='%s', dbname='%s', port=%d, retry_count=%d, retry_pause=%d, catmap_filename=\"%s\", tagmap_filename=\"%s\")" % (
            type(self).__name__, type(self.req).__name__, self.host, self.user, self.dbname, self.port, self.retry_count, self.retry_pause, self.catmap_filename, self.tagmap_filename)


    def connect(self):
        self.con = my.connect(host=self.host, user=self.user, passwd=self.password,
            db=self.dbname, port=self.port, cursorclass=mycursors.DictCursor)


    def close(self):
        try:
            if self.con:
                self.con.close()
        except Exception:
            pass


    __del__ = close
    
    
    def query(self, *args, **kwargs):
        """ Execute query on self.con, reconnecting if necessary """
        countdown = self.retry_count
        commit = kwargs.pop("commit", False)
        crs = kwargs.pop("crs", None)
        while True:
            try:
                if crs is None:
                    crs = self.con.cursor()
                logging.getLogger(__name__).debug("execute: %s %s" % (args, kwargs))
                crs.execute(*args, **kwargs)
                if commit:
                    self.con.commit()
                return crs
            except my.OperationalError:
                if not countdown:
                    raise
                logging.getLogger(__name__).info("execute: Database down, trying to reconnect (%d attempts left)..." % countdown)
                if countdown<self.retry_count:
                    sleep(self.retry_pause)    # no need to melt down server on longer outage
                self.close()
                self.connect()
                crs = None
                countdown -= 1


    def _get_comma_perc(self, l):
        return ','.join(['%s'] * len(l))


    def _get_not(self, b):
        return "" if b else "NOT"


    def get_client_by_name(self, cert_names=None, name=None, secret=None):
        query = ["SELECT * FROM clients WHERE valid = 1"]
        params = []
        if name:
            query.append(" AND name = %s")
            params.append(name.lower())
        if secret:
            query.append(" AND secret = %s")
            params.append(secret)
        if cert_names:
            query.append(" AND hostname IN (%s)" % self._get_comma_perc(cert_names))
            params.extend(n.lower() for n in cert_names)
        rows = self.query("".join(query), params, commit=True).fetchall()

        if len(rows)>1:
            logging.getLogger(__name__).warn("get_client_by_name: query returned more than one result (cert_names = %s, name = %s, secret = %s): %s" % (cert_names, name, secret, ", ".join([str(Client(**row)) for row in rows])))
            return None

        return Client(**rows[0]) if rows else None


    def get_clients(self, id=None):
        query = ["SELECT * FROM clients"]
        params = []
        if id:
            query.append("WHERE id = %s")
            params.append(id)
        query.append("ORDER BY id")
        rows = self.query(" ".join(query), params, commit=True).fetchall()
        return [Client(**row) for row in rows]


    def add_modify_client(self, id=None, **kwargs):
        query = []
        params = []
        uquery = []
        if id is None:
            query.append("INSERT INTO clients SET")
            uquery.append("registered = now()")
        else:
            query.append("UPDATE clients SET")
        for attr in set(Client._fields) - set(["id", "registered"]):
            val = kwargs.get(attr, None)
            if val is not None:
                uquery.append("`%s` = %%s" % attr)
                params.append(val)
        if not uquery:
            return id
        query.append(", ".join(uquery))
        if id is not None:
            query.append("WHERE id = %s")
            params.append(id)
        crs = self.query(" ".join(query), params, commit=True)
        newid = crs.lastrowid if id is None else id
        return newid


    def get_debug(self):
        rows = self.query("SELECT VERSION() AS VER", commit=True).fetchall()
        tablestat = self.query("SHOW TABLE STATUS", commit=True).fetchall()
        return {
            "db": "MySQL",
            "version": rows[0]["VER"],
            "tables": tablestat
        }


    def getMaps(self, section, variables):
        maps = []
        for v in variables:
            try:
                mapped = section[v]
            except KeyError:
                raise self.req.error(message="Wrong tag or category used in query.", error=422,
                    exc=sys.exc_info(), key=v)
            maps.append(mapped)
        return set(maps)    # unique


    def fetch_events(self, client, id, count,
            cat=None, nocat=None,
            tag=None, notag=None,
            group=None, nogroup=None):
       
        logging.debug("fetch_events: id=%i, count=%i, cat=%s, nocat=%s, tag=%s, notag=%s, group=%s, nogroup=%s" % (id, count, str(cat), str(nocat), str(tag), str(notag), str(group), str(nogroup)))

        if cat and nocat:
            raise self.req.error(message="Unrealizable conditions. Choose cat or nocat option.", error=422,
                        cat=cat, nocat=nocat)
        if tag and notag:
            raise self.req.error(message="Unrealizable conditions. Choose tag or notag option.", error=422,
                        tag=tag, notag=notag)
        if group and nogroup:
            raise self.req.error(message="Unrealizable conditions. Choose group or nogroup option.", error=422,
                        group=group, nogroup=nogroup)

        query = ["SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s"]
        params = [id or 0]

        if cat or nocat:
            cats = self.getMaps(self.catmap, (cat or nocat))
            query.append(
                " AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE category_id IN (%s))" % (
                    self._get_not(cat), self._get_comma_perc(cats)))
            params.extend(cats)

        if tag or notag:
            tags = self.getMaps(self.tagmap, (tag or notag))
            query.append(
                " AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE tag_id IN (%s))" % (
                    self._get_not(tag), self._get_comma_perc(tags)))
            params.extend(tags)

        if group or nogroup:
            subquery = []
            for name in (group or nogroup):
                subquery.append("c.name = %s")      # exact client
                params.append(name)
                subquery.append("c.name LIKE %s")   # whole subtree
                params.append(name + ".%")

            query.append(" AND %s (%s)" % (self._get_not(group), " OR ".join(subquery)))

        query.append(" AND e.valid = 1 LIMIT %s")
        params.append(count)

        query_string = "".join(query)

        row = self.query(query_string, params, commit=True).fetchall()

        if row:
            maxid = max(r['id'] for r in row)
        else:
            maxid = self.getLastEventId()

        events = []
        for r in row:
            try:
                e = json.loads(r["data"])
            except Exception:
                # Note that we use Error object just for proper formatting,
                # but do not raise it; from client perspective invalid
                # events get skipped silently.
                err = self.req.error(message="Unable to deserialize JSON event from db, id=%s" % r["id"], error=500,
                    exc=sys.exc_info(), id=r["id"])
                err.log(logging.getLogger(__name__), prio=logging.WARNING)
            events.append(e)

        return {
            "lastid": maxid,
            "events": events
        }


    def store_events(self, client, events, events_raw):
        crs = self.con.cursor()
        try:
            for event, raw_event in zip(events, events_raw):
                self.query("INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)",
                    (client.id, raw_event), crs=crs)
                lastid = crs.lastrowid

                catlist = event.get('Category', ["Other"])
                cats = set(catlist) | set(cat.split(".", 1)[0] for cat in catlist)
                for cat in cats:
                    cat_id = self.catmap.get(cat, self.catmap_other)
                    self.query("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id), crs=crs)

                nodes = event.get('Node', [])
                tags = []
                for node in nodes:
                    tags.extend(node.get('Type', []))
                for tag in set(tags):
                    tag_id = self.tagmap.get(tag, self.tagmap_other)
                    self.query("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id), crs=crs)
            self.con.commit()
            return []
        except Exception as e:
            self.con.rollback()
            exception = self.req.error(message="DB error", error=500, exc=sys.exc_info(), env=env)
            exception.log(logging.getLogger(__name__))
            return [{"error": 500, "message": "DB error %s" % type(e).__name__}]


    def insertLastReceivedId(self, client, id):
        logging.getLogger(__name__).debug("insertLastReceivedId: id %i for client %i(%s)" % (id, client.id, client.hostname))
        try:
            self.query("INSERT INTO last_events(client_id, event_id, timestamp) VALUES(%s, %s, NOW())", (client.id, id))
            self.con.commit()
        except Exception as e:
            self.con.rollback()
            raise


    def getLastEventId(self):
        row = self.query("SELECT MAX(id) as id FROM events", commit=True).fetchall()[0]

        return row['id'] or 0


    def getLastReceivedId(self, client):
        res = self.query("SELECT event_id as id FROM last_events WHERE client_id = %s ORDER BY last_events.id DESC LIMIT 1", client.id, commit=True).fetchall()
        try:
            row = res[0]
        except IndexError:
            id = None
            logging.getLogger(__name__).debug("getLastReceivedId: probably first access, unable to get id for client %i(%s)" % (client.id, client.hostname))
        else:
            id = row["id"]
            logging.getLogger(__name__).debug("getLastReceivedId: id %i for client %i(%s)" % (id, client.id, client.hostname))

        return id


    def load_maps(self):
        crs = self.con.crs()
        try:
            self.query("DELETE FROM tags", crs=crs)
            for tag, num in self.tagmap.iteritems():
                self.query("INSERT INTO tags(id, tag) VALUES (%s, %s)", (num, tag), crs=crs)
            self.query("DELETE FROM categories", crs=crs)
            for cat_subcat, num in self.catmap.iteritems():
                catsplit = cat_subcat.split(".", 1)
                category = catsplit[0]
                subcategory = catsplit[1] if len(catsplit)>1 else None
                self.query("INSERT INTO categories(id, category, subcategory, cat_subcat) VALUES (%s, %s, %s, %s)",
                    (num, category, subcategory, cat_subcat), crs=crs)
            self.con.commit()
        except Exception as e:
            self.con.rollback()
            raise


    def purge_lastlog(self, days):
        try:
            self.query(
                "DELETE FROM last_events "
                " USING last_events LEFT JOIN ("
                "    SELECT MAX(id) AS last FROM last_events"
                "    GROUP BY client_id"
                " ) AS maxids ON last=id"
                " WHERE timestamp < DATE_SUB(CURDATE(), INTERVAL %s DAY) AND last IS NULL",
                days)
            affected = self.con.affected_rows()
            self.con.commit()
        except Exception as e:
            self.con.rollback()
            raise
        return affected


    def purge_events(self, days):
        try:
            self.query(
                "DELETE FROM events WHERE received < DATE_SUB(CURDATE(), INTERVAL %s DAY)",
                days)
            affected = self.con.affected_rows()
            self.con.commit()
        except Exception as e:
            self.con.rollback()
            raise
        return affected



def expose(read=1, write=0, debug=0):

    def expose_deco(meth):
        meth.exposed = True
        meth.read = read
        meth.write = write
        meth.debug = debug
        return meth

    return expose_deco


class Server(ObjectReq):

    def __init__(self, req, auth, handler):
        ObjectReq.__init__(self, req)
        self.auth = auth
        self.handler = handler


    def __str__(self):
        return "%s(req=%s, auth=%s, handler=%s)" % (type(self).__name__, type(self.req).__name__, type(self.auth).__name__, type(self.handler).__name__)


    def sanitize_args(self, path, func, args, exclude=["self"]):
        # silently remove internal args, these should never be used
        # but if somebody does, we do not expose them by error message
        intargs = set(args).intersection(exclude)
        for a in intargs:
            del args[a]
        if intargs:
            logging.getLogger(__name__).info("sanitize_args: Called with internal args: %s" % ", ".join(intargs))

        # silently remove surplus arguments - potential forward
        # compatibility (unknown args will get ignored)
        badargs = set(args) - set(func.func_code.co_varnames[0:func.func_code.co_argcount])
        for a in badargs:
            del args[a]
        if badargs:
            logging.getLogger(__name__).info("sanitize_args: Called with superfluous args: %s" % ", ".join(badargs))

        return args


    def wsgi_app(self, environ, start_response, exc_info=None):
        path = environ.get("PATH_INFO", "").lstrip("/")
        self.req.reset(env=environ, path=path)
        output = ""
        status = "200 OK"
        headers = [('Content-type', 'application/json')]
        exception = None

        try:
            try:
                method = getattr(self.handler, path)
                method.exposed    # dummy access to trigger AttributeError
            except Exception:
                raise self.req.error(message="You've fallen off the cliff.", error=404)

            self.req.args = args = parse_qs(environ.get('QUERY_STRING', ""))

            self.req.client = client = self.auth.authenticate(environ, args)
            if not client:
                raise self.req.error(message="I'm watching. Authenticate.", error=403)

            auth = self.auth.authorize(self.req.env, self.req.client, self.req.path, method)
            if not auth:
                raise self.req.error(message="I'm watching. Not authorized.", error=403, client=client.name)

            args = self.sanitize_args(path, method, args)

            try:
                post_data = environ['wsgi.input'].read()
            except:
                raise self.req.error(message="Data read error.", error=408, exc=sys.exc_info())

            headers, output = method(post_data, **args)

        except Error as e:
            exception = e
        except Exception as e:
            exception = self.req.error(message="Server exception", error=500, exc=sys.exc_info())

        if exception:
            status = "%d %s" % exception.get_http_err_msg()
            output = json.dumps(exception.to_dict(), default=lambda v: str(v))
            exception.log(logging.getLogger(__name__))

        # Make sure everything is properly encoded - JSON and various function
        # may spit out unicode instead of str and it gets propagated up (str
        # + unicode = unicode). However, the right thing would be to be unicode
        # correct among whole source and always decode on input (json module
        # does that for us) and on output here.
        if isinstance(status, unicode):
            status = status.encode("utf-8")
        if isinstance(output, unicode):
            output = output.encode("utf-8")
        headers.append(('Content-Length', str(len(output))))
        start_response(status, headers)
        self.req.reset()
        return [output]


    __call__ = wsgi_app



def json_wrapper(method):

    def meth_deco(self, post, **args):
        if "events" in method.func_code.co_varnames[0:method.func_code.co_argcount]:
            try:
                events = json.loads(post) if post else None
            except Exception as e:
                raise self.req.error(message="Deserialization error.", error=400,
                    exc=sys.exc_info(), args=post, parser=str(e))
            if events:
                args["events"] = events

        result = method(self, **args)   # call requested method

        try:
            # 'default': takes care of non JSON serializable objects,
            # which could (although shouldn't) appear in handler code
            output = json.dumps(result, default=lambda v: str(v))
        except Exception as e:
            raise self.req.error(message="Serialization error", error=500,
                exc=sys.exc_info(), args=str(result))

        return [('Content-type', 'application/json')], output

    return meth_deco


class WardenHandler(ObjectReq):

    def __init__(self, req, validator, db, auth,
            send_events_limit=500, get_events_limit=1000,
            description=None):

        ObjectReq.__init__(self, req)
        self.auth = auth
        self.db = db
        self.validator = validator
        self.send_events_limit = send_events_limit
        self.get_events_limit = get_events_limit
        self.description = description


    def __str__(self):
        return "%s(req=%s, validator=%s, db=%s, send_events_limit=%s, get_events_limit=%s, description=\"%s\")" % (
            type(self).__name__, type(self.req).__name__, type(self.validator).__name__, type(self.db).__name__,
            self.get_events_limit, self.send_events_limit, self.description)


    @expose(read=1, debug=1)
    @json_wrapper
    def getDebug(self):
        return {
            "environment": self.req.env,
            "client": self.req.client.__dict__,
            "database": self.db.get_debug(),
            "system": {
                "uname": os.uname()
            },
            "process": {
                "cwd": os.getcwdu(),
                "pid": os.getpid(),
                "ppid": os.getppid(),
                "pgrp": os.getpgrp(),
                "uid": os.getuid(),
                "gid": os.getgid(),
                "euid": os.geteuid(),
                "egid": os.getegid(),
                "groups": os.getgroups()
            }
        }


    @expose(read=1)