Server now limits max accepted event size. events.data changed to longtext to...

Server now limits max accepted event size. events.data changed to longtext to accomodate longer events

warden3/warden_server/README

@@ -156,6 +156,7 @@ object from particular section list is used ("FileLogger" for example).
       port: database server port, default 3306
       retry_pause: retry in case of database errors, in seconds, defaults to 5
       retry_count: number of retries, defaults to 3
+      event_size_limit: max size of serialized event, defaults to 5 MB
       catmap_filename: IDEA category mapping to database ids, defaults to
                        "catmap_mysql.json" at installation directory
       tagmap_filename": IDEA node type mapping to database ids, defaults to

warden3/warden_server/warden_3.0.sql

@@ -66,7 +66,7 @@ CREATE TABLE IF NOT EXISTS `events` (
   `id` int(11) NOT NULL AUTO_INCREMENT,
   `received` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `client_id` int(11) NOT NULL,
-  `data` text NOT NULL,
+  `data` longtext NOT NULL,
   `valid` tinyint(1) NOT NULL DEFAULT '1',
   PRIMARY KEY (`id`),
   KEY `id` (`id`,`client_id`)

warden3/warden_server/warden_server.py

@@ -422,7 +422,8 @@ class JSONSchemaValidator(NoValidator):
 
 class MySQL(ObjectReq):
 
-    def __init__(self, req, host, user, password, dbname, port, retry_count, retry_pause, catmap_filename, tagmap_filename):
+    def __init__(self, req, host, user, password, dbname, port, retry_count,
+            retry_pause, event_size_limit, catmap_filename, tagmap_filename):
         ObjectReq.__init__(self, req)
         self.host = host
         self.user = user
@@ -431,6 +432,7 @@ class MySQL(ObjectReq):
         self.port = port
         self.retry_count = retry_count
         self.retry_pause = retry_pause
+        self.event_size_limit = event_size_limit
         self.catmap_filename = catmap_filename
         self.tagmap_filename = tagmap_filename
 
@@ -669,8 +671,12 @@ class MySQL(ObjectReq):
 
 
     def store_event(self, client, event):
+        json_event = json.dumps(event)
+        if len(json_event) >= self.event_size_limit:
+            return [{"error": 413, "message": "Event too long (>%i B)" % self.event_size_limit}]
         try:
-            self.query("INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)", (client.id, json.dumps(event)), dml=True)
+            self.query("INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)",
+                (client.id, json_event), dml=True)
             lastid = self.crs.lastrowid
 
             catlist = event.get('Category', ["Other"])
@@ -1151,6 +1157,7 @@ def build_server(conf):
             "port": {"type": natural, "default": 3306},
             "retry_pause": {"type": natural, "default": 5},
             "retry_count": {"type": natural, "default": 3},
+            "event_size_limit": {"type": natural, "default": 5*1024*1024},
             "catmap_filename": {"type": filepath, "default": path.join(path.dirname(__file__), "catmap_mysql.json")},
             "tagmap_filename": {"type": filepath, "default": path.join(path.dirname(__file__), "tagmap_mysql.json")}
         },