Mitigate too big Idea events (close context if too big)

contrib/connectors/hp-labrea/labrea-idea.py

@@ -34,6 +34,9 @@ class WindowContextMgr(object):
         self.ideagen = ideagen
         self.first_update_queue = OrderedDict()
         self.last_update_queue = OrderedDict()
+        # Hammer to mitigate too big events
+        self.max_count = 2000
+        self.max_src_ports = 1024
 
     def expire_queue(self, queue, window):
         aggr_events = []
@@ -68,9 +71,16 @@ class WindowContextMgr(object):
                     self.first_update_queue[ctx] = self.update_timestamp
                     self.last_update_queue[ctx] = self.update_timestamp
                 else:
-                    self.ctx_append(self.contexts[ctx], event)
-                    del self.last_update_queue[ctx]
-                    self.last_update_queue[ctx] = self.update_timestamp
+                    if not self.ctx_append(self.contexts[ctx], event):
+                        closed = self.ctx_close(self.contexts[ctx])
+                        if closed is not None:
+                            aggr_events.append(closed)
+                        del self.contexts[ctx]
+                        del self.first_update_queue[ctx]
+                        del self.last_update_queue[ctx]
+                    else:
+                        del self.last_update_queue[ctx]
+                        self.last_update_queue[ctx] = self.update_timestamp
 
         return aggr_events
 
@@ -107,6 +117,7 @@ class PingContextMgr(WindowContextMgr):
         ctx["tgt_ips"].add(event.tgt_ip)
         ctx["count"] += 1
         ctx["last_update"] = self.update_timestamp
+        return ctx["count"] < self.max_count
 
     def ctx_close(self, ctx):
         return self.ideagen.gen_idea(
@@ -143,11 +154,13 @@ class ConnectContextMgr(WindowContextMgr):
         ctx["src_ports"].add(event.src_port)
         ctx["count"] += 1
         ctx["last_update"] = self.update_timestamp
+        return ctx["count"] < self.max_count
 
     def ctx_close(self, ctx):
+        src_ports = ctx["src_ports"] if len(ctx["src_ports"]) <= self.max_src_ports else None
         return self.ideagen.gen_idea(
             src=ctx["src_ip"],
-            src_ports=ctx["src_ports"],
+            src_ports=src_ports,
             targets=ctx["tgt_ips_ports"].items(),
             detect_time=self.update_timestamp,
             event_time=ctx["first_update"],