Merge branch 'master' into devel

contrib/connectors/hp-labrea/labrea-idea.py

@@ -34,6 +34,9 @@ class WindowContextMgr(object):
         self.ideagen = ideagen
         self.first_update_queue = OrderedDict()
         self.last_update_queue = OrderedDict()
+        # Hammer to mitigate too big events
+        self.max_count = 2000
+        self.max_src_ports = 1024
 
     def expire_queue(self, queue, window):
         aggr_events = []
@@ -68,9 +71,16 @@ class WindowContextMgr(object):
                     self.first_update_queue[ctx] = self.update_timestamp
                     self.last_update_queue[ctx] = self.update_timestamp
                 else:
-                    self.ctx_append(self.contexts[ctx], event)
-                    del self.last_update_queue[ctx]
-                    self.last_update_queue[ctx] = self.update_timestamp
+                    if not self.ctx_append(self.contexts[ctx], event):
+                        closed = self.ctx_close(self.contexts[ctx])
+                        if closed is not None:
+                            aggr_events.append(closed)
+                        del self.contexts[ctx]
+                        del self.first_update_queue[ctx]
+                        del self.last_update_queue[ctx]
+                    else:
+                        del self.last_update_queue[ctx]
+                        self.last_update_queue[ctx] = self.update_timestamp
 
         return aggr_events
 
@@ -107,6 +117,7 @@ class PingContextMgr(WindowContextMgr):
         ctx["tgt_ips"].add(event.tgt_ip)
         ctx["count"] += 1
         ctx["last_update"] = self.update_timestamp
+        return ctx["count"] < self.max_count
 
     def ctx_close(self, ctx):
         return self.ideagen.gen_idea(
@@ -143,11 +154,13 @@ class ConnectContextMgr(WindowContextMgr):
         ctx["src_ports"].add(event.src_port)
         ctx["count"] += 1
         ctx["last_update"] = self.update_timestamp
+        return ctx["count"] < self.max_count
 
     def ctx_close(self, ctx):
+        src_ports = ctx["src_ports"] if len(ctx["src_ports"]) <= self.max_src_ports else None
         return self.ideagen.gen_idea(
             src=ctx["src_ip"],
-            src_ports=ctx["src_ports"],
+            src_ports=src_ports,
             targets=ctx["tgt_ips_ports"].items(),
             detect_time=self.update_timestamp,
             event_time=ctx["first_update"],
@@ -419,10 +432,6 @@ def daemonize(
                 os.close(fd)
             except Exception:
                 pass
-    # Redirect stdin, stdout, stderr to /dev/null
-    devnull = os.open(os.devnull, os.O_RDWR)
-    for fd in range(3):
-        os.dup2(devnull, fd)
     # PID file
     if pidfile is not None:
         pidd = os.open(pidfile, os.O_RDWR | os.O_CREAT | os.O_EXCL | os.O_TRUNC)
@@ -436,6 +445,10 @@ def daemonize(
                 os.unlink(pidfile)
             except Exception:
                 pass
+    # Redirect stdin, stdout, stderr to /dev/null
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for fd in range(3):
+        os.dup2(devnull, fd)
 
 
 def save_events(aggr, filer):

warden_client/README

@@ -15,7 +15,7 @@ A. Introduction
 
    The main goal of Warden 3 is to address the shortcomings, which emerged
 during several years of Warden 2.X operation. Warden 3 uses flexible and
-descriptive event format, based on JSON. Warden 3 protocol is based on  plain
+descriptive event format, based on JSON. Warden 3 protocol is based on plain
 HTTPS queries with help of JSON (Warden 2 SOAP is heavyweight, outdated and
 draws in many dependencies). Clients can be multilanguage, unlike SOAP/HTTPS,
 plain HTTPS and JSON is mature in many mainstream programming languages.
@@ -36,7 +36,7 @@ B. Quick start (TL;DR)
    sandbox URL, etc.
    If succesful, you will receive authentication secret.
  * Use warden_curl_test.sh to check you are able to talk to server.
- * See warden_client_examples.py on how to integrate sending/recieving
+ * See warden_client_examples.py on how to integrate sending/receiving
    into your Python application.
  * Alternatively, check 'contrib' directory in Warden GIT for various
    ready to use tools or recipes. You may find senders for various
@@ -65,7 +65,7 @@ C.3. Authentication
 
    In Warden 2, clients get authenticated by server certificate, however
 server certificate is usually same for the whole machine, so individual
-clients are differentiated only by telling its own name. However, client name
+clients are differentiated only by telling their own name. However, client name
 is widely known, so this allows for client impersonation within one machine.
 Warden 3 slightly improves this schema by replacing client name in
 authentication phase by "secret", random string, shared among particular
@@ -134,7 +134,7 @@ sending events). The keys of the object, which may be available, are:
     description.
 
    Client errors (4xx) are considered permanent - client must not try to send
-same event again as it will get always rejected - client administrator
+same event again as it will always get rejected - client administrator
 will need to inspect logs and rectify the cause.
 
    Server errors (5xx) may be considered by client as temporary and client is
@@ -465,4 +465,4 @@ for e in res:
 debug_str() output increasingly more detailed info.
 
 ------------------------------------------------------------------------------
-Copyright (C) 2011-2015 Cesnet z.s.p.o
+Copyright (C) 2011-2022 Cesnet z.s.p.o

warden_filer/warden_filer.py

-#!/usr/bin/python3
+#!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
 # Copyright (C) 2011-2015 Cesnet z.s.p.o

warden_filer/warden_filer_receiver

@@ -27,6 +27,8 @@ done
 function log_daemon_msg () { echo -n "$@"; }
 function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
 function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
 
 [ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
 

warden_filer/warden_filer_sender

@@ -27,6 +27,8 @@ done
 function log_daemon_msg () { echo -n "$@"; }
 function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
 function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
 
 [ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
 

warden_ra/ejbcaws.py

@@ -176,8 +176,8 @@ class HTTPSClientCertTransport(suds.transport.http.HttpTransport):
         self.key = key
         self.cert = cert
 
-    def u2open(self, u2request):
-        tm = self.options.timeout
+    def u2open(self, u2request, timeout=None):
+        tm = timeout or self.options.timeout
         if sys.version_info[0] >= 3:
             url = urllib.request.build_opener(HTTPSClientAuthHandler(self.key, self.cert))
         else:

warden_ra/warden_apply.sh

@@ -13,8 +13,8 @@ fi
 url="$1"
 client="$2"
 password="$3"
-incert="$4"
-inkey="$5"
+incert="$3"
+inkey="$4"
 
 trap 'rm -f "$config $result"' INT TERM HUP EXIT