* Filer now works on send_event_limit sized chunks to avoid running off memory of timing out

* Shorter example config

warden3/contrib/warden_filer/warden_filer.cfg

@@ -30,7 +30,6 @@
             "Type": ["Relay"],
             "SW": ["warden_filer-sender"],
             "AggrWin": "00:05:00",
-            "Note": "Test warden_filer sender"
         }
     },
     "receiver": {

warden3/contrib/warden_filer/warden_filer.py

@@ -21,6 +21,7 @@ from random import choice, randint;
 from daemon import DaemonContext
 from daemon.pidlockfile import TimeoutPIDLockFile
 
+VERSION = "3.0-beta1"
 
 class NamedFile(object):
     """ Wrapper class for file objects, which allows and tracks filename
@@ -213,7 +214,6 @@ def match_event(event, cat=None, nocat=None, tag=None, notag=None, group=None, n
 
 
 def sender(config, wclient, sdir, oneshot):
-    send_events_limit = config.get("send_events_limit", 500)
     poll_time = config.get("poll_time", 5)
     node = config.get("node", None)
     conf_filt = config.get("filter", {})
@@ -227,15 +227,17 @@ def sender(config, wclient, sdir, oneshot):
         if oneshot:
             terminate_me(None, None)
         while running_flag and not nflist:
+            # No new files, wait and try again
             time.sleep(poll_time)
             nflist = sdir.get_incoming()
-        # count chunk iterations rounded up
-        count = len(nflist)
 
+        nfindex = 0
+        nfchunk = wclient.send_events_limit
+        while nfindex<len(nflist):
             events = []
             nf_sent = []
             count_ok = count_err = count_unmatched = 0
-        for nf in nflist:
+            for nf in nflist[nfindex:nfindex+nfchunk]:
                 # prepare event array from files
                 try:
                     nf.moveto(sdir.temp)
@@ -272,7 +274,7 @@ def sender(config, wclient, sdir, oneshot):
                             nf_sent[i] = None
                             count_err += 1
 
-        # Cleanup rest - succesfully sent events
+            # Cleanup rest - the succesfully sent events
             for name in nf_sent:
                 if name:
                     name.remove()
@@ -280,6 +282,9 @@ def sender(config, wclient, sdir, oneshot):
             wclient.logger.info(
                 "warden_filer: saved %d, errors %d, unmatched %d" % (count_ok, count_err, count_unmatched))
 
+            nfindex += nfchunk  # skip to next chunk of files
+            nfchunk = wclient.send_events_limit # might get changed by server
+
 
 
 def get_logger_files(logger):