contrib/map/frontend/css/warden-map.css

+/*
+ *
+ * -*- coding: utf-8 -*-
+ *
+ * warden-map.css
+ *
+ * Copyright (C) 2016 Cesnet z.s.p.o
+ * Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+ *
+*/
+
+body {
+  font-family: 'Oswald', sans-serif;
+  background: #00253D;
+  border: 0px;
+  padding: 0px;
+  margin: 0px;
+}
+
+h2 {
+ color: #0062a2;
+}
+
+.hoverinfo {
+  font-family: 'Oswald', sans-serif;
+}
+
+#country {
+  color: #0062a2;	/* Cesnet blue */
+  font-weight: bold;
+}
+
+
+table {
+  text-align: left;
+  margin: 0;
+  padding: 0;
+  font-size: 12px;
+}
+
+table th {
+  color: #0062a2;	/* Cesnet blue */
+  padding: 0;
+}
+
+table td {
+  color: #4b4d4a;	/* Greenish gray */
+  padding: 0;
+}
+
+#container {
+  overflow: hidden;
+/*  border: 2px solid #0062a2;
+  border: 0px;
+  padding: 0px;
+  margin: 0px;
+  border-radius: 5px;*/
+  position: relative;
+/*  width: 1280px;
+  height: 720px;*/
+  max-width: 100%;
+  max-height: 100%
+  width: 100%;
+  height: 100vh;*/
+}
+
+.zoom-button {
+  width: 40px;
+  height: 40px;
+  border-radius: 5px;
+  border: none;
+  background: #dcdcda;
+  font-size: 23px;
+  font-weight: bold;
+  color: white;
+  cursor: pointer;
+}
+
+.zoom-button:hover {
+  background-color: #0062a2;
+}
+
+#zoom-info {
+  display: inline-block;
+  padding: 10px;
+  color: #0062a2;
+}
+
+#warden-logo {
+  position: absolute;
+  top: 30px;
+  left: 30px;
+  background: white;
+  padding: 10px;
+  border-radius: 10px;
+  width: 240px;
+  height: 92px;
+  text-align: center;
+}
+
+#cesnet-logo {
+  position: absolute;
+  top: 30px;
+  right: 30px;
+  background: white;
+  padding: 10px;
+  border-radius: 10px;
+  width: 240px;
+  height: 92px;
+  text-align: center;
+}
+
+#legend-box {
+  position: absolute;
+  bottom: 30px;
+  left: 30px;
+  background-color: rgba(0,0,0,0.3);
+  color: white;
+  padding: 10px;
+  border-radius: 10px;
+  /*width: 240px;
+  height: 92px;
+  text-align: center;*/
+}
+
+#heading {
+  position: absolute;
+  top: 30px;
+  left: 50%;
+  width: 40em;
+  height: 92px;
+  margin-left: -20em;
+  font-size: xx-large;
+  color: white;
+  text-align: center;
+  vertical-align: middle;
+  line-height: 92px;
+}

contrib/map/frontend/js/warden-map.js

+/*
+ *
+ * -*- coding: utf-8 -*-
+ *
+ * warden-map.js
+ *
+ * Copyright (C) 2016 Cesnet z.s.p.o
+ * Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+ *
+*/
+
+// NOTE: Change path in a function d3.json() if you separate backend and frontend!
+
+// Zooming functionality is based on WunderBart's implementation
+// Please see following links:
+// https://github.com/wunderbart
+// https://jsfiddle.net/wunderbart/Lom3b0gb/
+
+  function Zoom(args) {
+  $.extend(this, {
+    $buttons:   $(".zoom-button"),
+    $info:      $("#zoom-info"),
+    scale:      { max: 50, currentShift: 0 },
+    $container: args.$container,
+    datamap:    args.datamap
+  });
+
+  this.init();
+}
+
+Zoom.prototype.init = function() {
+  var paths = this.datamap.svg.selectAll("path"),
+      subunits = this.datamap.svg.selectAll(".datamaps-subunit");
+
+  // preserve stroke thickness
+  paths.style("vector-effect", "non-scaling-stroke");
+
+  // disable click on drag end
+  subunits.call(
+    d3.behavior.drag().on("dragend", function() {
+      d3.event.sourceEvent.stopPropagation();
+    })
+  );
+
+  this.scale.set = this._getScalesArray();
+  this.d3Zoom = d3.behavior.zoom().scaleExtent([ 1, this.scale.max ]);
+
+  this._displayPercentage(1);
+  this.listen();
+};
+
+Zoom.prototype.listen = function() {
+  this.$buttons.off("click").on("click", this._handleClick.bind(this));
+
+  this.datamap.svg
+    .call(this.d3Zoom.on("zoom", this._handleScroll.bind(this)))
+    .on("dblclick.zoom", null); // disable zoom on double-click
+};
+
+Zoom.prototype.reset = function() {
+  this._shift("reset");
+};
+
+Zoom.prototype._handleScroll = function() {
+  var translate = d3.event.translate,
+      scale = d3.event.scale,
+      limited = this._bound(translate, scale);
+
+  this.scrolled = true;
+
+  this._update(limited.translate, limited.scale);
+};
+
+Zoom.prototype._handleClick = function(event) {
+  var direction = $(event.target).data("zoom");
+
+  this._shift(direction);
+};
+
+Zoom.prototype._shift = function(direction) {
+  var center = [ this.$container.width() / 2, this.$container.height() / 2 ],
+      translate = this.d3Zoom.translate(), translate0 = [], l = [],
+      view = {
+        x: translate[0],
+        y: translate[1],
+        k: this.d3Zoom.scale()
+      }, bounded;
+
+  translate0 = [
+    (center[0] - view.x) / view.k,
+    (center[1] - view.y) / view.k
+  ];
+
+  if (direction == "reset") {
+    view.k = 1;
+    this.scrolled = true;
+  } else {
+    view.k = this._getNextScale(direction);
+  }
+
+l = [ translate0[0] * view.k + view.x, translate0[1] * view.k + view.y ];
+
+  view.x += center[0] - l[0];
+  view.y += center[1] - l[1];
+
+  bounded = this._bound([ view.x, view.y ], view.k);
+
+  this._animate(bounded.translate, bounded.scale);
+};
+
+Zoom.prototype._bound = function(translate, scale) {
+  var width = this.$container.width(),
+      height = this.$container.height();
+
+  translate[0] = Math.min(
+    (width / height)  * (scale - 1),
+    Math.max( width * (1 - scale), translate[0] )
+  );
+
+  translate[1] = Math.min(0, Math.max(height * (1 - scale), translate[1]));
+
+  return { translate: translate, scale: scale };
+};
+
+Zoom.prototype._update = function(translate, scale) {
+  this.d3Zoom
+    .translate(translate)
+    .scale(scale);
+
+  this.datamap.svg.selectAll("g")
+    .attr("transform", "translate(" + translate + ")scale(" + scale + ")");
+
+  this._displayPercentage(scale);
+};
+
+Zoom.prototype._animate = function(translate, scale) {
+  var _this = this,
+      d3Zoom = this.d3Zoom;
+
+  d3.transition().duration(350).tween("zoom", function() {
+    var iTranslate = d3.interpolate(d3Zoom.translate(), translate),
+        iScale = d3.interpolate(d3Zoom.scale(), scale);
+
+    return function(t) {
+      _this._update(iTranslate(t), iScale(t));
+    };
+  });
+};
+
+Zoom.prototype._displayPercentage = function(scale) {
+  var value;
+
+  value = Math.round(Math.log(scale) / Math.log(this.scale.max) * 100);
+  this.$info.text(value + "%");
+};
+
+Zoom.prototype._getScalesArray = function() {
+  var array = [],
+      scaleMaxLog = Math.log(this.scale.max);
+
+  for (var i = 0; i <= 10; i++) {
+    array.push(Math.pow(Math.E, 0.1 * i * scaleMaxLog));
+  }
+
+  return array;
+};
+
+Zoom.prototype._getNextScale = function(direction) {
+  var scaleSet = this.scale.set,
+      currentScale = this.d3Zoom.scale(),
+      lastShift = scaleSet.length - 1,
+      shift, temp = [];
+
+  if (this.scrolled) {
+
+    for (shift = 0; shift <= lastShift; shift++) {
+      temp.push(Math.abs(scaleSet[shift] - currentScale));
+    }
+
+    shift = temp.indexOf(Math.min.apply(null, temp));
+
+    if (currentScale >= scaleSet[shift] && shift < lastShift) {
+      shift++;
+    }
+
+    if (direction == "out" && shift > 0) {
+      shift--;
+    }
+
+    this.scrolled = false;
+
+  } else {
+
+    shift = this.scale.currentShift;
+
+    if (direction == "out") {
+      shift > 0 && shift--;
+    } else {
+      shift < lastShift && shift++;
+    }
+  }
+
+  this.scale.currentShift = shift;
+
+  return scaleSet[shift];
+};
+
+
+ function defaults(obj) {
+    Array.prototype.slice.call(arguments, 1).forEach(function(source) {
+      if (source) {
+        for (var prop in source) {
+          // Deep copy if property not set
+          if (obj[prop] == null) {
+            if (typeof source[prop] == 'function') {
+              obj[prop] = source[prop];
+            }
+            else {
+              obj[prop] = JSON.parse(JSON.stringify(source[prop]));
+            }
+          }
+        }
+      }
+    });
+    return obj;
+  }
+
+function val( datumValue, optionsValue, context ) {
+    if ( typeof context === 'undefined' ) {
+      context = optionsValue;
+      optionsValues = undefined;
+    }
+    var value = typeof datumValue !== 'undefined' ? datumValue : optionsValue;
+
+    if (typeof value === 'undefined') {
+      return  null;
+    }
+
+    if ( typeof value === 'function' ) {
+      var fnContext = [context];
+      if ( context.geography ) {
+        fnContext = [context.geography, context.data];
+      }
+      return value.apply(null, fnContext);
+    }
+    else {
+      return value;
+    }
+  }
+  
+var cat_color = {
+  "Abusive": "MediumPurple",
+  "Malware": "Red",
+  "Recon": "LightSlateGray",
+  "Attempt": "GhostWhite",
+  "Intrusion": "DarkTurquoise",
+  "Availability": "HotPink",
+  "Information": "PaleTurquoise",
+  "Fraud": "Yellow",
+  "Vulnerable": "DarkGoldenRod",
+  "Anomaly": "Brown",
+  "Other": "Green"
+}
+
+var cat_desc = {
+  "Abusive": "spam",
+  "Malware": "virus, worm, trojan, malware",
+  "Recon": "scanning, sniffing",
+  "Attempt": "bruteforce, exploitation attempt",
+  "Intrusion": "botnet, successful exploit",
+  "Availability": "(D)DOS",
+  "Information": "wiretapping, spoofing, hijacking",
+  "Fraud": "phishing, scam",
+  "Vulnerable": "open for abuse",
+  "Anomaly": "unusual traffic",
+  "Other": "unknown/unidentified"
+}
+
+ function handleArcs (layer, data, options) {
+    var self = this,
+        svg = this.svg;
+
+    if ( !data || (data && !data.slice) ) {
+      throw "Datamaps Error - arcs must be an array";
+    }
+
+    // For some reason arc options were put in an `options` object instead of the parent arc
+    // I don't like this, so to match bubbles and other plugins I'm moving it
+    // This is to keep backwards compatability
+    for ( var i = 0; i < data.length; i++ ) {
+      data[i] = defaults(data[i], data[i].options);
+      delete data[i].options;
+    }
+
+    if ( typeof options === "undefined" ) {
+      options = defaultOptions.arcConfig;
+    }
+
+    var arcs = layer.selectAll('path.datamaps-arc').data( data, JSON.stringify );
+
+    var path = d3.geo.path()
+        .projection(self.projection);
+
+    arcs
+      .enter()
+        .append('svg:path')
+        .attr('class', 'datamaps-arc')
+        .style('stroke-linecap', 'round')
+        .style('stroke', function(datum) {
+/*          return val(datum.strokeColor, options.strokeColor, datum);*/
+          for (cat in cat_color) {
+            if (datum.event.startsWith(cat)) {
+              return cat_color[cat];
+            }
+          }
+          return "Green";
+        })
+        .style('fill', 'none')
+        .style('stroke-width', function(datum) {
+            return val(datum.strokeWidth, options.strokeWidth, datum);
+        })
+        .attr('d', function(datum) {
+
+            var originXY, destXY;
+
+            originXY = self.latLngToXY(val(datum.origin.latitude, datum), val(datum.origin.longitude, datum))
+
+            destXY = self.latLngToXY(val(datum.destination.latitude, datum), val(datum.destination.longitude, datum));
+
+            var midXY = [ (originXY[0] + destXY[0]) / 2, (originXY[1] + destXY[1]) / 2];
+            if (options.greatArc) {
+                  // TODO: Move this to inside `if` clause when setting attr `d`
+              var greatArc = d3.geo.greatArc()
+                  .source(function(d) { return [val(d.origin.longitude, d), val(d.origin.latitude, d)]; })
+                  .target(function(d) { return [val(d.destination.longitude, d), val(d.destination.latitude, d)]; });
+
+              return path(greatArc(datum))
+            }
+            var sharpness = val(datum.arcSharpness, options.arcSharpness, datum);
+            return "M" + originXY[0] + ',' + originXY[1] + "S" + (midXY[0] + (50 * sharpness)) + "," + (midXY[1] - (75 * sharpness)) + "," + destXY[0] + "," + destXY[1];
+        })
+        .attr('data-info', function(datum) {
+          return JSON.stringify(datum);
+        })
+        .on('mouseover', function ( datum ) {
+          var $this = d3.select(this);
+
+          if (options.popupOnHover) {
+            self.updatePopup($this, datum, options, svg);
+          }
+        })
+        .on('mouseout', function ( datum ) {
+          var $this = d3.select(this);
+
+          d3.selectAll('.datamaps-hoverover').style('display', 'none');
+        })
+        .transition()
+          .style('fill', function(datum, i) {
+            /*
+              Thank you Jake Archibald, this is awesome.
+              Source: http://jakearchibald.com/2013/animated-line-drawing-svg/
+            */
+            var length = this.getTotalLength();
+            this.style.transition = this.style.WebkitTransition = 'none';
+            this.style.strokeDasharray = length + ' ' + length;
+            this.style.strokeDashoffset = length;
+            this.getBoundingClientRect();
+            this.style.transition = this.style.WebkitTransition = 'stroke-dashoffset ' + val(datum.animationSpeed, options.animationSpeed, datum) + 'ms ' + datum.delay*1000 + 'ms ease-out';
+            this.style.strokeDashoffset = '0';
+            return 'none';
+          });
+
+    arcs.exit()
+      .transition()
+      .duration(1000)
+      .style('opacity', 0)
+      .remove();
+  }
+
+var main_data = [];
+var prev_data = 0;
+
+// Configuration of datamap canvas
+// Futher reading can be found at https://datamaps.github.io/
+function Datamap() {
+  this.$container = $("#container");
+  instance = this.instance = new Datamaps({
+    scope: 'world',
+    element: this.$container.get(0),
+    done: this._handleMapReady.bind(this),
+    projection: 'mercator',
+        fills: {
+          /*defaultFill: '#454545'*/
+          defaultFill: 'black'
+        },
+        geographyConfig: {
+          hideAntarctica: true,
+          borderColor: '#0062a2',
+          highlightFillColor: '#4b4d4a',
+          highlightBorderColor: '#fdfdfd',
+          popupOnHover: true,
+          popupTemplate: function(geography, data) {
+          return '<div class="hoverinfo" id="country">' + geography.properties.name + '</div>';
+        },
+        },
+        ph_arcConfig: {
+          strokeColor: '#0062a2',
+          strokeWidth: 2,
+          arcSharpness: 2, /* 5 */
+          animationSpeed: 3000, // Milliseconds
+          popupOnHover: true,
+          // Case with latitude and longitude
+          popupTemplate: function(geography, data) {
+            if ( ( data.origin && data.destination ) && data.origin.latitude && data.origin.longitude && data.destination.latitude && data.destination.longitude ) {
+              // Content of info table
+              str = '<div class="hoverinfo"><table id="event"><tr><th>Warden Event</th></tr><tr><td>Type</td><td>'+ JSON.stringify(data.event) +'</td></tr><tr><td>Detect Time</td><td>'+ JSON.stringify(data.time) +'</td></tr><tr><th>Event origin</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.origin.ip) +  '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.origin.city) + ',&nbsp;' + JSON.stringify(data.origin.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.origin.latitude) + ',&nbsp;' + JSON.stringify(data.origin.longitude) + '</td></tr><tr><th>Event Destination</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.destination.ip) + '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.destination.city) + ',&nbsp;' + JSON.stringify(data.destination.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.destination.latitude) + ',&nbsp;' + JSON.stringify(data.destination.longitude) + '</td></tr></table></div>';
+              return str.replace(/&quot;/g,"");
+            }
+            // Missing information
+            else {
+              return '';
+            }
+          }
+        }
+      });
+
+  legend_data = d3.select("#legend")
+    .selectAll("li")
+    .data(Object.keys(cat_color).sort())
+    .enter()
+      .append("li")
+        .append("span")
+          .style("color", function(datum) { return cat_color[datum]})
+          .text(function(datum) { return datum; })
+          .append("span")
+          .text(function(datum) { return " – " + cat_desc[datum]})
+          .style("color", "white");
+  
+    instance.addPlugin('ph_arc', handleArcs);
+
+  setInterval(function(){
+      d3.json("./warden-map.json", function(error, data) {
+          if (data) {
+              var cur_data = data.pop()
+              var cur_time = new Date().getTime();
+              if (cur_data != prev_data) {
+                  prev_data = cur_data;
+                  for (var i=0; i<data.length; i++) {
+                      data[i].arrivalTime = cur_time;
+                      data[i].delay = i/data.length;
+                  }
+                  main_data = main_data.concat(data);
+              }
+          }
+          var trimmed_data = [];
+          for (var i=0; i<main_data.length; i++) {
+              if (main_data[i].arrivalTime + 3500 > cur_time) {
+                  trimmed_data.push(main_data[i]);
+              }
+          }
+          main_data = trimmed_data;
+          trimmed_data = cur_time = cur_data = error = data = null;
+          instance.ph_arc(main_data);
+      });
+  }, 1000);
+
+};
+
+Datamap.prototype._handleMapReady = function(datamap) {
+  this.zoom = new Zoom({
+    $container: this.$container,
+    datamap: datamap
+  });
+}

contrib/map/frontend/warden-map.html

+<!--                                                                                   -->
+<!--                                                                                   -->
+<!-- -*- coding: utf-8 -*-                                                             -->
+<!--                                                                                   -->
+<!--  warden-map.html                                                                  -->
+<!--                                                                                   -->
+<!-- Copyright (C) 2016 Cesnet z.s.p.o                                                 -->
+<!-- Use of this source is governed by a 3-clause BSD-style license, see LICENSE file. -->
+<!--                                                                                   -->
+<!--                                                                                   -->
+
+
+<!DOCTYPE html>
+<meta name="robots" content="noindex">
+<meta charset="utf-8">
+<link href='https://fonts.googleapis.com/css?family=Oswald&amp;subset=latin,latin-ext' rel='stylesheet' type='text/css'>
+<link rel="stylesheet" type="text/css" href="./css/warden-map.css"/>
+<body>
+
+<script src="https://d3js.org/d3.v3.min.js"></script>
+<script src="https://d3js.org/topojson.v1.min.js"></script>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+<script src="./js/datamaps.world.min.js"></script>
+<script src="./js/warden-map.js"></script>
+
+<!--
+<h2>Warden Map</h2>
+<div id="tools">
+  <button class="zoom-button" data-zoom="reset">&#x2302</button>
+  <button class="zoom-button" data-zoom="out">-</button>
+  <button class="zoom-button" data-zoom="in">+</button>
+  <div id="zoom-info"></div>
+</div>
+-->
+<div id="container"></div>
+<div id="heading">Attacks, detected in CESNET network<br/>
+SABU - Sharing and Analysis of Security Events
+</div>
+<div id="legend-box">
+	<p><b>Reported to Warden right <i>now</i>.</b></p>
+	<ul id="legend"></ul>
+</div>
+
+<!-- Draw datamap into id="container" -->
+<script>new Datamap();</script>
+
+</body>
+</html>

warden3/warden_client/warden_client.cfg

-{
-    "url": "https://midas.civ.zcu.cz:8888/warden3",
-
-    "certfile": "kostik.zcu.cz-cert.pem",
-    "keyfile": "kostik.zcu.cz-key.pem",
-    "cafile": "Warden_CA-cacert.pem",
-
-    "timeout": 60,
-    "recv_events_limit": 6000,
-
-    "errlog": {"level": "debug"},
-    "filelog": {"file": "warden_client.log", "level": "warning"},
-    #"syslog": {"socket": "/dev/log", "facility": "local7", "level": "warning"},
-
-    "idstore": "warden_client.id",
-
-    "name": "warden_client_kostik",
-
-    "secret": "Phaipe5ush7p"
-}

warden_client/warden_client.cfg.dist

+{
+    "url": "https://warden-hub.example.org/warden3",
+    "certfile": "cert.pem",
+    "keyfile": "key.pem",
+    "filelog": {"level": "debug"},
+    "name": "org.example.warden_client",
+    "secret": "ToP_SeCrEt"
+}

warden3/warden_client/warden_client.py → warden_client/warden_client.py

@@ -4,17 +4,34 @@
 # Copyright (C) 2011-2015 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-import json, httplib, ssl, socket, logging, logging.handlers, time
-from urlparse import urlparse
-from urllib import urlencode
-from sys import stderr, exc_info
-from pprint import pformat
+import hashlib  # Some Python/ssl versions incorrectly initialize hashes, this helps
+from sys import stderr, exc_info, version_info
+import json, ssl, socket, logging, logging.handlers, time
 from traceback import format_tb
 from os import path
 from operator import itemgetter
 
+fix_logging_filename = str if version_info<(2, 7) else lambda x: x
 
-VERSION = "3.0-beta0"
+if version_info[0] >= 3:
+    import http.client as httplib
+    from urllib.parse import urlparse
+    from urllib.parse import urlencode
+    basestring = str
+else:
+    import httplib
+    from urlparse import urlparse
+    from urllib import urlencode
+
+
+
+VERSION = "3.0-beta3"
+
+DEFAULT_CA_STORES = [
+    "/etc/ssl/certs/ca-certificates.crt",       # Deb
+    "/etc/pki/tls/certs/ca-bundle.crt",         # RH
+    "/var/lib/ca-certificates/ca-bundle.pem"    # SuSE
+ ]
 
 class HTTPSConnection(httplib.HTTPSConnection):
     '''
@@ -22,11 +39,12 @@ class HTTPSConnection(httplib.HTTPSConnection):
     of SSL/ TLS version and cipher selection.  See:
     http://hg.python.org/cpython/file/c1c45755397b/Lib/httplib.py#l1144
     and `ssl.wrap_socket()`
+    Used only if ssl.SSLContext is not available (Python version < 2.7.9)
     '''
     def __init__(self, host, **kwargs):
-        self.ciphers = kwargs.pop('ciphers',None)
-        self.ca_certs = kwargs.pop('ca_certs',None)
-        self.ssl_version = kwargs.pop('ssl_version',ssl.PROTOCOL_SSLv23)
+        self.ciphers = kwargs.pop('ciphers', None)
+        self.ca_certs = kwargs.pop('ca_certs', None)
+        self.ssl_version = kwargs.pop('ssl_version', getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23))
 
         httplib.HTTPSConnection.__init__(self,host,**kwargs)
 
@@ -103,6 +121,14 @@ class Error(Exception):
                 kwargs["send_events_limit"] = int(kwargs["send_events_limit"])
             except Exception:
                 del kwargs["send_events_limit"]
+        if "exc" in kwargs:
+            # Traceback objects cause reference loops, so memory may be not
+            # correctly free'd. We only need traceback to log it in str_debug(),
+            # so let's get the string representation now and forget the 
+            # traceback object, thus preventing the loop.
+            exctype, excvalue, tb = kwargs["exc"]
+            tb = format_tb(tb)
+            kwargs["exc"] = exctype, excvalue, tb
         self.errors.append(kwargs)
 
 
@@ -133,6 +159,8 @@ class Error(Exception):
         """ In list or iterable context we're empty """
         raise StopIteration
 
+    __next__ = next
+
 
     def __bool__(self):
         """ In boolean context we're never True """
@@ -192,10 +220,10 @@ class Error(Exception):
         out.append(self.str_preamble(e))
         if not "exc" in e or not e["exc"]:
             return ""
-        exc_tb = e["exc"][2]
+        exc_tb = e["exc"][2] # exc_tb is string repr. of traceback object
         if exc_tb:
             out.append("Traceback:\n")
-            out.extend(format_tb(exc_tb))
+            out.extend(exc_tb)
         return "".join(out)
 
 
@@ -208,9 +236,9 @@ class Client(object):
             keyfile=None,
             cafile=None,
             timeout=60,
-            retry=3,
-            pause=5,
-            recv_events_limit=6000,
+            retry=20,
+            pause=10,
+            get_events_limit=6000,
             send_events_limit=500,
             errlog={},
             syslog=None,
@@ -230,23 +258,53 @@ class Client(object):
         self.conn = None
 
         base = path.join(path.dirname(__file__))
-        self.certfile = path.join(base, certfile or "cert.pem")
-        self.keyfile  = path.join(base, keyfile or "key.pem")
-        self.cafile = path.join(base, cafile or "ca.pem")
+        self.certfile = self.get_readable_file(certfile or "cert.pem", base)
+        self.keyfile  = self.get_readable_file(keyfile or "key.pem", base)
+        self.cafile = self.get_readable_file(cafile if cafile is not None else DEFAULT_CA_STORES, base)
         self.timeout = int(timeout)
-        self.recv_events_limit = int(recv_events_limit)
+        self.get_events_limit = int(get_events_limit)
         self.idstore = path.join(base, idstore) if idstore is not None else None
 
         self.send_events_limit = int(send_events_limit)
         self.retry = int(retry)
         self.pause = int(pause)
 
-        self.ciphers = 'TLS_RSA_WITH_AES_256_CBC_SHA'
-        self.sslversion = ssl.PROTOCOL_TLSv1
+        self.ciphers = None
+        self.sslversion = getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23)
+
+        # If Python is new enough to have SSLContext, use it for SSL settings,
+        # otherwise our own class derived from httplib.HTTPSConnection is used
+        # later in connect().
+        if hasattr(ssl, 'SSLContext'):
+            self.sslcontext = ssl.SSLContext(self.sslversion)
+            self.sslcontext.load_cert_chain(self.certfile, self.keyfile)
+            if self.cafile:
+                self.sslcontext.load_verify_locations(self.cafile)
+                self.sslcontext.verify_mode = ssl.CERT_REQUIRED
+            else:
+                self.sslcontext.verify_mode = ssl.CERT_NONE
+        else:
+            self.sslcontext = None
 
         self.getInfo()  # Call to align limits with server opinion
 
 
+    def get_readable_file(self, name, base):
+        names = [name] if isinstance(name, basestring) else name
+        names = [path.join(base, n) for n in names]
+        errors = []
+        for n in names:
+            try:
+                open(n, "r").close()
+                self.logger.debug("Using %s" % n)
+                return n
+            except IOError as e:
+                errors.append(e)
+        for e in errors:
+            self.logger.error(str(e))
+        return names[0] if names else None
+
+
     def init_log(self, errlog, syslog, filelog):
 
         def loglevel(lev):
@@ -282,7 +340,8 @@ class Client(object):
                 fl = logging.FileHandler(
                     filename=path.join(
                         path.dirname(__file__),
-                        filelog.get("file", "%s.log" % self.name)))
+                        filelog.get("file", "%s.log" % self.name)),
+                        encoding="utf-8")
                 fl.setLevel(loglevel(filelog.get("level", "debug")))
                 fl.setFormatter(format_time)
                 self.logger.addHandler(fl)
@@ -292,7 +351,7 @@ class Client(object):
         if syslog is not None:
             try:
                 sl = logging.handlers.SysLogHandler(
-                    address=syslog.get("socket", "/dev/log"),
+                    address=fix_logging_filename(syslog.get("socket", "/dev/log")),
                     facility=facility(syslog.get("facility", "local7")))
                 sl.setLevel(loglevel(syslog.get("level", "debug")))
                 sl.setFormatter(format_notime)
@@ -317,19 +376,23 @@ class Client(object):
 
         try:
             if self.url.scheme=="https":
-                conn = HTTPSConnection(
-                    self.url.netloc,
-                    strict = False,
-                    key_file = self.keyfile,
-                    cert_file = self.certfile,
-                    timeout = self.timeout,
-                    ciphers = self.ciphers,
-                    ca_certs = self.cafile,
-                    ssl_version = self.sslversion)
+                if self.sslcontext:
+                    conn = httplib.HTTPSConnection(
+                        self.url.netloc,
+                        timeout = self.timeout,
+                        context = self.sslcontext)
+                else:
+                    conn = HTTPSConnection(
+                        self.url.netloc,
+                        key_file = self.keyfile,
+                        cert_file = self.certfile,
+                        timeout = self.timeout,
+                        ciphers = self.ciphers,
+                        ca_certs = self.cafile,
+                        ssl_version = self.sslversion)
             elif self.url.scheme=="http":
                 conn = httplib.HTTPConnection(
                     self.url.netloc,
-                    strict = False,
                     timeout = self.timeout)
             else:
                 return Error(message="Don't know how to connect to \"%s\"" % self.url.scheme,
@@ -355,27 +418,30 @@ class Client(object):
             kwargs["secret"] = self.secret
 
         if kwargs:
-            for k in kwargs.keys():
+            for k in list(kwargs.keys()):
                 if kwargs[k] is None:
                     del kwargs[k]
             argurl = "?" + urlencode(kwargs, doseq=True)
         else:
             argurl = ""
 
-        try:
-            if payload is None:
-                data = ""
-            else:
+        self.headers = {"Accept": "application/json"}
+        data = None
+
+        if payload is None:
+            method = "GET"
+        else:
+            method = "POST"
+            try:
                 data = json.dumps(payload)
-        except:
-            return Error(message="Serialization to JSON failed",
-                exc=exc_info(), method=func, payload=payload)
+            except:
+                return Error(message="Serialization to JSON failed",
+                    exc=exc_info(), method=func, payload=payload)
 
-        self.headers = {
-            "Content-Type": "application/json",
-            "Accept": "application/json",
-            "Content-Length": str(len(data))
-        }
+            self.headers.update({
+                "Content-Type": "application/json",
+                "Content-Length": str(len(data))
+            })
 
         # HTTP(S)Connection is oneshot object (and we don't speak "pipelining")
         conn = self.connect()
@@ -384,7 +450,7 @@ class Client(object):
 
         loc = '%s/%s%s' % (self.url.path, func, argurl)
         try:
-            conn.request("POST", loc, data, self.headers)
+            conn.request(method, loc, data, self.headers)
         except:
             conn.close()
             return Error(message="Sending of request to server failed",
@@ -408,13 +474,13 @@ class Client(object):
 
         if res.status==httplib.OK:
             try:
-                data = json.loads(response_data)
+                data = json.loads(response_data.decode("utf-8"))
             except:
                 data = Error(method=func, message="JSON message parsing failed",
                     exc=exc_info(), response=response_data)
         else:
             try:
-                data = json.loads(response_data)
+                data = json.loads(response_data.decode("utf-8"))
                 data["errors"]   # trigger exception if not dict or no error key
             except:
                 data = Error(method=func, message="Generic server HTTP error",
@@ -467,7 +533,7 @@ class Client(object):
         else:
             try:
                 self.send_events_limit = min(res["send_events_limit"], self.send_events_limit)
-                self.recv_events_limit = min(res["recv_events_limit"], self.recv_events_limit)
+                self.get_events_limit = min(res["get_events_limit"], self.get_events_limit)
             except (AttributeError, TypeError, KeyError):
                 pass
         return res
@@ -506,7 +572,7 @@ class Client(object):
         """ Send out "events" list to server, retrying on server errors.
         """
         ev = events
-        idx_xlat = range(len(ev))
+        idx_xlat = list(range(len(ev)))
         err = Error()
         retry = retry or self.retry
         attempt = retry
@@ -524,7 +590,7 @@ class Client(object):
                 res.errors.sort(key=itemgetter("error"))
                 for e in res.errors:
                     errno = e["error"]
-                    evlist = e.get("events", range(len(ev)))   # none means all
+                    evlist = e.get("events", list(range(len(ev))))   # none means all
                     if errno < 500 or not attempt:
                         # Fatal error or last try, translate indices
                         # to original and prepare for returning to caller
@@ -547,11 +613,11 @@ class Client(object):
             tag=None, notag=None,
             group=None, nogroup=None):
 
-        if not id:
+        if id is None:
             id = self._loadID(idstore)
 
         res = self.sendRequest(
-            "getEvents", id=id, count=count or self.recv_events_limit, cat=cat,
+            "getEvents", id=id, count=self.get_events_limit if count is None else count, cat=cat,
             nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
 
         if res:
@@ -589,7 +655,7 @@ def format_timestamp(epoch=None, utc=True, utcoffset=None):
     return format_time(*time.gmtime(epoch)[:6], microsec=us, utcoffset=utcoffset)
 
 
-def format_time(year, month, day, hour, minute, second, microsec=0, utcoffset=0):
+def format_time(year, month, day, hour, minute, second, microsec=0, utcoffset=None):
     if utcoffset is None:
         utcoffset = -(time.altzone if time.daylight else time.timezone)
     tstr = "%04d-%02d-%02dT%02d:%02d:%02d" % (year, month, day, hour, minute, second)
@@ -599,7 +665,6 @@ def format_time(year, month, day, hour, minute, second, microsec=0, utcoffset=0)
 
 
 def read_cfg(cfgfile):
-    abspath = path.join(path.dirname(__file__), cfgfile)
-    with open(abspath, "r") as f:
+    with open(cfgfile, "r") as f:
         stripcomments = "\n".join((l for l in f if not l.lstrip().startswith(("#", "//"))))
         return json.loads(stripcomments)

warden3/warden_client/warden_client_test.py → warden_client/warden_client_examples.py

 #!/usr/bin/python
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2011-2013 Cesnet z.s.p.o
+# Copyright (C) 2011-2015 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-from warden_client import Client, Error, read_cfg
-import json
 import string
-from time import time, gmtime
-from math import trunc
+from time import time
 from uuid import uuid4
 from pprint import pprint
-from os import path
-from random import randint, randrange, choice, random;
-from base64 import b64encode;
-
-def get_precise_timestamp():
-    t = time()
-    us = trunc((t-trunc(t))*1000000)
-    g = gmtime(t)
-    iso = '%04d-%02d-%02dT%02d:%02d:%02d.%0dZ' % (g[0:6]+(us,))
-    return iso
+from random import randint, randrange, choice, random
+from base64 import b64encode
+from warden_client import Client, Error, read_cfg, format_timestamp
 
 def gen_min_idea():
 
     return {
        "Format": "IDEA0",
        "ID": str(uuid4()),
-       "DetectTime": get_precise_timestamp(),
+       "DetectTime": format_timestamp(),
        "Category": ["Test"],
     }
 
@@ -63,18 +53,18 @@ def gen_random_idea(client_name="cz.example.warden.test"):
     def randip6():
         return [rand6ip, geniprange(rand6ip), rand6cidr][randint(0, 2)]()
 
-    def randstr(charlist=string.letters, maxlen=32, minlen=1):
+    def randstr(charlist=string.ascii_letters, maxlen=32, minlen=1):
         return ''.join(choice(charlist) for i in range(randint(minlen, maxlen)))
 
     event = {
        "Format": "IDEA0",
        "ID": str(uuid4()),
-       "CreateTime": get_precise_timestamp(),
-       "DetectTime": get_precise_timestamp(),
-       "WinStartTime": get_precise_timestamp(),
-       "WinEndTime": get_precise_timestamp(),
-       "EventTime": get_precise_timestamp(),
-       "CeaseTime": get_precise_timestamp(),
+       "CreateTime": format_timestamp(),
+       "DetectTime": format_timestamp(),
+       "WinStartTime": format_timestamp(),
+       "WinEndTime": format_timestamp(),
+       "EventTime": format_timestamp(),
+       "CeaseTime": format_timestamp(),
        #"Category": ["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"],
        # "Category": ["Abusive.Spam","Fraud.Copyright"],
        "Category": [choice(["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"]) for dummy in range(randint(1, 3))],
@@ -113,22 +103,25 @@ def gen_random_idea(client_name="cz.example.warden.test"):
              "Size": 46,
              "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())],
              "ContentEncoding": "base64",
-             "Content": b64encode(randstr())
+             "Content": b64encode(randstr().encode('ascii')).decode("ascii")
           }
        ],
        "Node": [
           {
              "Name": client_name,
-             "Tags": [choice(["Data", "Protocol", "Honeypot", "Heuristic", "Log"]) for dummy in range(randint(1, 3))],
+             "Type": [choice(["Data", "Protocol", "Honeypot", "Heuristic", "Log"]) for dummy in range(randint(1, 3))],
              "SW": ["Kippo"],
              "AggrWin": "00:05:00"
+          },
+          {
+             "Name": "org.example.warden.client",
+             "Type": [choice(["Connection", "Datagram"]) for dummy in range(randint(1, 2))],
           }
        ]
     }
 
     return event
 
-
 def main():
     wclient = Client(**read_cfg("warden_client.cfg"))
     # Also inline arguments are possible:
@@ -143,44 +136,53 @@ def main():
     #     idstore="MyClient.id",
     #     name="cz.example.warden.test")
 
-    print "=== Debug ==="
+    print("=== Debug ===")
     info = wclient.getDebug()
-    if not isinstance(info, Error):
-        pprint(info)
-
-    print "=== Server info ==="
+    pprint(info)
+
+    # All methods return something.
+    # If you want to catch possible errors (for example implement some
+    # form of persistent retry, or save failed events for later, you may
+    # check for Error instance and act based on contained info.
+    # If you want just to be informed, this is not necessary, just
+    # configure logging correctly and check logs.
+    if isinstance(info, Error):
+        print(info)
+
+    print("=== Server info ===")
     info = wclient.getInfo()
-    if not isinstance(info, Error):
-        pprint(info)
 
-    print "=== Sending 10 event(s) ==="
+    print("=== Sending 10 event(s) ===")
     start = time()
     ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)])
-    print ret
-    print "Time: %f" % (time()-start)
+    print(ret)
+    print("Time: %f" % (time()-start))
 
-    print "=== Getting 10 events ==="
+    print("=== Getting 10 events ===")
     start = time()
 
     # cat = ['Availability', 'Abusive.Spam','Attempt.Login']
     # cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus']
-    cat = ['Fraud', 'Abusive.Spam']
-    nocat = ['Availability', 'Information', 'Fraud.Scam']
-
-    tag = ['Log', 'Data']
-    notag = ['Flow', 'Datagram']
-
-    group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
-    nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
-
-    ret = wclient.getEvents(count=10, cat=cat, nocat=None, tag=tag, notag=None, group=None, nogroup=nogroup)
-    print "Time: %f" % (time()-start)
-    print "Got %i events" % len(ret)
+    # cat = ['Fraud', 'Abusive.Spam']
+    # nocat = ['Availability', 'Information', 'Fraud.Scam']
+    cat = []
+    nocat = []
+
+    #tag = ['Log', 'Data']
+    #notag = ['Flow', 'Datagram']
+    tag = []
+    notag = []
+
+    #group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
+    #nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
+    group = []
+    nogroup = []
+
+    ret = wclient.getEvents(count=10, cat=cat, nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
+    print("Time: %f" % (time()-start))
+    print("Got %i events" % len(ret))
     for e in ret:
-        print e["Category"], e["Node"][0]["Tags"], e["Node"][0]["Name"]
-    if isinstance(ret, Error):
-        print ret
-
+        print(e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name"))
 
 if __name__ == "__main__":
     main()

warden3/warden_client/warden_curl_test.sh → warden_client/warden_curl_test.sh

 #!/bin/sh
 #
-# Copyright (C) 2011-2013 Cesnet z.s.p.o
+# Copyright (C) 2011-2015 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-keyfile='key.pem'
-certfile='cert.pem'
-cafile='tcs-ca-bundle.pem'
+if [ "$#" -ne 6 ]; then
+    echo "Run me like:"
+    echo "${0##*/} 'https://warden-hub.example.org/warden3' org.example.warden.client 'ToPsEcReT' key.pem cert.pem tcs-ca-bundle.pem"
+    exit 1
+fi
+
+
 url="$1"
 client="$2"
 secret="$3"
-
-#    --fail \
-#    --show-error \
-#
+keyfile="$4"
+certfile="$5"
+cafile="$6"
 
 echo "Test  404"
 curl \
@@ -20,7 +23,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/blefub?client=$client&secret=$secret"
 echo
 
@@ -30,7 +32,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/?client=$client&secret=$secret"
 echo
 
@@ -40,7 +41,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client"
 echo
 
@@ -50,7 +50,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents"
 echo
 
@@ -60,7 +59,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=asdf.blefub"
 echo
 
@@ -70,7 +68,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=asdf.blefub&secret=$secret"
 echo
 
@@ -80,7 +77,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=ASDFblefub"
 echo
 
@@ -90,7 +86,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?secret=$secret"
 echo
 
@@ -111,7 +106,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&cat=bflm"
 echo
 
@@ -121,7 +115,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&cat=Other&nocat=Test"
 echo
 
@@ -142,7 +135,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&self=test"
 echo
 
@@ -152,7 +144,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&bad=guy"
 echo
 
@@ -162,7 +153,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret"
 echo
 
@@ -172,7 +162,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&count=3&id=10"
 echo
 
@@ -182,7 +171,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getDebug?client=$client&secret=$secret"
 echo
 
@@ -192,12 +180,5 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getInfo?client=$client&secret=$secret"
 echo
-
-#curl \
-#    --fail \
-#    --connect-timeout 3 \
-#    --request POST \
-#    $url/getEvents

warden3/contrib/warden_filer/README → warden_filer/README

-+---------------------------------+
-| Warden Filer 0.1 for Warden 3.X |
-+---------------------------------+
++---------------------------------------+
+| Warden Filer 3.0-beta3 for Warden 3.X |
++---------------------------------------+
 
 Content
 
@@ -49,15 +49,21 @@ C. Usage
      -c CONFIG, --config CONFIG
                            configuration file path
      --oneshot             don't daemonise, run just once
+     -d, --daemon          daemonize
+     -p PID_FILE, --pid_file PID_FILE
+                           create PID file with this name
 
 
    CONFIG denotes path to configuration file, default is warden_filer.cfg in
 current directory.
-   --oneshot prevents daemonizing, Filer just does its work once (fetches
-available events or sends event files present in directory), but obeys
-all other applicable options from configuration file (concerning logging,
-filtering, directories, etc.)
-   Without --oneshot Filer goes to full unix daemon mode.
+   --oneshot instructs Filer to just do its work once (fetch available events
+or send event files present in directory), but obeys all other applicable
+options from configuration file (concerning logging, filtering, directories,
+etc.)
+   --daemon instructs Filer to go to full unix daemon mode. Without it,
+Filer just stays on foreground.
+   --pid_file makes Filer to create the usual PID file. Without it, no PID
+file gets created.
 
 ------------------------------------------------------------------------------
 D. Configuration
@@ -71,12 +77,25 @@ JSON object, containing configuration. See also warden_filer.cfg as example.
    sender - configuration section for sender mode
       dir - directory, whose "incoming" subdir will be checked for Idea
             events to send out
+      done_dir - directory, into which the messages will be moved after
+            successful sending. If not set, processed messages will get
+            deleted, which is default, and usually what you want. Note that
+            this is just regular directory, no special locking precautions
+            and no subdirectories are done here, however if "done_dir" is on
+            the same filesystem as "dir"
       filter - filter fields (same as in Warden query, see Warden and Idea
-               doc, possible keys: cat, nocat, group, nogroup, tag, notag),
-               unmatched events get discarded and deleted
+            doc, possible keys: cat, nocat, group, nogroup, tag, notag),
+            unmatched events get discarded and deleted
       node - o information about detector to be prepended into event Node
-             array (see Idea doc). Note that Warden server may require it
-             to correspond with client registration
+            array (see Idea doc). Note that Warden server may require it to
+            correspond with client registration
+      poll_time - how often to check incoming directory (in seconds, defaults
+            to 5)
+      owait_timeout - how long to opportunistically wait for possible new
+            incoming files when number of files to process is less than
+            send_events_limit (in seconds, defaults to poll_time)
+      owait_poll_time - how often to check incoming directory during
+            opportunistic timeout (in seconds, defaults to 1)
    receiver - configuration section for receiver mode
       dir - directory, whose "incoming" subdir will serve as target for events
       filter - filter fields for Warden query (see Warden and Idea doc,
@@ -85,6 +104,21 @@ JSON object, containing configuration. See also warden_filer.cfg as example.
              array (see Idea doc). Be careful here, you may ruin Idea
              messages by wrongly formatted data and they are not checked
              here in any way
+      poll_time - how often to check Warden server for new events (in seconds,
+             defaults to 5)
+      file_limit - limit number of files in "incoming" directory. When the limit
+             is reached, polling is paused for "limit_wait_time" seconds
+      limit_wait_time - wait this number of seconds if limit on number of files
+             is reached (defaults to 5)
+      
+
+    Both the "sender" and "reciever" sections can also bear daemon
+configuration.
+
+      work_dir - where should daemon chdir
+      chroot_dir - confine daemon into chroot directory
+      umask - explicitly set umask for created files
+      uid, gid - uid/gid, under which daemon will run
 
 ------------------------------------------------------------------------------
 E. Directories and locking issues
@@ -95,39 +129,45 @@ confusion. Simple path suffers locking issue: when one process saves file
 there, another process has no way to know whether file is already complete
 or not, and starting to read prematurely can lead to corrupted data read.
 Also, two concurrent processes may decide to work on one file, stomping on
-others legs. So, your scripts and tools inserting data or taking data from
-working directories must obey simple protocols.
+others legs.
+   So, your scripts and tools inserting data or taking data from working
+directories must obey simple protocols, which use atomic "rename" to avoid
+locking issues.
+   Also, your directory (and its structure) _must_ reside on the same
+filesystem to keep "rename" atomic. _Never_ try to mount some of the
+subdirectories ("tmp", "incoming", "errors") from other filesystem.
 
  1. Inserting file
 
-   * Use "temp" subdirectory to create new file; filename is arbitrary, but
-     must be unique among all subdirectories. 
+   * The file you want to create _must_ be created in the "tmp" subdirectory
+     first, _not_ "incoming". Filename is arbitrary, but must be unique among
+     all subdirectories.
 
    * When done writing, rename the file into "incoming" subdir. Rename is
      atomic operation, so for readers, file will appear either nonexistent
      or complete.
 
    For simple usage (bash scripts, etc.), just creating sufficiently random
-   filename in "temp" and then moving into "incoming" may be enough.
+   filename in "tmp" and then moving into "incoming" may be enough.
    Concatenating $RANDOM couple of times will do. :)
    
    For advanced or potentially concurrent usage inserting enough of unique
    information into name is recommended - Filer itself uses hostname, pid,
    unixtime, milliseconds, device number and file inode number to avoid
    locking issues both on local and network based filesystems and to be
-   prepared for hight traffic.
+   prepared for high traffic.
 
  2. Picking up file
 
-   * Rename the file to work with into "temp" directory.
+   * Rename the file to work with into "tmp" directory.
 
    * Do whatever you want with contents, and when finished, rename file back
      into "incoming", or remove, or move somewhere else, or move into "errors"
-     directory, after all, it's your file.
+     directory - what suits your needs, after all, it's your file.
 
    Note that in concurrent environment file can disappear between directory
    enumeration and attempt to rename - then just pick another one (and
-   repeat), someone was swifter.)
+   possibly repeat), someone was swifter.
 
 ------------------------------------------------------------------------------
 Copyright (C) 2011-2015 Cesnet z.s.p.o

warden_filer/check_file_count

+#!/bin/bash
+
+script=${0##*/}
+warn=0
+crit=65536
+read -rd '' helps <<EOF
+$script: Icinga plugin to check too high number of files in directory.
+
+Usage: $script -d dir [-w num] [-c num] [-h ]
+
+  -d dir    directory to watch
+  -w num    warning if number of files exceeds this value (default $warn)
+  -c num    critical if number of files exceeds this value (default $crit)
+EOF
+
+function bailout {
+	echo -n "$script" | tr '[:lower:]' '[:upper:]'
+	echo " $2 $3"
+	exit "$1"
+}
+
+while getopts hvVd:w:c: opt; do
+	case "$opt" in
+		h) bailout 3 "UNKNOWN" "$helps";;
+		d) dir="$OPTARG";;
+		w) warn="$OPTARG";;
+		c) crit="$OPTARG";;
+		"?") bailout 3 "UNKNOWN" "Unknown option, use -h for help";;
+	esac
+done
+
+[ -z "$dir" ] && bailout 3 "UNKNOWN" "-d not specified"
+
+count=$(find "$dir" -mindepth 1 -maxdepth 1 | wc -l)
+
+[ "$count" -gt "$crit" ] && bailout 2 "CRIT" "$count"
+
+[ "$count" -gt "$warn" ] && bailout 1 "WARN" "$count"
+
+bailout 0 "OK" "$count"

warden3/contrib/warden_filer/warden_filer.cfg → warden_filer/warden_filer.cfg.dist

+// For all options see documentation
 {
     // Warden config can be also referenced as:
     // "warden": "/path/to/warden_client.cfg"
     "warden": {
         "url": "https://example.com/warden3",
         "cafile": "tcs-ca-bundle.pem",
-        "timeout": 10,
-        "errlog": {"level": "debug"},
+        "keyfile": "my.key.pem",
+        "certfile": "my.cert.pem",
+        "timeout": 60,
+        "retry": 20,
+        "pause": 5,
         "filelog": {"level": "debug"},
-        "idstore": "myclient.id",
         "name": "com.example.warden.test",
         "secret": "SeCrEt"
     },
@@ -16,42 +19,38 @@
         // for Idea events to send out
         "dir": "warden_sender",
         // Optional filter fields, unmatched events are discarded (and removed)
-        "filter": {
-            "cat": ["Test", "Recon.Scanning"],
-            "nocat": null,
-            "group": ["cz.example"],
-            "nogroup": null,
-            "tag": null,
-            "notag": ["Honeypot"]
-        },
+        //"filter": {
+        //    "cat": ["Test", "Recon.Scanning"],
+        //    "nocat": null,
+        //    "group": ["cz.example"],
+        //    "nogroup": null,
+        //    "tag": null,
+        //    "notag": ["Honeypot"]
+        //},
         // Optional information about detector to be prepended into Idea Node array
-        "node": {
-            "Name": "cz.example.warden.test_sender",
-            "Type": ["Relay"],
-            "SW": ["warden_filer-sender"],
-            "AggrWin": "00:05:00",
-            "Note": "Test warden_filer sender"
-        }
+        //"node": {
+        //    "Name": "cz.example.warden.test_sender",
+        //    "Type": ["Relay"]
+        //}
     },
     "receiver": {
         // Maildir like directory, whose "incoming" will serve as target for events
         "dir": "warden_receiver",
         // Optional filter fields for Warden query
-        "filter": {
-            "cat": ["Test", "Recon.Scanning"],
-            "nocat": null,
-            "group": ["cz.cesnet"],
-            "nogroup": null,
-            "tag": null,
-            "notag": ["Honeypot"]
-        },
+        //"filter": {
+        //    "cat": ["Test", "Recon.Scanning"],
+        //    "nocat": null,
+        //    "group": ["cz.cesnet"],
+        //    "nogroup": null,
+        //    "tag": null,
+        //    "notag": ["Honeypot"]
+        //},
         // Optional information about detector to be prepended into Idea Node array
-        "node": {
-            "Name": "cz.example.warden.test_receiver",
-            "Type": ["Relay"],
-            "SW": ["warden_filer-receiver"],
-            "AggrWin": "00:05:00",
-            "Note": "Test warden_filer receiver"
-        }
+        //"node": {
+        //    "Name": "cz.example.warden.test_receiver",
+        //    "Type": ["Relay"]
+        //},
+        // Optional limit on number of files in "incoming" directory
+        //"file_limit": 10000
     }
 }

warden_filer/warden_filer_logrotate.dist

+# You may want to review and/or change the logfile path, user/group in
+# 'create' and daemon to restart in 'postrotate'
+
+/var/log/warden_filer.log
+{
+	rotate 52
+	weekly
+	missingok
+	notifempty
+	compress
+	delaycompress
+	dateext
+	create 640 mentat mentat
+	postrotate
+		/etc/init.d/warden_filer_sender restart
+		/etc/init.d/warden_filer_receiver restart
+	endscript
+}

warden3/contrib/warden_filer/warden_filer_receiver → warden_filer/warden_filer_receiver

-#!/bin/sh
+#!/bin/bash
 #
 ### BEGIN INIT INFO
 # Provides:          warden_filer_receiver
@@ -18,15 +18,27 @@ SERVICE_NAME="${DAEMON_NAME}_${FUNC}"
 PID=/var/run/"$DAEMON_NAME"/"$FUNC".pid
 CONFIG=/etc/"$DAEMON_NAME".cfg
 
-test -f /etc/default/"$SERVICE_NAME" && . /etc/default/"$SERVICE_NAME"
-. /lib/lsb/init-functions
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$SERVICE_NAME" ] && . /etc/$n/"$SERVICE_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
 
 ACTION="$1"
 
 case "$ACTION" in
 	start)
+		mkdir -p "${PID%/*}"
 		log_daemon_msg "Starting $SERVICE_NAME" "$SERVICE_NAME"
-		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" "$FUNC"
+		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" --pid_file "$PID" --daemon "$FUNC"
 		log_end_msg $?
 		;;
 	stop)

warden3/contrib/warden_filer/warden_filer_sender → warden_filer/warden_filer_sender

-#!/bin/sh
+#!/bin/bash
 #
 ### BEGIN INIT INFO
 # Provides:          warden_filer_sender
@@ -18,15 +18,27 @@ SERVICE_NAME="${DAEMON_NAME}_${FUNC}"
 PID=/var/run/"$DAEMON_NAME"/"$FUNC".pid
 CONFIG=/etc/"$DAEMON_NAME".cfg
 
-test -f /etc/default/"$SERVICE_NAME" && . /etc/default/"$SERVICE_NAME"
-. /lib/lsb/init-functions
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$SERVICE_NAME" ] && . /etc/$n/"$SERVICE_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
 
 ACTION="$1"
 
 case "$ACTION" in
 	start)
+		mkdir -p "${PID%/*}"
 		log_daemon_msg "Starting $SERVICE_NAME" "$SERVICE_NAME"
-		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" "$FUNC"
+		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" --pid_file "$PID" --daemon "$FUNC"
 		log_end_msg $?
 		;;
 	stop)

warden_ra/LICENSE

+Copyright (c) 2011-2016 Cesnet z.s.p.o <warden-info@cesnet.cz>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\ No newline at end of file