contrib/map/frontend/css/warden-map.css

+/*
+ *
+ * -*- coding: utf-8 -*-
+ *
+ * warden-map.css
+ *
+ * Copyright (C) 2016 Cesnet z.s.p.o
+ * Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+ *
+*/
+
+body {
+  font-family: 'Oswald', sans-serif;
+  background: #00253D;
+  border: 0px;
+  padding: 0px;
+  margin: 0px;
+}
+
+h2 {
+ color: #0062a2;
+}
+
+.hoverinfo {
+  font-family: 'Oswald', sans-serif;
+}
+
+#country {
+  color: #0062a2;	/* Cesnet blue */
+  font-weight: bold;
+}
+
+
+table {
+  text-align: left;
+  margin: 0;
+  padding: 0;
+  font-size: 12px;
+}
+
+table th {
+  color: #0062a2;	/* Cesnet blue */
+  padding: 0;
+}
+
+table td {
+  color: #4b4d4a;	/* Greenish gray */
+  padding: 0;
+}
+
+#container {
+  overflow: hidden;
+/*  border: 2px solid #0062a2;
+  border: 0px;
+  padding: 0px;
+  margin: 0px;
+  border-radius: 5px;*/
+  position: relative;
+/*  width: 1280px;
+  height: 720px;*/
+  max-width: 100%;
+  max-height: 100%
+  width: 100%;
+  height: 100vh;*/
+}
+
+.zoom-button {
+  width: 40px;
+  height: 40px;
+  border-radius: 5px;
+  border: none;
+  background: #dcdcda;
+  font-size: 23px;
+  font-weight: bold;
+  color: white;
+  cursor: pointer;
+}
+
+.zoom-button:hover {
+  background-color: #0062a2;
+}
+
+#zoom-info {
+  display: inline-block;
+  padding: 10px;
+  color: #0062a2;
+}
+
+#warden-logo {
+  position: absolute;
+  top: 30px;
+  left: 30px;
+  background: white;
+  padding: 10px;
+  border-radius: 10px;
+  width: 240px;
+  height: 92px;
+  text-align: center;
+}
+
+#cesnet-logo {
+  position: absolute;
+  top: 30px;
+  right: 30px;
+  background: white;
+  padding: 10px;
+  border-radius: 10px;
+  width: 240px;
+  height: 92px;
+  text-align: center;
+}
+
+#legend-box {
+  position: absolute;
+  bottom: 30px;
+  left: 30px;
+  background-color: rgba(0,0,0,0.3);
+  color: white;
+  padding: 10px;
+  border-radius: 10px;
+  /*width: 240px;
+  height: 92px;
+  text-align: center;*/
+}
+
+#heading {
+  position: absolute;
+  top: 30px;
+  left: 50%;
+  width: 40em;
+  height: 92px;
+  margin-left: -20em;
+  font-size: xx-large;
+  color: white;
+  text-align: center;
+  vertical-align: middle;
+  line-height: 92px;
+}

contrib/map/frontend/js/warden-map.js

+/*
+ *
+ * -*- coding: utf-8 -*-
+ *
+ * warden-map.js
+ *
+ * Copyright (C) 2016 Cesnet z.s.p.o
+ * Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+ *
+*/
+
+// NOTE: Change path in a function d3.json() if you separate backend and frontend!
+
+// Zooming functionality is based on WunderBart's implementation
+// Please see following links:
+// https://github.com/wunderbart
+// https://jsfiddle.net/wunderbart/Lom3b0gb/
+
+  function Zoom(args) {
+  $.extend(this, {
+    $buttons:   $(".zoom-button"),
+    $info:      $("#zoom-info"),
+    scale:      { max: 50, currentShift: 0 },
+    $container: args.$container,
+    datamap:    args.datamap
+  });
+
+  this.init();
+}
+
+Zoom.prototype.init = function() {
+  var paths = this.datamap.svg.selectAll("path"),
+      subunits = this.datamap.svg.selectAll(".datamaps-subunit");
+
+  // preserve stroke thickness
+  paths.style("vector-effect", "non-scaling-stroke");
+
+  // disable click on drag end
+  subunits.call(
+    d3.behavior.drag().on("dragend", function() {
+      d3.event.sourceEvent.stopPropagation();
+    })
+  );
+
+  this.scale.set = this._getScalesArray();
+  this.d3Zoom = d3.behavior.zoom().scaleExtent([ 1, this.scale.max ]);
+
+  this._displayPercentage(1);
+  this.listen();
+};
+
+Zoom.prototype.listen = function() {
+  this.$buttons.off("click").on("click", this._handleClick.bind(this));
+
+  this.datamap.svg
+    .call(this.d3Zoom.on("zoom", this._handleScroll.bind(this)))
+    .on("dblclick.zoom", null); // disable zoom on double-click
+};
+
+Zoom.prototype.reset = function() {
+  this._shift("reset");
+};
+
+Zoom.prototype._handleScroll = function() {
+  var translate = d3.event.translate,
+      scale = d3.event.scale,
+      limited = this._bound(translate, scale);
+
+  this.scrolled = true;
+
+  this._update(limited.translate, limited.scale);
+};
+
+Zoom.prototype._handleClick = function(event) {
+  var direction = $(event.target).data("zoom");
+
+  this._shift(direction);
+};
+
+Zoom.prototype._shift = function(direction) {
+  var center = [ this.$container.width() / 2, this.$container.height() / 2 ],
+      translate = this.d3Zoom.translate(), translate0 = [], l = [],
+      view = {
+        x: translate[0],
+        y: translate[1],
+        k: this.d3Zoom.scale()
+      }, bounded;
+
+  translate0 = [
+    (center[0] - view.x) / view.k,
+    (center[1] - view.y) / view.k
+  ];
+
+  if (direction == "reset") {
+    view.k = 1;
+    this.scrolled = true;
+  } else {
+    view.k = this._getNextScale(direction);
+  }
+
+l = [ translate0[0] * view.k + view.x, translate0[1] * view.k + view.y ];
+
+  view.x += center[0] - l[0];
+  view.y += center[1] - l[1];
+
+  bounded = this._bound([ view.x, view.y ], view.k);
+
+  this._animate(bounded.translate, bounded.scale);
+};
+
+Zoom.prototype._bound = function(translate, scale) {
+  var width = this.$container.width(),
+      height = this.$container.height();
+
+  translate[0] = Math.min(
+    (width / height)  * (scale - 1),
+    Math.max( width * (1 - scale), translate[0] )
+  );
+
+  translate[1] = Math.min(0, Math.max(height * (1 - scale), translate[1]));
+
+  return { translate: translate, scale: scale };
+};
+
+Zoom.prototype._update = function(translate, scale) {
+  this.d3Zoom
+    .translate(translate)
+    .scale(scale);
+
+  this.datamap.svg.selectAll("g")
+    .attr("transform", "translate(" + translate + ")scale(" + scale + ")");
+
+  this._displayPercentage(scale);
+};
+
+Zoom.prototype._animate = function(translate, scale) {
+  var _this = this,
+      d3Zoom = this.d3Zoom;
+
+  d3.transition().duration(350).tween("zoom", function() {
+    var iTranslate = d3.interpolate(d3Zoom.translate(), translate),
+        iScale = d3.interpolate(d3Zoom.scale(), scale);
+
+    return function(t) {
+      _this._update(iTranslate(t), iScale(t));
+    };
+  });
+};
+
+Zoom.prototype._displayPercentage = function(scale) {
+  var value;
+
+  value = Math.round(Math.log(scale) / Math.log(this.scale.max) * 100);
+  this.$info.text(value + "%");
+};
+
+Zoom.prototype._getScalesArray = function() {
+  var array = [],
+      scaleMaxLog = Math.log(this.scale.max);
+
+  for (var i = 0; i <= 10; i++) {
+    array.push(Math.pow(Math.E, 0.1 * i * scaleMaxLog));
+  }
+
+  return array;
+};
+
+Zoom.prototype._getNextScale = function(direction) {
+  var scaleSet = this.scale.set,
+      currentScale = this.d3Zoom.scale(),
+      lastShift = scaleSet.length - 1,
+      shift, temp = [];
+
+  if (this.scrolled) {
+
+    for (shift = 0; shift <= lastShift; shift++) {
+      temp.push(Math.abs(scaleSet[shift] - currentScale));
+    }
+
+    shift = temp.indexOf(Math.min.apply(null, temp));
+
+    if (currentScale >= scaleSet[shift] && shift < lastShift) {
+      shift++;
+    }
+
+    if (direction == "out" && shift > 0) {
+      shift--;
+    }
+
+    this.scrolled = false;
+
+  } else {
+
+    shift = this.scale.currentShift;
+
+    if (direction == "out") {
+      shift > 0 && shift--;
+    } else {
+      shift < lastShift && shift++;
+    }
+  }
+
+  this.scale.currentShift = shift;
+
+  return scaleSet[shift];
+};
+
+
+ function defaults(obj) {
+    Array.prototype.slice.call(arguments, 1).forEach(function(source) {
+      if (source) {
+        for (var prop in source) {
+          // Deep copy if property not set
+          if (obj[prop] == null) {
+            if (typeof source[prop] == 'function') {
+              obj[prop] = source[prop];
+            }
+            else {
+              obj[prop] = JSON.parse(JSON.stringify(source[prop]));
+            }
+          }
+        }
+      }
+    });
+    return obj;
+  }
+
+function val( datumValue, optionsValue, context ) {
+    if ( typeof context === 'undefined' ) {
+      context = optionsValue;
+      optionsValues = undefined;
+    }
+    var value = typeof datumValue !== 'undefined' ? datumValue : optionsValue;
+
+    if (typeof value === 'undefined') {
+      return  null;
+    }
+
+    if ( typeof value === 'function' ) {
+      var fnContext = [context];
+      if ( context.geography ) {
+        fnContext = [context.geography, context.data];
+      }
+      return value.apply(null, fnContext);
+    }
+    else {
+      return value;
+    }
+  }
+  
+var cat_color = {
+  "Abusive": "MediumPurple",
+  "Malware": "Red",
+  "Recon": "LightSlateGray",
+  "Attempt": "GhostWhite",
+  "Intrusion": "DarkTurquoise",
+  "Availability": "HotPink",
+  "Information": "PaleTurquoise",
+  "Fraud": "Yellow",
+  "Vulnerable": "DarkGoldenRod",
+  "Anomaly": "Brown",
+  "Other": "Green"
+}
+
+var cat_desc = {
+  "Abusive": "spam",
+  "Malware": "virus, worm, trojan, malware",
+  "Recon": "scanning, sniffing",
+  "Attempt": "bruteforce, exploitation attempt",
+  "Intrusion": "botnet, successful exploit",
+  "Availability": "(D)DOS",
+  "Information": "wiretapping, spoofing, hijacking",
+  "Fraud": "phishing, scam",
+  "Vulnerable": "open for abuse",
+  "Anomaly": "unusual traffic",
+  "Other": "unknown/unidentified"
+}
+
+ function handleArcs (layer, data, options) {
+    var self = this,
+        svg = this.svg;
+
+    if ( !data || (data && !data.slice) ) {
+      throw "Datamaps Error - arcs must be an array";
+    }
+
+    // For some reason arc options were put in an `options` object instead of the parent arc
+    // I don't like this, so to match bubbles and other plugins I'm moving it
+    // This is to keep backwards compatability
+    for ( var i = 0; i < data.length; i++ ) {
+      data[i] = defaults(data[i], data[i].options);
+      delete data[i].options;
+    }
+
+    if ( typeof options === "undefined" ) {
+      options = defaultOptions.arcConfig;
+    }
+
+    var arcs = layer.selectAll('path.datamaps-arc').data( data, JSON.stringify );
+
+    var path = d3.geo.path()
+        .projection(self.projection);
+
+    arcs
+      .enter()
+        .append('svg:path')
+        .attr('class', 'datamaps-arc')
+        .style('stroke-linecap', 'round')
+        .style('stroke', function(datum) {
+/*          return val(datum.strokeColor, options.strokeColor, datum);*/
+          for (cat in cat_color) {
+            if (datum.event.startsWith(cat)) {
+              return cat_color[cat];
+            }
+          }
+          return "Green";
+        })
+        .style('fill', 'none')
+        .style('stroke-width', function(datum) {
+            return val(datum.strokeWidth, options.strokeWidth, datum);
+        })
+        .attr('d', function(datum) {
+
+            var originXY, destXY;
+
+            originXY = self.latLngToXY(val(datum.origin.latitude, datum), val(datum.origin.longitude, datum))
+
+            destXY = self.latLngToXY(val(datum.destination.latitude, datum), val(datum.destination.longitude, datum));
+
+            var midXY = [ (originXY[0] + destXY[0]) / 2, (originXY[1] + destXY[1]) / 2];
+            if (options.greatArc) {
+                  // TODO: Move this to inside `if` clause when setting attr `d`
+              var greatArc = d3.geo.greatArc()
+                  .source(function(d) { return [val(d.origin.longitude, d), val(d.origin.latitude, d)]; })
+                  .target(function(d) { return [val(d.destination.longitude, d), val(d.destination.latitude, d)]; });
+
+              return path(greatArc(datum))
+            }
+            var sharpness = val(datum.arcSharpness, options.arcSharpness, datum);
+            return "M" + originXY[0] + ',' + originXY[1] + "S" + (midXY[0] + (50 * sharpness)) + "," + (midXY[1] - (75 * sharpness)) + "," + destXY[0] + "," + destXY[1];
+        })
+        .attr('data-info', function(datum) {
+          return JSON.stringify(datum);
+        })
+        .on('mouseover', function ( datum ) {
+          var $this = d3.select(this);
+
+          if (options.popupOnHover) {
+            self.updatePopup($this, datum, options, svg);
+          }
+        })
+        .on('mouseout', function ( datum ) {
+          var $this = d3.select(this);
+
+          d3.selectAll('.datamaps-hoverover').style('display', 'none');
+        })
+        .transition()
+          .style('fill', function(datum, i) {
+            /*
+              Thank you Jake Archibald, this is awesome.
+              Source: http://jakearchibald.com/2013/animated-line-drawing-svg/
+            */
+            var length = this.getTotalLength();
+            this.style.transition = this.style.WebkitTransition = 'none';
+            this.style.strokeDasharray = length + ' ' + length;
+            this.style.strokeDashoffset = length;
+            this.getBoundingClientRect();
+            this.style.transition = this.style.WebkitTransition = 'stroke-dashoffset ' + val(datum.animationSpeed, options.animationSpeed, datum) + 'ms ' + datum.delay*1000 + 'ms ease-out';
+            this.style.strokeDashoffset = '0';
+            return 'none';
+          });
+
+    arcs.exit()
+      .transition()
+      .duration(1000)
+      .style('opacity', 0)
+      .remove();
+  }
+
+var main_data = [];
+var prev_data = 0;
+
+// Configuration of datamap canvas
+// Futher reading can be found at https://datamaps.github.io/
+function Datamap() {
+  this.$container = $("#container");
+  instance = this.instance = new Datamaps({
+    scope: 'world',
+    element: this.$container.get(0),
+    done: this._handleMapReady.bind(this),
+    projection: 'mercator',
+        fills: {
+          /*defaultFill: '#454545'*/
+          defaultFill: 'black'
+        },
+        geographyConfig: {
+          hideAntarctica: true,
+          borderColor: '#0062a2',
+          highlightFillColor: '#4b4d4a',
+          highlightBorderColor: '#fdfdfd',
+          popupOnHover: true,
+          popupTemplate: function(geography, data) {
+          return '<div class="hoverinfo" id="country">' + geography.properties.name + '</div>';
+        },
+        },
+        ph_arcConfig: {
+          strokeColor: '#0062a2',
+          strokeWidth: 2,
+          arcSharpness: 2, /* 5 */
+          animationSpeed: 3000, // Milliseconds
+          popupOnHover: true,
+          // Case with latitude and longitude
+          popupTemplate: function(geography, data) {
+            if ( ( data.origin && data.destination ) && data.origin.latitude && data.origin.longitude && data.destination.latitude && data.destination.longitude ) {
+              // Content of info table
+              str = '<div class="hoverinfo"><table id="event"><tr><th>Warden Event</th></tr><tr><td>Type</td><td>'+ JSON.stringify(data.event) +'</td></tr><tr><td>Detect Time</td><td>'+ JSON.stringify(data.time) +'</td></tr><tr><th>Event origin</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.origin.ip) +  '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.origin.city) + ',&nbsp;' + JSON.stringify(data.origin.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.origin.latitude) + ',&nbsp;' + JSON.stringify(data.origin.longitude) + '</td></tr><tr><th>Event Destination</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.destination.ip) + '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.destination.city) + ',&nbsp;' + JSON.stringify(data.destination.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.destination.latitude) + ',&nbsp;' + JSON.stringify(data.destination.longitude) + '</td></tr></table></div>';
+              return str.replace(/&quot;/g,"");
+            }
+            // Missing information
+            else {
+              return '';
+            }
+          }
+        }
+      });
+
+  legend_data = d3.select("#legend")
+    .selectAll("li")
+    .data(Object.keys(cat_color).sort())
+    .enter()
+      .append("li")
+        .append("span")
+          .style("color", function(datum) { return cat_color[datum]})
+          .text(function(datum) { return datum; })
+          .append("span")
+          .text(function(datum) { return " – " + cat_desc[datum]})
+          .style("color", "white");
+  
+    instance.addPlugin('ph_arc', handleArcs);
+
+  setInterval(function(){
+      d3.json("./warden-map.json", function(error, data) {
+          if (data) {
+              var cur_data = data.pop()
+              var cur_time = new Date().getTime();
+              if (cur_data != prev_data) {
+                  prev_data = cur_data;
+                  for (var i=0; i<data.length; i++) {
+                      data[i].arrivalTime = cur_time;
+                      data[i].delay = i/data.length;
+                  }
+                  main_data = main_data.concat(data);
+              }
+          }
+          var trimmed_data = [];
+          for (var i=0; i<main_data.length; i++) {
+              if (main_data[i].arrivalTime + 3500 > cur_time) {
+                  trimmed_data.push(main_data[i]);
+              }
+          }
+          main_data = trimmed_data;
+          trimmed_data = cur_time = cur_data = error = data = null;
+          instance.ph_arc(main_data);
+      });
+  }, 1000);
+
+};
+
+Datamap.prototype._handleMapReady = function(datamap) {
+  this.zoom = new Zoom({
+    $container: this.$container,
+    datamap: datamap
+  });
+}

contrib/map/frontend/warden-map.html

+<!--                                                                                   -->
+<!--                                                                                   -->
+<!-- -*- coding: utf-8 -*-                                                             -->
+<!--                                                                                   -->
+<!--  warden-map.html                                                                  -->
+<!--                                                                                   -->
+<!-- Copyright (C) 2016 Cesnet z.s.p.o                                                 -->
+<!-- Use of this source is governed by a 3-clause BSD-style license, see LICENSE file. -->
+<!--                                                                                   -->
+<!--                                                                                   -->
+
+
+<!DOCTYPE html>
+<meta name="robots" content="noindex">
+<meta charset="utf-8">
+<link href='https://fonts.googleapis.com/css?family=Oswald&amp;subset=latin,latin-ext' rel='stylesheet' type='text/css'>
+<link rel="stylesheet" type="text/css" href="./css/warden-map.css"/>
+<body>
+
+<script src="https://d3js.org/d3.v3.min.js"></script>
+<script src="https://d3js.org/topojson.v1.min.js"></script>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+<script src="./js/datamaps.world.min.js"></script>
+<script src="./js/warden-map.js"></script>
+
+<!--
+<h2>Warden Map</h2>
+<div id="tools">
+  <button class="zoom-button" data-zoom="reset">&#x2302</button>
+  <button class="zoom-button" data-zoom="out">-</button>
+  <button class="zoom-button" data-zoom="in">+</button>
+  <div id="zoom-info"></div>
+</div>
+-->
+<div id="container"></div>
+<div id="heading">Attacks, detected in CESNET network<br/>
+SABU - Sharing and Analysis of Security Events
+</div>
+<div id="legend-box">
+	<p><b>Reported to Warden right <i>now</i>.</b></p>
+	<ul id="legend"></ul>
+</div>
+
+<!-- Draw datamap into id="container" -->
+<script>new Datamap();</script>
+
+</body>
+</html>

warden3/warden_client/README → warden_client/README

 +---------------------------------+
-| Warden Client Library 3.0-beta2 |
+| Warden Client Library 3.0-beta3 |
 +---------------------------------+
 
 Content
@@ -15,7 +15,7 @@ A. Introduction
 
    The main goal of Warden 3 is to address the shortcomings, which emerged
 during several years of Warden 2.X operation. Warden 3 uses flexible and
-descriptive event format, based on JSON. Warden 3 protocol is based on  plain
+descriptive event format, based on JSON. Warden 3 protocol is based on plain
 HTTPS queries with help of JSON (Warden 2 SOAP is heavyweight, outdated and
 draws in many dependencies). Clients can be multilanguage, unlike SOAP/HTTPS,
 plain HTTPS and JSON is mature in many mainstream programming languages.
@@ -36,13 +36,14 @@ B. Quick start (TL;DR)
    sandbox URL, etc.
    If succesful, you will receive authentication secret.
  * Use warden_curl_test.sh to check you are able to talk to server.
- * See warden_client_examples.py on how to integrate sending/recieving
+ * See warden_client_examples.py on how to integrate sending/receiving
    into your Python application.
  * Alternatively, check 'contrib' directory in Warden GIT for various
    ready to use tools or recipes. You may find senders for various
    honeypots, or warden_filer may get handy if you do not want to delve
    into Python at all.
- * Welcome! Thanks for your security data, and use others' for common good.
+ * Welcome! Thanks for participating in in the data exchange to improve
+   network security awareness.
 
 ------------------------------------------------------------------------------
 C. Concepts
@@ -52,7 +53,7 @@ C.1. Event description format
    IDEA - Intrusion Detection Extensible Alert, flexible extensible format
 for security events, see:
 
-   https://csirt.cesnet.cz/IDEA
+   https://idea.cesnet.cz/
 
 C.2. Event serial ID
 
@@ -64,7 +65,7 @@ C.3. Authentication
 
    In Warden 2, clients get authenticated by server certificate, however
 server certificate is usually same for the whole machine, so individual
-clients are differentiated only by telling its own name. However, client name
+clients are differentiated only by telling their own name. However, client name
 is widely known, so this allows for client impersonation within one machine.
 Warden 3 slightly improves this schema by replacing client name in
 authentication phase by "secret", random string, shared among particular
@@ -133,7 +134,7 @@ sending events). The keys of the object, which may be available, are:
     description.
 
    Client errors (4xx) are considered permanent - client must not try to send
-same event again as it will get always rejected - client administrator
+same event again as it will always get rejected - client administrator
 will need to inspect logs and rectify the cause.
 
    Server errors (5xx) may be considered by client as temporary and client is
@@ -464,4 +465,4 @@ for e in res:
 debug_str() output increasingly more detailed info.
 
 ------------------------------------------------------------------------------
-Copyright (C) 2011-2015 Cesnet z.s.p.o
+Copyright (C) 2011-2022 Cesnet z.s.p.o

warden3/warden_client/warden_client.cfg.dist → warden_client/warden_client.cfg.dist

 {
     "url": "https://warden-hub.example.org/warden3",
-
     "certfile": "cert.pem",
     "keyfile": "key.pem",
-    "cafile": "cacert.pem",
-
-    "timeout": 60,
-    "get_events_limit": 1000,
-
-    "errlog": {"level": "debug"},
-    "filelog": {"file": "warden_client.log", "level": "warning"},
-    //"syslog": {"socket": "/dev/log", "facility": "local7", "level": "warning"},
-
-    "idstore": "warden_client.id",
-
+    "filelog": {"level": "debug"},
     "name": "org.example.warden_client",
-
     "secret": "ToP_SeCrEt"
 }

warden3/warden_client/warden_client.py → warden_client/warden_client.py

@@ -4,19 +4,34 @@
 # Copyright (C) 2011-2015 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-import json, httplib, ssl, socket, logging, logging.handlers, time
-from urlparse import urlparse
-from urllib import urlencode
-from sys import stderr, exc_info
+import hashlib  # Some Python/ssl versions incorrectly initialize hashes, this helps
+from sys import stderr, exc_info, version_info
+import json, ssl, socket, logging, logging.handlers, time
 from traceback import format_tb
 from os import path
 from operator import itemgetter
-from sys import version_info
 
-fix_logging_filename = str if version_info<(2.7) else lambda(x): x
+fix_logging_filename = str if version_info<(2, 7) else lambda x: x
 
+if version_info[0] >= 3:
+    import http.client as httplib
+    from urllib.parse import urlparse
+    from urllib.parse import urlencode
+    basestring = str
+else:
+    import httplib
+    from urlparse import urlparse
+    from urllib import urlencode
 
-VERSION = "3.0-beta2"
+
+
+VERSION = "3.0-beta3"
+
+DEFAULT_CA_STORES = [
+    "/etc/ssl/certs/ca-certificates.crt",       # Deb
+    "/etc/pki/tls/certs/ca-bundle.crt",         # RH
+    "/var/lib/ca-certificates/ca-bundle.pem"    # SuSE
+ ]
 
 class HTTPSConnection(httplib.HTTPSConnection):
     '''
@@ -24,11 +39,12 @@ class HTTPSConnection(httplib.HTTPSConnection):
     of SSL/ TLS version and cipher selection.  See:
     http://hg.python.org/cpython/file/c1c45755397b/Lib/httplib.py#l1144
     and `ssl.wrap_socket()`
+    Used only if ssl.SSLContext is not available (Python version < 2.7.9)
     '''
     def __init__(self, host, **kwargs):
-        self.ciphers = kwargs.pop('ciphers',None)
-        self.ca_certs = kwargs.pop('ca_certs',None)
-        self.ssl_version = kwargs.pop('ssl_version',ssl.PROTOCOL_SSLv23)
+        self.ciphers = kwargs.pop('ciphers', None)
+        self.ca_certs = kwargs.pop('ca_certs', None)
+        self.ssl_version = kwargs.pop('ssl_version', getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23))
 
         httplib.HTTPSConnection.__init__(self,host,**kwargs)
 
@@ -105,6 +121,14 @@ class Error(Exception):
                 kwargs["send_events_limit"] = int(kwargs["send_events_limit"])
             except Exception:
                 del kwargs["send_events_limit"]
+        if "exc" in kwargs:
+            # Traceback objects cause reference loops, so memory may be not
+            # correctly free'd. We only need traceback to log it in str_debug(),
+            # so let's get the string representation now and forget the 
+            # traceback object, thus preventing the loop.
+            exctype, excvalue, tb = kwargs["exc"]
+            tb = format_tb(tb)
+            kwargs["exc"] = exctype, excvalue, tb
         self.errors.append(kwargs)
 
 
@@ -135,6 +159,8 @@ class Error(Exception):
         """ In list or iterable context we're empty """
         raise StopIteration
 
+    __next__ = next
+
 
     def __bool__(self):
         """ In boolean context we're never True """
@@ -194,10 +220,10 @@ class Error(Exception):
         out.append(self.str_preamble(e))
         if not "exc" in e or not e["exc"]:
             return ""
-        exc_tb = e["exc"][2]
+        exc_tb = e["exc"][2] # exc_tb is string repr. of traceback object
         if exc_tb:
             out.append("Traceback:\n")
-            out.extend(format_tb(exc_tb))
+            out.extend(exc_tb)
         return "".join(out)
 
 
@@ -210,8 +236,8 @@ class Client(object):
             keyfile=None,
             cafile=None,
             timeout=60,
-            retry=3,
-            pause=5,
+            retry=20,
+            pause=10,
             get_events_limit=6000,
             send_events_limit=500,
             errlog={},
@@ -232,9 +258,9 @@ class Client(object):
         self.conn = None
 
         base = path.join(path.dirname(__file__))
-        self.certfile = path.join(base, certfile or "cert.pem")
-        self.keyfile  = path.join(base, keyfile or "key.pem")
-        self.cafile = path.join(base, cafile or "ca.pem")
+        self.certfile = self.get_readable_file(certfile or "cert.pem", base)
+        self.keyfile  = self.get_readable_file(keyfile or "key.pem", base)
+        self.cafile = self.get_readable_file(cafile if cafile is not None else DEFAULT_CA_STORES, base)
         self.timeout = int(timeout)
         self.get_events_limit = int(get_events_limit)
         self.idstore = path.join(base, idstore) if idstore is not None else None
@@ -243,12 +269,42 @@ class Client(object):
         self.retry = int(retry)
         self.pause = int(pause)
 
-        self.ciphers = 'TLS_RSA_WITH_AES_256_CBC_SHA'
-        self.sslversion = ssl.PROTOCOL_TLSv1
+        self.ciphers = None
+        self.sslversion = getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23)
+
+        # If Python is new enough to have SSLContext, use it for SSL settings,
+        # otherwise our own class derived from httplib.HTTPSConnection is used
+        # later in connect().
+        if hasattr(ssl, 'SSLContext'):
+            self.sslcontext = ssl.SSLContext(self.sslversion)
+            self.sslcontext.load_cert_chain(self.certfile, self.keyfile)
+            if self.cafile:
+                self.sslcontext.load_verify_locations(self.cafile)
+                self.sslcontext.verify_mode = ssl.CERT_REQUIRED
+            else:
+                self.sslcontext.verify_mode = ssl.CERT_NONE
+        else:
+            self.sslcontext = None
 
         self.getInfo()  # Call to align limits with server opinion
 
 
+    def get_readable_file(self, name, base):
+        names = [name] if isinstance(name, basestring) else name
+        names = [path.join(base, n) for n in names]
+        errors = []
+        for n in names:
+            try:
+                open(n, "r").close()
+                self.logger.debug("Using %s" % n)
+                return n
+            except IOError as e:
+                errors.append(e)
+        for e in errors:
+            self.logger.error(str(e))
+        return names[0] if names else None
+
+
     def init_log(self, errlog, syslog, filelog):
 
         def loglevel(lev):
@@ -320,19 +376,23 @@ class Client(object):
 
         try:
             if self.url.scheme=="https":
-                conn = HTTPSConnection(
-                    self.url.netloc,
-                    strict = False,
-                    key_file = self.keyfile,
-                    cert_file = self.certfile,
-                    timeout = self.timeout,
-                    ciphers = self.ciphers,
-                    ca_certs = self.cafile,
-                    ssl_version = self.sslversion)
+                if self.sslcontext:
+                    conn = httplib.HTTPSConnection(
+                        self.url.netloc,
+                        timeout = self.timeout,
+                        context = self.sslcontext)
+                else:
+                    conn = HTTPSConnection(
+                        self.url.netloc,
+                        key_file = self.keyfile,
+                        cert_file = self.certfile,
+                        timeout = self.timeout,
+                        ciphers = self.ciphers,
+                        ca_certs = self.cafile,
+                        ssl_version = self.sslversion)
             elif self.url.scheme=="http":
                 conn = httplib.HTTPConnection(
                     self.url.netloc,
-                    strict = False,
                     timeout = self.timeout)
             else:
                 return Error(message="Don't know how to connect to \"%s\"" % self.url.scheme,
@@ -358,27 +418,30 @@ class Client(object):
             kwargs["secret"] = self.secret
 
         if kwargs:
-            for k in kwargs.keys():
+            for k in list(kwargs.keys()):
                 if kwargs[k] is None:
                     del kwargs[k]
             argurl = "?" + urlencode(kwargs, doseq=True)
         else:
             argurl = ""
 
-        try:
-            if payload is None:
-                data = ""
-            else:
+        self.headers = {"Accept": "application/json"}
+        data = None
+
+        if payload is None:
+            method = "GET"
+        else:
+            method = "POST"
+            try:
                 data = json.dumps(payload)
-        except:
-            return Error(message="Serialization to JSON failed",
-                exc=exc_info(), method=func, payload=payload)
+            except:
+                return Error(message="Serialization to JSON failed",
+                    exc=exc_info(), method=func, payload=payload)
 
-        self.headers = {
-            "Content-Type": "application/json",
-            "Accept": "application/json",
-            "Content-Length": str(len(data))
-        }
+            self.headers.update({
+                "Content-Type": "application/json",
+                "Content-Length": str(len(data))
+            })
 
         # HTTP(S)Connection is oneshot object (and we don't speak "pipelining")
         conn = self.connect()
@@ -387,7 +450,7 @@ class Client(object):
 
         loc = '%s/%s%s' % (self.url.path, func, argurl)
         try:
-            conn.request("POST", loc, data, self.headers)
+            conn.request(method, loc, data, self.headers)
         except:
             conn.close()
             return Error(message="Sending of request to server failed",
@@ -411,13 +474,13 @@ class Client(object):
 
         if res.status==httplib.OK:
             try:
-                data = json.loads(response_data)
+                data = json.loads(response_data.decode("utf-8"))
             except:
                 data = Error(method=func, message="JSON message parsing failed",
                     exc=exc_info(), response=response_data)
         else:
             try:
-                data = json.loads(response_data)
+                data = json.loads(response_data.decode("utf-8"))
                 data["errors"]   # trigger exception if not dict or no error key
             except:
                 data = Error(method=func, message="Generic server HTTP error",
@@ -509,7 +572,7 @@ class Client(object):
         """ Send out "events" list to server, retrying on server errors.
         """
         ev = events
-        idx_xlat = range(len(ev))
+        idx_xlat = list(range(len(ev)))
         err = Error()
         retry = retry or self.retry
         attempt = retry
@@ -527,7 +590,7 @@ class Client(object):
                 res.errors.sort(key=itemgetter("error"))
                 for e in res.errors:
                     errno = e["error"]
-                    evlist = e.get("events", range(len(ev)))   # none means all
+                    evlist = e.get("events", list(range(len(ev))))   # none means all
                     if errno < 500 or not attempt:
                         # Fatal error or last try, translate indices
                         # to original and prepare for returning to caller
@@ -550,11 +613,11 @@ class Client(object):
             tag=None, notag=None,
             group=None, nogroup=None):
 
-        if not id:
+        if id is None:
             id = self._loadID(idstore)
 
         res = self.sendRequest(
-            "getEvents", id=id, count=count or self.get_events_limit, cat=cat,
+            "getEvents", id=id, count=self.get_events_limit if count is None else count, cat=cat,
             nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
 
         if res:
@@ -602,7 +665,6 @@ def format_time(year, month, day, hour, minute, second, microsec=0, utcoffset=No
 
 
 def read_cfg(cfgfile):
-    abspath = path.join(path.dirname(__file__), cfgfile)
-    with open(abspath, "r") as f:
+    with open(cfgfile, "r") as f:
         stripcomments = "\n".join((l for l in f if not l.lstrip().startswith(("#", "//"))))
         return json.loads(stripcomments)

warden3/warden_client/warden_client_examples.py → warden_client/warden_client_examples.py

@@ -4,16 +4,13 @@
 # Copyright (C) 2011-2015 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-from warden_client import Client, Error, read_cfg, format_timestamp
-import json
 import string
-from time import time, gmtime
-from math import trunc
+from time import time
 from uuid import uuid4
 from pprint import pprint
-from os import path
-from random import randint, randrange, choice, random;
-from base64 import b64encode;
+from random import randint, randrange, choice, random
+from base64 import b64encode
+from warden_client import Client, Error, read_cfg, format_timestamp
 
 def gen_min_idea():
 
@@ -56,7 +53,7 @@ def gen_random_idea(client_name="cz.example.warden.test"):
     def randip6():
         return [rand6ip, geniprange(rand6ip), rand6cidr][randint(0, 2)]()
 
-    def randstr(charlist=string.letters, maxlen=32, minlen=1):
+    def randstr(charlist=string.ascii_letters, maxlen=32, minlen=1):
         return ''.join(choice(charlist) for i in range(randint(minlen, maxlen)))
 
     event = {
@@ -106,7 +103,7 @@ def gen_random_idea(client_name="cz.example.warden.test"):
              "Size": 46,
              "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())],
              "ContentEncoding": "base64",
-             "Content": b64encode(randstr())
+             "Content": b64encode(randstr().encode('ascii')).decode("ascii")
           }
        ],
        "Node": [
@@ -139,7 +136,7 @@ def main():
     #     idstore="MyClient.id",
     #     name="cz.example.warden.test")
 
-    print "=== Debug ==="
+    print("=== Debug ===")
     info = wclient.getDebug()
     pprint(info)
 
@@ -150,18 +147,18 @@ def main():
     # If you want just to be informed, this is not necessary, just
     # configure logging correctly and check logs.
     if isinstance(info, Error):
-        print info
+        print(info)
 
-    print "=== Server info ==="
+    print("=== Server info ===")
     info = wclient.getInfo()
 
-    print "=== Sending 10 event(s) ==="
+    print("=== Sending 10 event(s) ===")
     start = time()
     ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)])
-    print ret
-    print "Time: %f" % (time()-start)
+    print(ret)
+    print("Time: %f" % (time()-start))
 
-    print "=== Getting 10 events ==="
+    print("=== Getting 10 events ===")
     start = time()
 
     # cat = ['Availability', 'Abusive.Spam','Attempt.Login']
@@ -182,10 +179,10 @@ def main():
     nogroup = []
 
     ret = wclient.getEvents(count=10, cat=cat, nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
-    print "Time: %f" % (time()-start)
-    print "Got %i events" % len(ret)
+    print("Time: %f" % (time()-start))
+    print("Got %i events" % len(ret))
     for e in ret:
-        print e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name")
+        print(e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name"))
 
 if __name__ == "__main__":
     main()

warden3/warden_client/warden_curl_test.sh → warden_client/warden_curl_test.sh

@@ -23,7 +23,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/blefub?client=$client&secret=$secret"
 echo
 
@@ -33,7 +32,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/?client=$client&secret=$secret"
 echo
 
@@ -43,7 +41,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client"
 echo
 
@@ -53,7 +50,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents"
 echo
 
@@ -63,7 +59,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=asdf.blefub"
 echo
 
@@ -73,7 +68,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=asdf.blefub&secret=$secret"
 echo
 
@@ -83,7 +77,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=ASDFblefub"
 echo
 
@@ -93,7 +86,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?secret=$secret"
 echo
 
@@ -114,7 +106,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&cat=bflm"
 echo
 
@@ -124,7 +115,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&cat=Other&nocat=Test"
 echo
 
@@ -145,7 +135,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&self=test"
 echo
 
@@ -155,7 +144,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&bad=guy"
 echo
 
@@ -165,7 +153,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret"
 echo
 
@@ -175,7 +162,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&count=3&id=10"
 echo
 
@@ -185,7 +171,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getDebug?client=$client&secret=$secret"
 echo
 
@@ -195,6 +180,5 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getInfo?client=$client&secret=$secret"
 echo

warden3/contrib/warden_filer/README → warden_filer/README

 +---------------------------------------+
-| Warden Filer 3.0-beta2 for Warden 3.X |
+| Warden Filer 3.0-beta3 for Warden 3.X |
 +---------------------------------------+
 
 Content
@@ -104,6 +104,13 @@ JSON object, containing configuration. See also warden_filer.cfg as example.
              array (see Idea doc). Be careful here, you may ruin Idea
              messages by wrongly formatted data and they are not checked
              here in any way
+      poll_time - how often to check Warden server for new events (in seconds,
+             defaults to 5)
+      file_limit - limit number of files in "incoming" directory. When the limit
+             is reached, polling is paused for "limit_wait_time" seconds
+      limit_wait_time - wait this number of seconds if limit on number of files
+             is reached (defaults to 5)
+      
 
     Both the "sender" and "reciever" sections can also bear daemon
 configuration.
@@ -128,11 +135,11 @@ directories must obey simple protocols, which use atomic "rename" to avoid
 locking issues.
    Also, your directory (and its structure) _must_ reside on the same
 filesystem to keep "rename" atomic. _Never_ try to mount some of the
-subdirectories ("temp", "incoming", "errors") from other filesystem.
+subdirectories ("tmp", "incoming", "errors") from other filesystem.
 
  1. Inserting file
 
-   * The file you want to create _must_ be created in the "temp" subdirectory
+   * The file you want to create _must_ be created in the "tmp" subdirectory
      first, _not_ "incoming". Filename is arbitrary, but must be unique among
      all subdirectories.
 
@@ -141,7 +148,7 @@ subdirectories ("temp", "incoming", "errors") from other filesystem.
      or complete.
 
    For simple usage (bash scripts, etc.), just creating sufficiently random
-   filename in "temp" and then moving into "incoming" may be enough.
+   filename in "tmp" and then moving into "incoming" may be enough.
    Concatenating $RANDOM couple of times will do. :)
    
    For advanced or potentially concurrent usage inserting enough of unique
@@ -152,7 +159,7 @@ subdirectories ("temp", "incoming", "errors") from other filesystem.
 
  2. Picking up file
 
-   * Rename the file to work with into "temp" directory.
+   * Rename the file to work with into "tmp" directory.
 
    * Do whatever you want with contents, and when finished, rename file back
      into "incoming", or remove, or move somewhere else, or move into "errors"

warden_filer/check_file_count

+#!/bin/bash
+
+script=${0##*/}
+warn=0
+crit=65536
+read -rd '' helps <<EOF
+$script: Icinga plugin to check too high number of files in directory.
+
+Usage: $script -d dir [-w num] [-c num] [-h ]
+
+  -d dir    directory to watch
+  -w num    warning if number of files exceeds this value (default $warn)
+  -c num    critical if number of files exceeds this value (default $crit)
+EOF
+
+function bailout {
+	echo -n "$script" | tr '[:lower:]' '[:upper:]'
+	echo " $2 $3"
+	exit "$1"
+}
+
+while getopts hvVd:w:c: opt; do
+	case "$opt" in
+		h) bailout 3 "UNKNOWN" "$helps";;
+		d) dir="$OPTARG";;
+		w) warn="$OPTARG";;
+		c) crit="$OPTARG";;
+		"?") bailout 3 "UNKNOWN" "Unknown option, use -h for help";;
+	esac
+done
+
+[ -z "$dir" ] && bailout 3 "UNKNOWN" "-d not specified"
+
+count=$(find "$dir" -mindepth 1 -maxdepth 1 | wc -l)
+
+[ "$count" -gt "$crit" ] && bailout 2 "CRIT" "$count"
+
+[ "$count" -gt "$warn" ] && bailout 1 "WARN" "$count"
+
+bailout 0 "OK" "$count"

warden3/contrib/warden_filer/warden_filer.cfg → warden_filer/warden_filer.cfg.dist

+// For all options see documentation
 {
     // Warden config can be also referenced as:
     // "warden": "/path/to/warden_client.cfg"
     "warden": {
         "url": "https://example.com/warden3",
         "cafile": "tcs-ca-bundle.pem",
-        "timeout": 10,
-        "errlog": {"level": "debug"},
+        "keyfile": "my.key.pem",
+        "certfile": "my.cert.pem",
+        "timeout": 60,
+        "retry": 20,
+        "pause": 5,
         "filelog": {"level": "debug"},
-        "idstore": "myclient.id",
         "name": "com.example.warden.test",
         "secret": "SeCrEt"
     },
@@ -16,41 +19,38 @@
         // for Idea events to send out
         "dir": "warden_sender",
         // Optional filter fields, unmatched events are discarded (and removed)
-        "filter": {
-            "cat": ["Test", "Recon.Scanning"],
-            "nocat": null,
-            "group": ["cz.example"],
-            "nogroup": null,
-            "tag": null,
-            "notag": ["Honeypot"]
-        },
+        //"filter": {
+        //    "cat": ["Test", "Recon.Scanning"],
+        //    "nocat": null,
+        //    "group": ["cz.example"],
+        //    "nogroup": null,
+        //    "tag": null,
+        //    "notag": ["Honeypot"]
+        //},
         // Optional information about detector to be prepended into Idea Node array
-        "node": {
-            "Name": "cz.example.warden.test_sender",
-            "Type": ["Relay"],
-            "SW": ["warden_filer-sender"],
-            "AggrWin": "00:05:00",
-        }
+        //"node": {
+        //    "Name": "cz.example.warden.test_sender",
+        //    "Type": ["Relay"]
+        //}
     },
     "receiver": {
         // Maildir like directory, whose "incoming" will serve as target for events
         "dir": "warden_receiver",
         // Optional filter fields for Warden query
-        "filter": {
-            "cat": ["Test", "Recon.Scanning"],
-            "nocat": null,
-            "group": ["cz.cesnet"],
-            "nogroup": null,
-            "tag": null,
-            "notag": ["Honeypot"]
-        },
+        //"filter": {
+        //    "cat": ["Test", "Recon.Scanning"],
+        //    "nocat": null,
+        //    "group": ["cz.cesnet"],
+        //    "nogroup": null,
+        //    "tag": null,
+        //    "notag": ["Honeypot"]
+        //},
         // Optional information about detector to be prepended into Idea Node array
-        "node": {
-            "Name": "cz.example.warden.test_receiver",
-            "Type": ["Relay"],
-            "SW": ["warden_filer-receiver"],
-            "AggrWin": "00:05:00",
-            "Note": "Test warden_filer receiver"
-        }
+        //"node": {
+        //    "Name": "cz.example.warden.test_receiver",
+        //    "Type": ["Relay"]
+        //},
+        // Optional limit on number of files in "incoming" directory
+        //"file_limit": 10000
     }
 }

warden3/contrib/warden_filer/warden_filer.py → warden_filer/warden_filer.py

@@ -18,9 +18,15 @@ import resource
 import atexit
 import argparse
 from os import path, mkdir
-from random import choice, randint;
+from random import choice, randint
 
-VERSION = "3.0-beta2"
+# for py2/py3 compatibility
+try:
+    basestring
+except NameError:
+    basestring = str
+
+VERSION = "3.0-beta3"
 
 class NamedFile(object):
     """ Wrapper class for file objects, which allows and tracks filename
@@ -65,9 +71,9 @@ class NamedFile(object):
 
 class SafeDir(object):
     """ Maildir like directory for safe file exchange.
-        - Producers are expected to drop files into "temp" under globally unique
+        - Producers are expected to drop files into "tmp" under globally unique
           filename and rename it into "incoming" atomically (newfile method)
-        - Workers pick files in "incoming", rename them into "temp",
+        - Workers pick files in "incoming", rename them into "tmp",
           do whatever they want, and either discard them or move into
           "errors" directory
     """
@@ -76,7 +82,7 @@ class SafeDir(object):
         self.path = self._ensure_path(p)
         self.incoming = self._ensure_path(path.join(self.path, "incoming"))
         self.errors = self._ensure_path(path.join(self.path, "errors"))
-        self.temp = self._ensure_path(path.join(self.path, "temp"))
+        self.temp = self._ensure_path(path.join(self.path, "tmp"))
         self.hostname = socket.gethostname()
         self.pid = os.getpid()
 
@@ -116,7 +122,7 @@ class SafeDir(object):
         #       which checked uniqueness among all directories by atomic
         #       links.
 
-        # First find and open name unique within temp
+        # First find and open name unique within tmp
         tmpname = None
         while not tmpname:
             tmpname = self._get_new_name()
@@ -138,46 +144,73 @@ class SafeDir(object):
         return [NamedFile(self.incoming, n) for n in os.listdir(self.incoming)]
 
 
+    def get_incoming_cnt(self):
+        """Get number of files in the incoming directory"""
+        return len(os.listdir(self.incoming))
+
+
 
 def receiver(config, wclient, sdir, oneshot):
     poll_time = config.get("poll_time", 5)
     node = config.get("node", None)
     conf_filt = config.get("filter", {})
+    file_limit = config.get("file_limit", None)
+    wait_time = config.get("limit_wait_time", 5)
     filt = {}
     # Extract filter explicitly to be sure we have right param names for getEvents
     for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"):
         filt[s] = conf_filt.get(s, None)
 
     while running_flag:
-        events = wclient.getEvents(**filt)
         count_ok = count_err = 0
-        while events:
-            for event in events:
-                if node:
-                    nodelist = event.setdefault("Node", [])
-                    nodelist.insert(0, node)
-                try:
-                    nf = None
-                    nf = sdir.newfile()
-                    with nf.f as f:
-                        data = json.dumps(event)
-                        f.write(data)
-                    nf.moveto(sdir.incoming)
-                    count_ok += 1
-                except Exception as e:
-                    Error(message="Error saving event", exc=sys.exc_info(), file=str(nf),
-                          event_ids=[event.get("ID")], sdir=sdir.path).log(wclient.logger)
-                    count_err += 1
+        limit_reached = False
+        if file_limit:
+            cnt_files = sdir.get_incoming_cnt() # Count files in 'incoming' dir
+            remain_to_limit = file_limit - cnt_files
+            # Query server, but not for more events than what can fit into limit
+            if remain_to_limit > 0:
+                events = wclient.getEvents(count=remain_to_limit, **filt)
+            else:
+                events = []
+            # Check whether limit was reached
+            if len(events) >= remain_to_limit:
+                limit_reached = True
+        else:
+            events = wclient.getEvents(**filt)
+
+        for event in events:
+            if node:
+                nodelist = event.setdefault("Node", [])
+                nodelist.insert(0, node)
+            try:
+                nf = None
+                nf = sdir.newfile()
+                with nf.f as f:
+                    data = json.dumps(event)
+                    f.write(data.encode('utf-8'))
+                nf.moveto(sdir.incoming)
+                count_ok += 1
+            except Exception as e:
+                Error(message="Error saving event", exc=sys.exc_info(), file=str(nf),
+                      event_ids=[event.get("ID")], sdir=sdir.path).log(wclient.logger)
+                count_err += 1
+
+        if events:
             wclient.logger.info(
                 "warden_filer: received %d, errors %d"
                 % (count_ok, count_err))
-            events = wclient.getEvents(**filt)
-            count_ok = count_err = 0
+
+        if limit_reached:
+            wclient.logger.info("Limit on number of files in 'incoming' dir reached.")
+
         if oneshot:
-            if not events:
+            if not events or limit_reached:
                 terminate_me(None, None)
         else:
-            time.sleep(poll_time)
+            if limit_reached:
+                time.sleep(wait_time)
+            elif not events:
+                time.sleep(poll_time)
 
 
 
@@ -264,7 +297,7 @@ def sender(config, wclient, sdir, oneshot):
                     continue    # Silently go to next filename, somebody else might have interfered
                 try:
                     with nf.open("rb") as fd:
-                        data = fd.read()
+                        data = fd.read().decode('utf-8')
                         event = json.loads(data)
                         if not match_event(event, **filt):
                             wclient.logger.debug("Unmatched event: %s" % data)
@@ -365,7 +398,7 @@ def daemonize(
     # PID file
     if pidfile is not None:
         pidd = os.open(pidfile, os.O_RDWR|os.O_CREAT|os.O_EXCL|os.O_TRUNC)
-        os.write(pidd, str(os.getpid())+"\n")
+        os.write(pidd, (str(os.getpid())+"\n").encode())
         os.close(pidd)
         # Define and setup atexit closure
         @atexit.register
@@ -437,6 +470,18 @@ def get_configs():
 
 
 
+def get_uid_gid(str_id, get_nam_func):
+    if str_id:
+        try:
+            id = int(str_id)
+        except ValueError:
+            id = get_nam_func(str_id)[2]
+    else:
+        id = None
+    return id
+
+
+
 if __name__ == "__main__":
 
     args = get_args()
@@ -449,12 +494,17 @@ if __name__ == "__main__":
 
     try:
         if args.daemon:
+            from pwd import getpwnam
+            from grp import getgrnam
+            uid = get_uid_gid(fconfig.get("uid"), getpwnam)
+            gid = get_uid_gid(fconfig.get("gid"), getgrnam)
+
             daemonize(
                 work_dir = fconfig.get("work_dir", "."),
                 chroot_dir = fconfig.get("chroot_dir"),
                 umask = fconfig.get("umask"),
-                uid = fconfig.get("uid"),
-                gid = fconfig.get("gid"),
+                uid = uid,
+                gid = gid,
                 pidfile = args.pid_file,
                 files_preserve = get_logger_files(wclient.logger),
                 signals = {

warden_filer/warden_filer_logrotate.dist

+# You may want to review and/or change the logfile path, user/group in
+# 'create' and daemon to restart in 'postrotate'
+
+/var/log/warden_filer.log
+{
+	rotate 52
+	weekly
+	missingok
+	notifempty
+	compress
+	delaycompress
+	dateext
+	create 640 mentat mentat
+	postrotate
+		/etc/init.d/warden_filer_sender restart
+		/etc/init.d/warden_filer_receiver restart
+	endscript
+}

warden3/contrib/warden_filer/warden_filer_receiver → warden_filer/warden_filer_receiver

-#!/bin/sh
+#!/bin/bash
 #
 ### BEGIN INIT INFO
 # Provides:          warden_filer_receiver
@@ -18,8 +18,19 @@ SERVICE_NAME="${DAEMON_NAME}_${FUNC}"
 PID=/var/run/"$DAEMON_NAME"/"$FUNC".pid
 CONFIG=/etc/"$DAEMON_NAME".cfg
 
-test -f /etc/default/"$SERVICE_NAME" && . /etc/default/"$SERVICE_NAME"
-. /lib/lsb/init-functions
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$SERVICE_NAME" ] && . /etc/$n/"$SERVICE_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
 
 ACTION="$1"
 

warden3/contrib/warden_filer/warden_filer_sender → warden_filer/warden_filer_sender

-#!/bin/sh
+#!/bin/bash
 #
 ### BEGIN INIT INFO
 # Provides:          warden_filer_sender
@@ -18,8 +18,19 @@ SERVICE_NAME="${DAEMON_NAME}_${FUNC}"
 PID=/var/run/"$DAEMON_NAME"/"$FUNC".pid
 CONFIG=/etc/"$DAEMON_NAME".cfg
 
-test -f /etc/default/"$SERVICE_NAME" && . /etc/default/"$SERVICE_NAME"
-. /lib/lsb/init-functions
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$SERVICE_NAME" ] && . /etc/$n/"$SERVICE_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
 
 ACTION="$1"
 

warden_ra/LICENSE

+Copyright (c) 2011-2016 Cesnet z.s.p.o <warden-info@cesnet.cz>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\ No newline at end of file

warden_ra/README

+Warden Registration Authority for Warden 3.X
+============================================
+
+Introduction
+------------
+
+Warden RA is a certificate registration authority for Warden_ server.
+It is meant to support the client registration process and simplification
+of the credential transport.
+
+As Warden clients are authenticated by X509 certificate, the usual certificate
+generation process can be used - local key and certificate request gets
+generated, the request is submitted to registration authority, and after
+review, certificate is issued and delivered back.
+
+However in centralised Warden setup, it is useful to be able to preallow
+certificate for particular client during negotiation phase (thus removing
+another round-trip).
+
+This is done by issuing 'token' by Warden registration officer during client
+registration, which is just a oneshot password, allowing sending the request
+and getting new certificate in one step through web API.
+
+Password is meant only for completely new clients or unusual situations,
+however RA also allows prolongation - generating new certificate by using
+old certificate (which must be still valid, of course) instead of password.
+
+The application script, which can be distributed to newly registered clients,
+is also included.
+
+Dependencies
+------------
+
+ 1. Platform
+
+    Python 2.7+
+    Apache 2.2+
+    mod_wsgi 3.3+
+
+
+Registration process
+--------------------
+
+New client credentials
+``````````````````````
+
+After succesful negotiation of new Warden client parameters, the registration
+officer enables new certificate generation by issuing (on the server side):
+
+    warden_ra.py applicant --name org.example.warden.client
+
+The tool generates and sets one time password on the registration authority
+side, and this password can be sent (preferably through the secured channel)
+to the new client administrator along with other setup information.
+
+The client administrator runs the application script with application
+password:
+
+    warden_apply.sh https://warden.example.org/warden-ra/ org.example.warden.client P4SSW0RD
+
+The script creates new X509 key, CSR certificate request and makes call to
+the Warden RA web service, where it obtains the new complete certificate.
+
+
+Prolonging existing client credentials
+``````````````````````````````````````
+
+The client administrator runs the application script with his existing valid
+Warden credentials, which he needs to prolong:
+
+    warden_apply.sh https://warden.example.org/warden-ra/ org.example.warden.client cert.pem key.pem
+
+The script creates new X509 key, CSR certificate request and makes call to
+the Warden RA web service, where it obtains the new complete certificate.
+
+
+Installation
+------------
+
+This depends heavily on your distribution and Apache configuration.
+Basically you need to create and include apache.conf:
+
+  Include /opt/warden_ra/apache22.conf
+
+or paste the contents into whichever Directory, Location or VirtualHost
+you dedicate for Warden RA. Note that you have to use different host than
+the one for Warden server, as Warden RA needs different Apache options
+for SSL authentication.
+
+You may need to change paths to certificate/key/ca material, path to
+warden_server.wsgi and web path alias.
+
+Note that warden_ra itself is NOT thread safe, so included configuration
+expects Apache with mpm-prefork worker, or you will have to configure
+mod_wsgi as separate process with threads=1.
+
+Also, for warden_server.wsgi, you can use warden_server.wsgi.dist as
+a template. You will possibly need to change at least configuration
+file path.
+
+ * Now install and/or configure RA backend (see README.openssl or README.ejbca)
+
+ * Configure Warden RA (see next chapter)
+
+ * Reload Apache
+
+
+Configuration
+-------------
+
+   Configuration is JSON object in file (warden_server.cfg by default),
+however, lines starting with "#" or "//" are allowed and will be ignored as
+comments. File must contain valid JSON object, containing configuration. See
+also warden_server.cfg.dist as example.
+
+   Top level JSON object contains subsections, which configure particular
+aspect of the server. Each subsection can contain "type" key, which chooses
+particular implementation object of the aspect, for example type of logger
+(file, syslog), such as:
+
+   {
+      "Log": {
+         "type": "SysLogger",
+         ...
+      },
+      "DB": { ... }
+   }
+
+Sections and their "type" objects can be:
+
+    Log: FileLogger, SysLogger
+    Auth: OptionalAuthenticator
+    Registry: EjbcaRegistry, OpenSSLRegistry
+    Handler: CertHandler
+
+"type" keyword is not mandatory, if not specified, first implementation
+object from particular section list is used ("FileLogger" for example).
+
+Object function and configuration keys are as follows:
+
+   FileLogger: logging into file on the filesystem
+      filename: name of the file, defaults to "warden_ra.log" at
+                installation directory
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+   SysLogger: logging into unix syslog
+      socket: path to syslog socket, defaults to "/dev/log"
+      facility: syslog facility, defaults to "daemon"
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+    OptionalAuthenticator: authenticate based on X509 certificate, or
+        signal the password auth for the registry
+
+    CertHandler: the main certificate requestor implementation
+
+    For OpenSSLRegistry or EJBCARegistry configuration please see
+    README.openssl or README.ejbca respectively.
+
+
+Command line
+------------
+
+When run from the command line, RA allows for client and request management.
+
+
+    warden_ra.py [--help] [-c CONFIG] [-v]
+                        {list,register,applicant,request,gencert} ...
+
+    Warden server certificate registry
+    
+    arguments:
+      --help                show this help message and exit
+      -c CONFIG, --config CONFIG
+                            path to configuration file
+      -v, --verbose         be more chatty
+    
+    commands:
+      {list,register,applicant,request,gencert}
+        list                list clients
+        register            register client
+        applicant           allow for certificate application
+        request             generate CSR
+        gencert             get new certificate
+
+
+    warden_ra.py list [--help] [--name NAME]
+    
+    List registered clients.
+    
+    arguments:
+      --help       show this help message and exit
+      --name NAME  client name
+
+
+    warden_ra.py register [--help] --name NAME --admins
+                                 [ADMINS [ADMINS ...]]
+    
+    Add client registration entry.
+    
+    arguments:
+      --help                show this help message and exit
+      --name NAME           client name
+      --admins [ADMINS [ADMINS ...]]
+                            administrator list
+
+
+    warden_ra.py applicant [--help] --name NAME [--password PASSWORD]
+    
+    Set client into certificate application mode and set its password
+    
+    optional arguments:
+      --help               show this help message and exit
+      --name NAME          client name
+      --password PASSWORD  password for application (will be autogenerated if not
+                           set)
+
+    
+.. _Warden: https://warden.cesnet.cz/
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o