contrib/map/frontend/css/warden-map.css

+/*
+ *
+ * -*- coding: utf-8 -*-
+ *
+ * warden-map.css
+ *
+ * Copyright (C) 2016 Cesnet z.s.p.o
+ * Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+ *
+*/
+
+body {
+  font-family: 'Oswald', sans-serif;
+  background: #00253D;
+  border: 0px;
+  padding: 0px;
+  margin: 0px;
+}
+
+h2 {
+ color: #0062a2;
+}
+
+.hoverinfo {
+  font-family: 'Oswald', sans-serif;
+}
+
+#country {
+  color: #0062a2;	/* Cesnet blue */
+  font-weight: bold;
+}
+
+
+table {
+  text-align: left;
+  margin: 0;
+  padding: 0;
+  font-size: 12px;
+}
+
+table th {
+  color: #0062a2;	/* Cesnet blue */
+  padding: 0;
+}
+
+table td {
+  color: #4b4d4a;	/* Greenish gray */
+  padding: 0;
+}
+
+#container {
+  overflow: hidden;
+/*  border: 2px solid #0062a2;
+  border: 0px;
+  padding: 0px;
+  margin: 0px;
+  border-radius: 5px;*/
+  position: relative;
+/*  width: 1280px;
+  height: 720px;*/
+  max-width: 100%;
+  max-height: 100%
+  width: 100%;
+  height: 100vh;*/
+}
+
+.zoom-button {
+  width: 40px;
+  height: 40px;
+  border-radius: 5px;
+  border: none;
+  background: #dcdcda;
+  font-size: 23px;
+  font-weight: bold;
+  color: white;
+  cursor: pointer;
+}
+
+.zoom-button:hover {
+  background-color: #0062a2;
+}
+
+#zoom-info {
+  display: inline-block;
+  padding: 10px;
+  color: #0062a2;
+}
+
+#warden-logo {
+  position: absolute;
+  top: 30px;
+  left: 30px;
+  background: white;
+  padding: 10px;
+  border-radius: 10px;
+  width: 240px;
+  height: 92px;
+  text-align: center;
+}
+
+#cesnet-logo {
+  position: absolute;
+  top: 30px;
+  right: 30px;
+  background: white;
+  padding: 10px;
+  border-radius: 10px;
+  width: 240px;
+  height: 92px;
+  text-align: center;
+}
+
+#legend-box {
+  position: absolute;
+  bottom: 30px;
+  left: 30px;
+  background-color: rgba(0,0,0,0.3);
+  color: white;
+  padding: 10px;
+  border-radius: 10px;
+  /*width: 240px;
+  height: 92px;
+  text-align: center;*/
+}
+
+#heading {
+  position: absolute;
+  top: 30px;
+  left: 50%;
+  width: 40em;
+  height: 92px;
+  margin-left: -20em;
+  font-size: xx-large;
+  color: white;
+  text-align: center;
+  vertical-align: middle;
+  line-height: 92px;
+}

contrib/map/frontend/js/warden-map.js

+/*
+ *
+ * -*- coding: utf-8 -*-
+ *
+ * warden-map.js
+ *
+ * Copyright (C) 2016 Cesnet z.s.p.o
+ * Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
+ *
+*/
+
+// NOTE: Change path in a function d3.json() if you separate backend and frontend!
+
+// Zooming functionality is based on WunderBart's implementation
+// Please see following links:
+// https://github.com/wunderbart
+// https://jsfiddle.net/wunderbart/Lom3b0gb/
+
+  function Zoom(args) {
+  $.extend(this, {
+    $buttons:   $(".zoom-button"),
+    $info:      $("#zoom-info"),
+    scale:      { max: 50, currentShift: 0 },
+    $container: args.$container,
+    datamap:    args.datamap
+  });
+
+  this.init();
+}
+
+Zoom.prototype.init = function() {
+  var paths = this.datamap.svg.selectAll("path"),
+      subunits = this.datamap.svg.selectAll(".datamaps-subunit");
+
+  // preserve stroke thickness
+  paths.style("vector-effect", "non-scaling-stroke");
+
+  // disable click on drag end
+  subunits.call(
+    d3.behavior.drag().on("dragend", function() {
+      d3.event.sourceEvent.stopPropagation();
+    })
+  );
+
+  this.scale.set = this._getScalesArray();
+  this.d3Zoom = d3.behavior.zoom().scaleExtent([ 1, this.scale.max ]);
+
+  this._displayPercentage(1);
+  this.listen();
+};
+
+Zoom.prototype.listen = function() {
+  this.$buttons.off("click").on("click", this._handleClick.bind(this));
+
+  this.datamap.svg
+    .call(this.d3Zoom.on("zoom", this._handleScroll.bind(this)))
+    .on("dblclick.zoom", null); // disable zoom on double-click
+};
+
+Zoom.prototype.reset = function() {
+  this._shift("reset");
+};
+
+Zoom.prototype._handleScroll = function() {
+  var translate = d3.event.translate,
+      scale = d3.event.scale,
+      limited = this._bound(translate, scale);
+
+  this.scrolled = true;
+
+  this._update(limited.translate, limited.scale);
+};
+
+Zoom.prototype._handleClick = function(event) {
+  var direction = $(event.target).data("zoom");
+
+  this._shift(direction);
+};
+
+Zoom.prototype._shift = function(direction) {
+  var center = [ this.$container.width() / 2, this.$container.height() / 2 ],
+      translate = this.d3Zoom.translate(), translate0 = [], l = [],
+      view = {
+        x: translate[0],
+        y: translate[1],
+        k: this.d3Zoom.scale()
+      }, bounded;
+
+  translate0 = [
+    (center[0] - view.x) / view.k,
+    (center[1] - view.y) / view.k
+  ];
+
+  if (direction == "reset") {
+    view.k = 1;
+    this.scrolled = true;
+  } else {
+    view.k = this._getNextScale(direction);
+  }
+
+l = [ translate0[0] * view.k + view.x, translate0[1] * view.k + view.y ];
+
+  view.x += center[0] - l[0];
+  view.y += center[1] - l[1];
+
+  bounded = this._bound([ view.x, view.y ], view.k);
+
+  this._animate(bounded.translate, bounded.scale);
+};
+
+Zoom.prototype._bound = function(translate, scale) {
+  var width = this.$container.width(),
+      height = this.$container.height();
+
+  translate[0] = Math.min(
+    (width / height)  * (scale - 1),
+    Math.max( width * (1 - scale), translate[0] )
+  );
+
+  translate[1] = Math.min(0, Math.max(height * (1 - scale), translate[1]));
+
+  return { translate: translate, scale: scale };
+};
+
+Zoom.prototype._update = function(translate, scale) {
+  this.d3Zoom
+    .translate(translate)
+    .scale(scale);
+
+  this.datamap.svg.selectAll("g")
+    .attr("transform", "translate(" + translate + ")scale(" + scale + ")");
+
+  this._displayPercentage(scale);
+};
+
+Zoom.prototype._animate = function(translate, scale) {
+  var _this = this,
+      d3Zoom = this.d3Zoom;
+
+  d3.transition().duration(350).tween("zoom", function() {
+    var iTranslate = d3.interpolate(d3Zoom.translate(), translate),
+        iScale = d3.interpolate(d3Zoom.scale(), scale);
+
+    return function(t) {
+      _this._update(iTranslate(t), iScale(t));
+    };
+  });
+};
+
+Zoom.prototype._displayPercentage = function(scale) {
+  var value;
+
+  value = Math.round(Math.log(scale) / Math.log(this.scale.max) * 100);
+  this.$info.text(value + "%");
+};
+
+Zoom.prototype._getScalesArray = function() {
+  var array = [],
+      scaleMaxLog = Math.log(this.scale.max);
+
+  for (var i = 0; i <= 10; i++) {
+    array.push(Math.pow(Math.E, 0.1 * i * scaleMaxLog));
+  }
+
+  return array;
+};
+
+Zoom.prototype._getNextScale = function(direction) {
+  var scaleSet = this.scale.set,
+      currentScale = this.d3Zoom.scale(),
+      lastShift = scaleSet.length - 1,
+      shift, temp = [];
+
+  if (this.scrolled) {
+
+    for (shift = 0; shift <= lastShift; shift++) {
+      temp.push(Math.abs(scaleSet[shift] - currentScale));
+    }
+
+    shift = temp.indexOf(Math.min.apply(null, temp));
+
+    if (currentScale >= scaleSet[shift] && shift < lastShift) {
+      shift++;
+    }
+
+    if (direction == "out" && shift > 0) {
+      shift--;
+    }
+
+    this.scrolled = false;
+
+  } else {
+
+    shift = this.scale.currentShift;
+
+    if (direction == "out") {
+      shift > 0 && shift--;
+    } else {
+      shift < lastShift && shift++;
+    }
+  }
+
+  this.scale.currentShift = shift;
+
+  return scaleSet[shift];
+};
+
+
+ function defaults(obj) {
+    Array.prototype.slice.call(arguments, 1).forEach(function(source) {
+      if (source) {
+        for (var prop in source) {
+          // Deep copy if property not set
+          if (obj[prop] == null) {
+            if (typeof source[prop] == 'function') {
+              obj[prop] = source[prop];
+            }
+            else {
+              obj[prop] = JSON.parse(JSON.stringify(source[prop]));
+            }
+          }
+        }
+      }
+    });
+    return obj;
+  }
+
+function val( datumValue, optionsValue, context ) {
+    if ( typeof context === 'undefined' ) {
+      context = optionsValue;
+      optionsValues = undefined;
+    }
+    var value = typeof datumValue !== 'undefined' ? datumValue : optionsValue;
+
+    if (typeof value === 'undefined') {
+      return  null;
+    }
+
+    if ( typeof value === 'function' ) {
+      var fnContext = [context];
+      if ( context.geography ) {
+        fnContext = [context.geography, context.data];
+      }
+      return value.apply(null, fnContext);
+    }
+    else {
+      return value;
+    }
+  }
+  
+var cat_color = {
+  "Abusive": "MediumPurple",
+  "Malware": "Red",
+  "Recon": "LightSlateGray",
+  "Attempt": "GhostWhite",
+  "Intrusion": "DarkTurquoise",
+  "Availability": "HotPink",
+  "Information": "PaleTurquoise",
+  "Fraud": "Yellow",
+  "Vulnerable": "DarkGoldenRod",
+  "Anomaly": "Brown",
+  "Other": "Green"
+}
+
+var cat_desc = {
+  "Abusive": "spam",
+  "Malware": "virus, worm, trojan, malware",
+  "Recon": "scanning, sniffing",
+  "Attempt": "bruteforce, exploitation attempt",
+  "Intrusion": "botnet, successful exploit",
+  "Availability": "(D)DOS",
+  "Information": "wiretapping, spoofing, hijacking",
+  "Fraud": "phishing, scam",
+  "Vulnerable": "open for abuse",
+  "Anomaly": "unusual traffic",
+  "Other": "unknown/unidentified"
+}
+
+ function handleArcs (layer, data, options) {
+    var self = this,
+        svg = this.svg;
+
+    if ( !data || (data && !data.slice) ) {
+      throw "Datamaps Error - arcs must be an array";
+    }
+
+    // For some reason arc options were put in an `options` object instead of the parent arc
+    // I don't like this, so to match bubbles and other plugins I'm moving it
+    // This is to keep backwards compatability
+    for ( var i = 0; i < data.length; i++ ) {
+      data[i] = defaults(data[i], data[i].options);
+      delete data[i].options;
+    }
+
+    if ( typeof options === "undefined" ) {
+      options = defaultOptions.arcConfig;
+    }
+
+    var arcs = layer.selectAll('path.datamaps-arc').data( data, JSON.stringify );
+
+    var path = d3.geo.path()
+        .projection(self.projection);
+
+    arcs
+      .enter()
+        .append('svg:path')
+        .attr('class', 'datamaps-arc')
+        .style('stroke-linecap', 'round')
+        .style('stroke', function(datum) {
+/*          return val(datum.strokeColor, options.strokeColor, datum);*/
+          for (cat in cat_color) {
+            if (datum.event.startsWith(cat)) {
+              return cat_color[cat];
+            }
+          }
+          return "Green";
+        })
+        .style('fill', 'none')
+        .style('stroke-width', function(datum) {
+            return val(datum.strokeWidth, options.strokeWidth, datum);
+        })
+        .attr('d', function(datum) {
+
+            var originXY, destXY;
+
+            originXY = self.latLngToXY(val(datum.origin.latitude, datum), val(datum.origin.longitude, datum))
+
+            destXY = self.latLngToXY(val(datum.destination.latitude, datum), val(datum.destination.longitude, datum));
+
+            var midXY = [ (originXY[0] + destXY[0]) / 2, (originXY[1] + destXY[1]) / 2];
+            if (options.greatArc) {
+                  // TODO: Move this to inside `if` clause when setting attr `d`
+              var greatArc = d3.geo.greatArc()
+                  .source(function(d) { return [val(d.origin.longitude, d), val(d.origin.latitude, d)]; })
+                  .target(function(d) { return [val(d.destination.longitude, d), val(d.destination.latitude, d)]; });
+
+              return path(greatArc(datum))
+            }
+            var sharpness = val(datum.arcSharpness, options.arcSharpness, datum);
+            return "M" + originXY[0] + ',' + originXY[1] + "S" + (midXY[0] + (50 * sharpness)) + "," + (midXY[1] - (75 * sharpness)) + "," + destXY[0] + "," + destXY[1];
+        })
+        .attr('data-info', function(datum) {
+          return JSON.stringify(datum);
+        })
+        .on('mouseover', function ( datum ) {
+          var $this = d3.select(this);
+
+          if (options.popupOnHover) {
+            self.updatePopup($this, datum, options, svg);
+          }
+        })
+        .on('mouseout', function ( datum ) {
+          var $this = d3.select(this);
+
+          d3.selectAll('.datamaps-hoverover').style('display', 'none');
+        })
+        .transition()
+          .style('fill', function(datum, i) {
+            /*
+              Thank you Jake Archibald, this is awesome.
+              Source: http://jakearchibald.com/2013/animated-line-drawing-svg/
+            */
+            var length = this.getTotalLength();
+            this.style.transition = this.style.WebkitTransition = 'none';
+            this.style.strokeDasharray = length + ' ' + length;
+            this.style.strokeDashoffset = length;
+            this.getBoundingClientRect();
+            this.style.transition = this.style.WebkitTransition = 'stroke-dashoffset ' + val(datum.animationSpeed, options.animationSpeed, datum) + 'ms ' + datum.delay*1000 + 'ms ease-out';
+            this.style.strokeDashoffset = '0';
+            return 'none';
+          });
+
+    arcs.exit()
+      .transition()
+      .duration(1000)
+      .style('opacity', 0)
+      .remove();
+  }
+
+var main_data = [];
+var prev_data = 0;
+
+// Configuration of datamap canvas
+// Futher reading can be found at https://datamaps.github.io/
+function Datamap() {
+  this.$container = $("#container");
+  instance = this.instance = new Datamaps({
+    scope: 'world',
+    element: this.$container.get(0),
+    done: this._handleMapReady.bind(this),
+    projection: 'mercator',
+        fills: {
+          /*defaultFill: '#454545'*/
+          defaultFill: 'black'
+        },
+        geographyConfig: {
+          hideAntarctica: true,
+          borderColor: '#0062a2',
+          highlightFillColor: '#4b4d4a',
+          highlightBorderColor: '#fdfdfd',
+          popupOnHover: true,
+          popupTemplate: function(geography, data) {
+          return '<div class="hoverinfo" id="country">' + geography.properties.name + '</div>';
+        },
+        },
+        ph_arcConfig: {
+          strokeColor: '#0062a2',
+          strokeWidth: 2,
+          arcSharpness: 2, /* 5 */
+          animationSpeed: 3000, // Milliseconds
+          popupOnHover: true,
+          // Case with latitude and longitude
+          popupTemplate: function(geography, data) {
+            if ( ( data.origin && data.destination ) && data.origin.latitude && data.origin.longitude && data.destination.latitude && data.destination.longitude ) {
+              // Content of info table
+              str = '<div class="hoverinfo"><table id="event"><tr><th>Warden Event</th></tr><tr><td>Type</td><td>'+ JSON.stringify(data.event) +'</td></tr><tr><td>Detect Time</td><td>'+ JSON.stringify(data.time) +'</td></tr><tr><th>Event origin</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.origin.ip) +  '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.origin.city) + ',&nbsp;' + JSON.stringify(data.origin.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.origin.latitude) + ',&nbsp;' + JSON.stringify(data.origin.longitude) + '</td></tr><tr><th>Event Destination</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.destination.ip) + '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.destination.city) + ',&nbsp;' + JSON.stringify(data.destination.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.destination.latitude) + ',&nbsp;' + JSON.stringify(data.destination.longitude) + '</td></tr></table></div>';
+              return str.replace(/&quot;/g,"");
+            }
+            // Missing information
+            else {
+              return '';
+            }
+          }
+        }
+      });
+
+  legend_data = d3.select("#legend")
+    .selectAll("li")
+    .data(Object.keys(cat_color).sort())
+    .enter()
+      .append("li")
+        .append("span")
+          .style("color", function(datum) { return cat_color[datum]})
+          .text(function(datum) { return datum; })
+          .append("span")
+          .text(function(datum) { return " – " + cat_desc[datum]})
+          .style("color", "white");
+  
+    instance.addPlugin('ph_arc', handleArcs);
+
+  setInterval(function(){
+      d3.json("./warden-map.json", function(error, data) {
+          if (data) {
+              var cur_data = data.pop()
+              var cur_time = new Date().getTime();
+              if (cur_data != prev_data) {
+                  prev_data = cur_data;
+                  for (var i=0; i<data.length; i++) {
+                      data[i].arrivalTime = cur_time;
+                      data[i].delay = i/data.length;
+                  }
+                  main_data = main_data.concat(data);
+              }
+          }
+          var trimmed_data = [];
+          for (var i=0; i<main_data.length; i++) {
+              if (main_data[i].arrivalTime + 3500 > cur_time) {
+                  trimmed_data.push(main_data[i]);
+              }
+          }
+          main_data = trimmed_data;
+          trimmed_data = cur_time = cur_data = error = data = null;
+          instance.ph_arc(main_data);
+      });
+  }, 1000);
+
+};
+
+Datamap.prototype._handleMapReady = function(datamap) {
+  this.zoom = new Zoom({
+    $container: this.$container,
+    datamap: datamap
+  });
+}

contrib/map/frontend/warden-map.html

+<!--                                                                                   -->
+<!--                                                                                   -->
+<!-- -*- coding: utf-8 -*-                                                             -->
+<!--                                                                                   -->
+<!--  warden-map.html                                                                  -->
+<!--                                                                                   -->
+<!-- Copyright (C) 2016 Cesnet z.s.p.o                                                 -->
+<!-- Use of this source is governed by a 3-clause BSD-style license, see LICENSE file. -->
+<!--                                                                                   -->
+<!--                                                                                   -->
+
+
+<!DOCTYPE html>
+<meta name="robots" content="noindex">
+<meta charset="utf-8">
+<link href='https://fonts.googleapis.com/css?family=Oswald&amp;subset=latin,latin-ext' rel='stylesheet' type='text/css'>
+<link rel="stylesheet" type="text/css" href="./css/warden-map.css"/>
+<body>
+
+<script src="https://d3js.org/d3.v3.min.js"></script>
+<script src="https://d3js.org/topojson.v1.min.js"></script>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"></script>
+<script src="./js/datamaps.world.min.js"></script>
+<script src="./js/warden-map.js"></script>
+
+<!--
+<h2>Warden Map</h2>
+<div id="tools">
+  <button class="zoom-button" data-zoom="reset">&#x2302</button>
+  <button class="zoom-button" data-zoom="out">-</button>
+  <button class="zoom-button" data-zoom="in">+</button>
+  <div id="zoom-info"></div>
+</div>
+-->
+<div id="container"></div>
+<div id="heading">Attacks, detected in CESNET network<br/>
+SABU - Sharing and Analysis of Security Events
+</div>
+<div id="legend-box">
+	<p><b>Reported to Warden right <i>now</i>.</b></p>
+	<ul id="legend"></ul>
+</div>
+
+<!-- Draw datamap into id="container" -->
+<script>new Datamap();</script>
+
+</body>
+</html>

warden3/warden_client/README → warden_client/README

-+---------------------------+
-| Warden Client Library 3.0 |
-+---------------------------+
++---------------------------------+
+| Warden Client Library 3.0-beta3 |
++---------------------------------+
 
 Content
 
@@ -15,7 +15,7 @@ A. Introduction
 
    The main goal of Warden 3 is to address the shortcomings, which emerged
 during several years of Warden 2.X operation. Warden 3 uses flexible and
-descriptive event format, based on JSON. Warden 3 protocol is based on  plain
+descriptive event format, based on JSON. Warden 3 protocol is based on plain
 HTTPS queries with help of JSON (Warden 2 SOAP is heavyweight, outdated and
 draws in many dependencies). Clients can be multilanguage, unlike SOAP/HTTPS,
 plain HTTPS and JSON is mature in many mainstream programming languages.
@@ -36,13 +36,14 @@ B. Quick start (TL;DR)
    sandbox URL, etc.
    If succesful, you will receive authentication secret.
  * Use warden_curl_test.sh to check you are able to talk to server.
- * See warden_client_examples.py on how to integrate sending/recieving
+ * See warden_client_examples.py on how to integrate sending/receiving
    into your Python application.
  * Alternatively, check 'contrib' directory in Warden GIT for various
    ready to use tools or recipes. You may find senders for various
    honeypots, or warden_filer may get handy if you do not want to delve
    into Python at all.
- * Welcome! Thanks for your security data, and use others' for common good.
+ * Welcome! Thanks for participating in in the data exchange to improve
+   network security awareness.
 
 ------------------------------------------------------------------------------
 C. Concepts
@@ -52,7 +53,7 @@ C.1. Event description format
    IDEA - Intrusion Detection Extensible Alert, flexible extensible format
 for security events, see:
 
-   https://csirt.cesnet.cz/IDEA
+   https://idea.cesnet.cz/
 
 C.2. Event serial ID
 
@@ -64,7 +65,7 @@ C.3. Authentication
 
    In Warden 2, clients get authenticated by server certificate, however
 server certificate is usually same for the whole machine, so individual
-clients are differentiated only by telling its own name. However, client name
+clients are differentiated only by telling their own name. However, client name
 is widely known, so this allows for client impersonation within one machine.
 Warden 3 slightly improves this schema by replacing client name in
 authentication phase by "secret", random string, shared among particular
@@ -133,7 +134,7 @@ sending events). The keys of the object, which may be available, are:
     description.
 
    Client errors (4xx) are considered permanent - client must not try to send
-same event again as it will get always rejected - client administrator
+same event again as it will always get rejected - client administrator
 will need to inspect logs and rectify the cause.
 
    Server errors (5xx) may be considered by client as temporary and client is
@@ -290,7 +291,7 @@ $ curl \
     --request POST \
     "https://warden.example.org/getInfo?secret=SeCrEt"
 
-{"version": "3.0-beta1",
+{"version": "3.0",
  "send_events_limit": 500,
  "get_events_limit": 1000,
  "description": "Warden 3 server"}
@@ -464,4 +465,4 @@ for e in res:
 debug_str() output increasingly more detailed info.
 
 ------------------------------------------------------------------------------
-Copyright (C) 2011-2015 Cesnet z.s.p.o
+Copyright (C) 2011-2022 Cesnet z.s.p.o

warden3/warden_client/warden_client.cfg.dist → warden_client/warden_client.cfg.dist

 {
     "url": "https://warden-hub.example.org/warden3",
-
     "certfile": "cert.pem",
     "keyfile": "key.pem",
-    "cafile": "cacert.pem",
-
-    "timeout": 60,
-    "get_events_limit": 1000,
-
-    "errlog": {"level": "debug"},
-    "filelog": {"file": "warden_client.log", "level": "warning"},
-    //"syslog": {"socket": "/dev/log", "facility": "local7", "level": "warning"},
-
-    "idstore": "warden_client.id",
-
+    "filelog": {"level": "debug"},
     "name": "org.example.warden_client",
-
     "secret": "ToP_SeCrEt"
 }

warden3/warden_client/warden_client.py → warden_client/warden_client.py

@@ -4,16 +4,34 @@
 # Copyright (C) 2011-2015 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-import json, httplib, ssl, socket, logging, logging.handlers, time
-from urlparse import urlparse
-from urllib import urlencode
-from sys import stderr, exc_info
+import hashlib  # Some Python/ssl versions incorrectly initialize hashes, this helps
+from sys import stderr, exc_info, version_info
+import json, ssl, socket, logging, logging.handlers, time
 from traceback import format_tb
 from os import path
 from operator import itemgetter
 
+fix_logging_filename = str if version_info<(2, 7) else lambda x: x
 
-VERSION = "3.0-beta1"
+if version_info[0] >= 3:
+    import http.client as httplib
+    from urllib.parse import urlparse
+    from urllib.parse import urlencode
+    basestring = str
+else:
+    import httplib
+    from urlparse import urlparse
+    from urllib import urlencode
+
+
+
+VERSION = "3.0-beta3"
+
+DEFAULT_CA_STORES = [
+    "/etc/ssl/certs/ca-certificates.crt",       # Deb
+    "/etc/pki/tls/certs/ca-bundle.crt",         # RH
+    "/var/lib/ca-certificates/ca-bundle.pem"    # SuSE
+ ]
 
 class HTTPSConnection(httplib.HTTPSConnection):
     '''
@@ -21,11 +39,12 @@ class HTTPSConnection(httplib.HTTPSConnection):
     of SSL/ TLS version and cipher selection.  See:
     http://hg.python.org/cpython/file/c1c45755397b/Lib/httplib.py#l1144
     and `ssl.wrap_socket()`
+    Used only if ssl.SSLContext is not available (Python version < 2.7.9)
     '''
     def __init__(self, host, **kwargs):
-        self.ciphers = kwargs.pop('ciphers',None)
-        self.ca_certs = kwargs.pop('ca_certs',None)
-        self.ssl_version = kwargs.pop('ssl_version',ssl.PROTOCOL_SSLv23)
+        self.ciphers = kwargs.pop('ciphers', None)
+        self.ca_certs = kwargs.pop('ca_certs', None)
+        self.ssl_version = kwargs.pop('ssl_version', getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23))
 
         httplib.HTTPSConnection.__init__(self,host,**kwargs)
 
@@ -102,6 +121,14 @@ class Error(Exception):
                 kwargs["send_events_limit"] = int(kwargs["send_events_limit"])
             except Exception:
                 del kwargs["send_events_limit"]
+        if "exc" in kwargs:
+            # Traceback objects cause reference loops, so memory may be not
+            # correctly free'd. We only need traceback to log it in str_debug(),
+            # so let's get the string representation now and forget the 
+            # traceback object, thus preventing the loop.
+            exctype, excvalue, tb = kwargs["exc"]
+            tb = format_tb(tb)
+            kwargs["exc"] = exctype, excvalue, tb
         self.errors.append(kwargs)
 
 
@@ -132,6 +159,8 @@ class Error(Exception):
         """ In list or iterable context we're empty """
         raise StopIteration
 
+    __next__ = next
+
 
     def __bool__(self):
         """ In boolean context we're never True """
@@ -191,10 +220,10 @@ class Error(Exception):
         out.append(self.str_preamble(e))
         if not "exc" in e or not e["exc"]:
             return ""
-        exc_tb = e["exc"][2]
+        exc_tb = e["exc"][2] # exc_tb is string repr. of traceback object
         if exc_tb:
             out.append("Traceback:\n")
-            out.extend(format_tb(exc_tb))
+            out.extend(exc_tb)
         return "".join(out)
 
 
@@ -207,8 +236,8 @@ class Client(object):
             keyfile=None,
             cafile=None,
             timeout=60,
-            retry=3,
-            pause=5,
+            retry=20,
+            pause=10,
             get_events_limit=6000,
             send_events_limit=500,
             errlog={},
@@ -229,9 +258,9 @@ class Client(object):
         self.conn = None
 
         base = path.join(path.dirname(__file__))
-        self.certfile = path.join(base, certfile or "cert.pem")
-        self.keyfile  = path.join(base, keyfile or "key.pem")
-        self.cafile = path.join(base, cafile or "ca.pem")
+        self.certfile = self.get_readable_file(certfile or "cert.pem", base)
+        self.keyfile  = self.get_readable_file(keyfile or "key.pem", base)
+        self.cafile = self.get_readable_file(cafile if cafile is not None else DEFAULT_CA_STORES, base)
         self.timeout = int(timeout)
         self.get_events_limit = int(get_events_limit)
         self.idstore = path.join(base, idstore) if idstore is not None else None
@@ -240,12 +269,42 @@ class Client(object):
         self.retry = int(retry)
         self.pause = int(pause)
 
-        self.ciphers = 'TLS_RSA_WITH_AES_256_CBC_SHA'
-        self.sslversion = ssl.PROTOCOL_TLSv1
+        self.ciphers = None
+        self.sslversion = getattr(ssl, "PROTOCOL_TLS", ssl.PROTOCOL_SSLv23)
+
+        # If Python is new enough to have SSLContext, use it for SSL settings,
+        # otherwise our own class derived from httplib.HTTPSConnection is used
+        # later in connect().
+        if hasattr(ssl, 'SSLContext'):
+            self.sslcontext = ssl.SSLContext(self.sslversion)
+            self.sslcontext.load_cert_chain(self.certfile, self.keyfile)
+            if self.cafile:
+                self.sslcontext.load_verify_locations(self.cafile)
+                self.sslcontext.verify_mode = ssl.CERT_REQUIRED
+            else:
+                self.sslcontext.verify_mode = ssl.CERT_NONE
+        else:
+            self.sslcontext = None
 
         self.getInfo()  # Call to align limits with server opinion
 
 
+    def get_readable_file(self, name, base):
+        names = [name] if isinstance(name, basestring) else name
+        names = [path.join(base, n) for n in names]
+        errors = []
+        for n in names:
+            try:
+                open(n, "r").close()
+                self.logger.debug("Using %s" % n)
+                return n
+            except IOError as e:
+                errors.append(e)
+        for e in errors:
+            self.logger.error(str(e))
+        return names[0] if names else None
+
+
     def init_log(self, errlog, syslog, filelog):
 
         def loglevel(lev):
@@ -281,7 +340,8 @@ class Client(object):
                 fl = logging.FileHandler(
                     filename=path.join(
                         path.dirname(__file__),
-                        filelog.get("file", "%s.log" % self.name)))
+                        filelog.get("file", "%s.log" % self.name)),
+                        encoding="utf-8")
                 fl.setLevel(loglevel(filelog.get("level", "debug")))
                 fl.setFormatter(format_time)
                 self.logger.addHandler(fl)
@@ -291,7 +351,7 @@ class Client(object):
         if syslog is not None:
             try:
                 sl = logging.handlers.SysLogHandler(
-                    address=syslog.get("socket", "/dev/log"),
+                    address=fix_logging_filename(syslog.get("socket", "/dev/log")),
                     facility=facility(syslog.get("facility", "local7")))
                 sl.setLevel(loglevel(syslog.get("level", "debug")))
                 sl.setFormatter(format_notime)
@@ -316,19 +376,23 @@ class Client(object):
 
         try:
             if self.url.scheme=="https":
-                conn = HTTPSConnection(
-                    self.url.netloc,
-                    strict = False,
-                    key_file = self.keyfile,
-                    cert_file = self.certfile,
-                    timeout = self.timeout,
-                    ciphers = self.ciphers,
-                    ca_certs = self.cafile,
-                    ssl_version = self.sslversion)
+                if self.sslcontext:
+                    conn = httplib.HTTPSConnection(
+                        self.url.netloc,
+                        timeout = self.timeout,
+                        context = self.sslcontext)
+                else:
+                    conn = HTTPSConnection(
+                        self.url.netloc,
+                        key_file = self.keyfile,
+                        cert_file = self.certfile,
+                        timeout = self.timeout,
+                        ciphers = self.ciphers,
+                        ca_certs = self.cafile,
+                        ssl_version = self.sslversion)
             elif self.url.scheme=="http":
                 conn = httplib.HTTPConnection(
                     self.url.netloc,
-                    strict = False,
                     timeout = self.timeout)
             else:
                 return Error(message="Don't know how to connect to \"%s\"" % self.url.scheme,
@@ -354,27 +418,30 @@ class Client(object):
             kwargs["secret"] = self.secret
 
         if kwargs:
-            for k in kwargs.keys():
+            for k in list(kwargs.keys()):
                 if kwargs[k] is None:
                     del kwargs[k]
             argurl = "?" + urlencode(kwargs, doseq=True)
         else:
             argurl = ""
 
-        try:
-            if payload is None:
-                data = ""
-            else:
+        self.headers = {"Accept": "application/json"}
+        data = None
+
+        if payload is None:
+            method = "GET"
+        else:
+            method = "POST"
+            try:
                 data = json.dumps(payload)
-        except:
-            return Error(message="Serialization to JSON failed",
-                exc=exc_info(), method=func, payload=payload)
+            except:
+                return Error(message="Serialization to JSON failed",
+                    exc=exc_info(), method=func, payload=payload)
 
-        self.headers = {
-            "Content-Type": "application/json",
-            "Accept": "application/json",
-            "Content-Length": str(len(data))
-        }
+            self.headers.update({
+                "Content-Type": "application/json",
+                "Content-Length": str(len(data))
+            })
 
         # HTTP(S)Connection is oneshot object (and we don't speak "pipelining")
         conn = self.connect()
@@ -383,7 +450,7 @@ class Client(object):
 
         loc = '%s/%s%s' % (self.url.path, func, argurl)
         try:
-            conn.request("POST", loc, data, self.headers)
+            conn.request(method, loc, data, self.headers)
         except:
             conn.close()
             return Error(message="Sending of request to server failed",
@@ -407,13 +474,13 @@ class Client(object):
 
         if res.status==httplib.OK:
             try:
-                data = json.loads(response_data)
+                data = json.loads(response_data.decode("utf-8"))
             except:
                 data = Error(method=func, message="JSON message parsing failed",
                     exc=exc_info(), response=response_data)
         else:
             try:
-                data = json.loads(response_data)
+                data = json.loads(response_data.decode("utf-8"))
                 data["errors"]   # trigger exception if not dict or no error key
             except:
                 data = Error(method=func, message="Generic server HTTP error",
@@ -505,7 +572,7 @@ class Client(object):
         """ Send out "events" list to server, retrying on server errors.
         """
         ev = events
-        idx_xlat = range(len(ev))
+        idx_xlat = list(range(len(ev)))
         err = Error()
         retry = retry or self.retry
         attempt = retry
@@ -523,7 +590,7 @@ class Client(object):
                 res.errors.sort(key=itemgetter("error"))
                 for e in res.errors:
                     errno = e["error"]
-                    evlist = e.get("events", range(len(ev)))   # none means all
+                    evlist = e.get("events", list(range(len(ev))))   # none means all
                     if errno < 500 or not attempt:
                         # Fatal error or last try, translate indices
                         # to original and prepare for returning to caller
@@ -546,11 +613,11 @@ class Client(object):
             tag=None, notag=None,
             group=None, nogroup=None):
 
-        if not id:
+        if id is None:
             id = self._loadID(idstore)
 
         res = self.sendRequest(
-            "getEvents", id=id, count=count or self.get_events_limit, cat=cat,
+            "getEvents", id=id, count=self.get_events_limit if count is None else count, cat=cat,
             nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
 
         if res:
@@ -598,7 +665,6 @@ def format_time(year, month, day, hour, minute, second, microsec=0, utcoffset=No
 
 
 def read_cfg(cfgfile):
-    abspath = path.join(path.dirname(__file__), cfgfile)
-    with open(abspath, "r") as f:
+    with open(cfgfile, "r") as f:
         stripcomments = "\n".join((l for l in f if not l.lstrip().startswith(("#", "//"))))
         return json.loads(stripcomments)

warden3/warden_client/warden_client_examples.py → warden_client/warden_client_examples.py

@@ -4,16 +4,13 @@
 # Copyright (C) 2011-2015 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-from warden_client import Client, Error, read_cfg, format_timestamp
-import json
 import string
-from time import time, gmtime
-from math import trunc
+from time import time
 from uuid import uuid4
 from pprint import pprint
-from os import path
-from random import randint, randrange, choice, random;
-from base64 import b64encode;
+from random import randint, randrange, choice, random
+from base64 import b64encode
+from warden_client import Client, Error, read_cfg, format_timestamp
 
 def gen_min_idea():
 
@@ -56,7 +53,7 @@ def gen_random_idea(client_name="cz.example.warden.test"):
     def randip6():
         return [rand6ip, geniprange(rand6ip), rand6cidr][randint(0, 2)]()
 
-    def randstr(charlist=string.letters, maxlen=32, minlen=1):
+    def randstr(charlist=string.ascii_letters, maxlen=32, minlen=1):
         return ''.join(choice(charlist) for i in range(randint(minlen, maxlen)))
 
     event = {
@@ -106,7 +103,7 @@ def gen_random_idea(client_name="cz.example.warden.test"):
              "Size": 46,
              "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr())],
              "ContentEncoding": "base64",
-             "Content": b64encode(randstr())
+             "Content": b64encode(randstr().encode('ascii')).decode("ascii")
           }
        ],
        "Node": [
@@ -139,7 +136,7 @@ def main():
     #     idstore="MyClient.id",
     #     name="cz.example.warden.test")
 
-    print "=== Debug ==="
+    print("=== Debug ===")
     info = wclient.getDebug()
     pprint(info)
 
@@ -150,18 +147,18 @@ def main():
     # If you want just to be informed, this is not necessary, just
     # configure logging correctly and check logs.
     if isinstance(info, Error):
-        print info
+        print(info)
 
-    print "=== Server info ==="
+    print("=== Server info ===")
     info = wclient.getInfo()
 
-    print "=== Sending 10 event(s) ==="
+    print("=== Sending 10 event(s) ===")
     start = time()
     ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)])
-    print ret
-    print "Time: %f" % (time()-start)
+    print(ret)
+    print("Time: %f" % (time()-start))
 
-    print "=== Getting 10 events ==="
+    print("=== Getting 10 events ===")
     start = time()
 
     # cat = ['Availability', 'Abusive.Spam','Attempt.Login']
@@ -182,10 +179,10 @@ def main():
     nogroup = []
 
     ret = wclient.getEvents(count=10, cat=cat, nocat=nocat, tag=tag, notag=notag, group=group, nogroup=nogroup)
-    print "Time: %f" % (time()-start)
-    print "Got %i events" % len(ret)
+    print("Time: %f" % (time()-start))
+    print("Got %i events" % len(ret))
     for e in ret:
-        print e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name")
+        print(e.get("Category"), e.get("Node")[0].get("Type"), e.get("Node")[0].get("Name"))
 
 if __name__ == "__main__":
     main()

warden3/warden_client/warden_curl_test.sh → warden_client/warden_curl_test.sh

@@ -23,7 +23,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/blefub?client=$client&secret=$secret"
 echo
 
@@ -33,7 +32,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/?client=$client&secret=$secret"
 echo
 
@@ -43,7 +41,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client"
 echo
 
@@ -53,7 +50,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents"
 echo
 
@@ -63,7 +59,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=asdf.blefub"
 echo
 
@@ -73,7 +68,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=asdf.blefub&secret=$secret"
 echo
 
@@ -83,7 +77,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=ASDFblefub"
 echo
 
@@ -93,7 +86,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?secret=$secret"
 echo
 
@@ -114,7 +106,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&cat=bflm"
 echo
 
@@ -124,7 +115,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&cat=Other&nocat=Test"
 echo
 
@@ -145,7 +135,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&self=test"
 echo
 
@@ -155,7 +144,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&bad=guy"
 echo
 
@@ -165,7 +153,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret"
 echo
 
@@ -175,7 +162,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getEvents?client=$client&secret=$secret&count=3&id=10"
 echo
 
@@ -185,7 +171,6 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getDebug?client=$client&secret=$secret"
 echo
 
@@ -195,6 +180,5 @@ curl \
     --cert $certfile \
     --cacert $cafile \
     --connect-timeout 3 \
-    --request POST \
     "$url/getInfo?client=$client&secret=$secret"
 echo

warden3/contrib/warden_filer/README → warden_filer/README

-+---------------------------------+
-| Warden Filer 0.1 for Warden 3.X |
-+---------------------------------+
++---------------------------------------+
+| Warden Filer 3.0-beta3 for Warden 3.X |
++---------------------------------------+
 
 Content
 
@@ -49,15 +49,21 @@ C. Usage
      -c CONFIG, --config CONFIG
                            configuration file path
      --oneshot             don't daemonise, run just once
+     -d, --daemon          daemonize
+     -p PID_FILE, --pid_file PID_FILE
+                           create PID file with this name
 
 
    CONFIG denotes path to configuration file, default is warden_filer.cfg in
 current directory.
-   --oneshot prevents daemonizing, Filer just does its work once (fetches
-available events or sends event files present in directory), but obeys
-all other applicable options from configuration file (concerning logging,
-filtering, directories, etc.)
-   Without --oneshot Filer goes to full unix daemon mode.
+   --oneshot instructs Filer to just do its work once (fetch available events
+or send event files present in directory), but obeys all other applicable
+options from configuration file (concerning logging, filtering, directories,
+etc.)
+   --daemon instructs Filer to go to full unix daemon mode. Without it,
+Filer just stays on foreground.
+   --pid_file makes Filer to create the usual PID file. Without it, no PID
+file gets created.
 
 ------------------------------------------------------------------------------
 D. Configuration
@@ -71,12 +77,25 @@ JSON object, containing configuration. See also warden_filer.cfg as example.
    sender - configuration section for sender mode
       dir - directory, whose "incoming" subdir will be checked for Idea
             events to send out
+      done_dir - directory, into which the messages will be moved after
+            successful sending. If not set, processed messages will get
+            deleted, which is default, and usually what you want. Note that
+            this is just regular directory, no special locking precautions
+            and no subdirectories are done here, however if "done_dir" is on
+            the same filesystem as "dir"
       filter - filter fields (same as in Warden query, see Warden and Idea
-               doc, possible keys: cat, nocat, group, nogroup, tag, notag),
-               unmatched events get discarded and deleted
+            doc, possible keys: cat, nocat, group, nogroup, tag, notag),
+            unmatched events get discarded and deleted
       node - o information about detector to be prepended into event Node
-             array (see Idea doc). Note that Warden server may require it
-             to correspond with client registration
+            array (see Idea doc). Note that Warden server may require it to
+            correspond with client registration
+      poll_time - how often to check incoming directory (in seconds, defaults
+            to 5)
+      owait_timeout - how long to opportunistically wait for possible new
+            incoming files when number of files to process is less than
+            send_events_limit (in seconds, defaults to poll_time)
+      owait_poll_time - how often to check incoming directory during
+            opportunistic timeout (in seconds, defaults to 1)
    receiver - configuration section for receiver mode
       dir - directory, whose "incoming" subdir will serve as target for events
       filter - filter fields for Warden query (see Warden and Idea doc,
@@ -85,6 +104,21 @@ JSON object, containing configuration. See also warden_filer.cfg as example.
              array (see Idea doc). Be careful here, you may ruin Idea
              messages by wrongly formatted data and they are not checked
              here in any way
+      poll_time - how often to check Warden server for new events (in seconds,
+             defaults to 5)
+      file_limit - limit number of files in "incoming" directory. When the limit
+             is reached, polling is paused for "limit_wait_time" seconds
+      limit_wait_time - wait this number of seconds if limit on number of files
+             is reached (defaults to 5)
+      
+
+    Both the "sender" and "reciever" sections can also bear daemon
+configuration.
+
+      work_dir - where should daemon chdir
+      chroot_dir - confine daemon into chroot directory
+      umask - explicitly set umask for created files
+      uid, gid - uid/gid, under which daemon will run
 
 ------------------------------------------------------------------------------
 E. Directories and locking issues
@@ -95,39 +129,45 @@ confusion. Simple path suffers locking issue: when one process saves file
 there, another process has no way to know whether file is already complete
 or not, and starting to read prematurely can lead to corrupted data read.
 Also, two concurrent processes may decide to work on one file, stomping on
-others legs. So, your scripts and tools inserting data or taking data from
-working directories must obey simple protocols.
+others legs.
+   So, your scripts and tools inserting data or taking data from working
+directories must obey simple protocols, which use atomic "rename" to avoid
+locking issues.
+   Also, your directory (and its structure) _must_ reside on the same
+filesystem to keep "rename" atomic. _Never_ try to mount some of the
+subdirectories ("tmp", "incoming", "errors") from other filesystem.
 
  1. Inserting file
 
-   * Use "temp" subdirectory to create new file; filename is arbitrary, but
-     must be unique among all subdirectories. 
+   * The file you want to create _must_ be created in the "tmp" subdirectory
+     first, _not_ "incoming". Filename is arbitrary, but must be unique among
+     all subdirectories.
 
    * When done writing, rename the file into "incoming" subdir. Rename is
      atomic operation, so for readers, file will appear either nonexistent
      or complete.
 
    For simple usage (bash scripts, etc.), just creating sufficiently random
-   filename in "temp" and then moving into "incoming" may be enough.
+   filename in "tmp" and then moving into "incoming" may be enough.
    Concatenating $RANDOM couple of times will do. :)
    
    For advanced or potentially concurrent usage inserting enough of unique
    information into name is recommended - Filer itself uses hostname, pid,
    unixtime, milliseconds, device number and file inode number to avoid
    locking issues both on local and network based filesystems and to be
-   prepared for hight traffic.
+   prepared for high traffic.
 
  2. Picking up file
 
-   * Rename the file to work with into "temp" directory.
+   * Rename the file to work with into "tmp" directory.
 
    * Do whatever you want with contents, and when finished, rename file back
      into "incoming", or remove, or move somewhere else, or move into "errors"
-     directory, after all, it's your file.
+     directory - what suits your needs, after all, it's your file.
 
    Note that in concurrent environment file can disappear between directory
    enumeration and attempt to rename - then just pick another one (and
-   repeat), someone was swifter.)
+   possibly repeat), someone was swifter.
 
 ------------------------------------------------------------------------------
 Copyright (C) 2011-2015 Cesnet z.s.p.o

warden_filer/check_file_count

+#!/bin/bash
+
+script=${0##*/}
+warn=0
+crit=65536
+read -rd '' helps <<EOF
+$script: Icinga plugin to check too high number of files in directory.
+
+Usage: $script -d dir [-w num] [-c num] [-h ]
+
+  -d dir    directory to watch
+  -w num    warning if number of files exceeds this value (default $warn)
+  -c num    critical if number of files exceeds this value (default $crit)
+EOF
+
+function bailout {
+	echo -n "$script" | tr '[:lower:]' '[:upper:]'
+	echo " $2 $3"
+	exit "$1"
+}
+
+while getopts hvVd:w:c: opt; do
+	case "$opt" in
+		h) bailout 3 "UNKNOWN" "$helps";;
+		d) dir="$OPTARG";;
+		w) warn="$OPTARG";;
+		c) crit="$OPTARG";;
+		"?") bailout 3 "UNKNOWN" "Unknown option, use -h for help";;
+	esac
+done
+
+[ -z "$dir" ] && bailout 3 "UNKNOWN" "-d not specified"
+
+count=$(find "$dir" -mindepth 1 -maxdepth 1 | wc -l)
+
+[ "$count" -gt "$crit" ] && bailout 2 "CRIT" "$count"
+
+[ "$count" -gt "$warn" ] && bailout 1 "WARN" "$count"
+
+bailout 0 "OK" "$count"

warden3/contrib/warden_filer/warden_filer.cfg → warden_filer/warden_filer.cfg.dist

+// For all options see documentation
 {
     // Warden config can be also referenced as:
     // "warden": "/path/to/warden_client.cfg"
     "warden": {
         "url": "https://example.com/warden3",
         "cafile": "tcs-ca-bundle.pem",
-        "timeout": 10,
-        "errlog": {"level": "debug"},
+        "keyfile": "my.key.pem",
+        "certfile": "my.cert.pem",
+        "timeout": 60,
+        "retry": 20,
+        "pause": 5,
         "filelog": {"level": "debug"},
-        "idstore": "myclient.id",
         "name": "com.example.warden.test",
         "secret": "SeCrEt"
     },
@@ -16,42 +19,38 @@
         // for Idea events to send out
         "dir": "warden_sender",
         // Optional filter fields, unmatched events are discarded (and removed)
-        "filter": {
-            "cat": ["Test", "Recon.Scanning"],
-            "nocat": null,
-            "group": ["cz.example"],
-            "nogroup": null,
-            "tag": null,
-            "notag": ["Honeypot"]
-        },
+        //"filter": {
+        //    "cat": ["Test", "Recon.Scanning"],
+        //    "nocat": null,
+        //    "group": ["cz.example"],
+        //    "nogroup": null,
+        //    "tag": null,
+        //    "notag": ["Honeypot"]
+        //},
         // Optional information about detector to be prepended into Idea Node array
-        "node": {
-            "Name": "cz.example.warden.test_sender",
-            "Type": ["Relay"],
-            "SW": ["warden_filer-sender"],
-            "AggrWin": "00:05:00",
-            "Note": "Test warden_filer sender"
-        }
+        //"node": {
+        //    "Name": "cz.example.warden.test_sender",
+        //    "Type": ["Relay"]
+        //}
     },
     "receiver": {
         // Maildir like directory, whose "incoming" will serve as target for events
         "dir": "warden_receiver",
         // Optional filter fields for Warden query
-        "filter": {
-            "cat": ["Test", "Recon.Scanning"],
-            "nocat": null,
-            "group": ["cz.cesnet"],
-            "nogroup": null,
-            "tag": null,
-            "notag": ["Honeypot"]
-        },
+        //"filter": {
+        //    "cat": ["Test", "Recon.Scanning"],
+        //    "nocat": null,
+        //    "group": ["cz.cesnet"],
+        //    "nogroup": null,
+        //    "tag": null,
+        //    "notag": ["Honeypot"]
+        //},
         // Optional information about detector to be prepended into Idea Node array
-        "node": {
-            "Name": "cz.example.warden.test_receiver",
-            "Type": ["Relay"],
-            "SW": ["warden_filer-receiver"],
-            "AggrWin": "00:05:00",
-            "Note": "Test warden_filer receiver"
-        }
+        //"node": {
+        //    "Name": "cz.example.warden.test_receiver",
+        //    "Type": ["Relay"]
+        //},
+        // Optional limit on number of files in "incoming" directory
+        //"file_limit": 10000
     }
 }

warden3/contrib/warden_filer/warden_filer.py → warden_filer/warden_filer.py

@@ -14,13 +14,19 @@ import socket
 import time
 import logging
 import signal
-import lockfile
+import resource
+import atexit
 import argparse
 from os import path, mkdir
-from random import choice, randint;
-from daemon import DaemonContext
-from daemon.pidlockfile import TimeoutPIDLockFile
+from random import choice, randint
 
+# for py2/py3 compatibility
+try:
+    basestring
+except NameError:
+    basestring = str
+
+VERSION = "3.0-beta3"
 
 class NamedFile(object):
     """ Wrapper class for file objects, which allows and tracks filename
@@ -65,9 +71,9 @@ class NamedFile(object):
 
 class SafeDir(object):
     """ Maildir like directory for safe file exchange.
-        - Producers are expected to drop files into "temp" under globally unique
+        - Producers are expected to drop files into "tmp" under globally unique
           filename and rename it into "incoming" atomically (newfile method)
-        - Workers pick files in "incoming", rename them into "temp",
+        - Workers pick files in "incoming", rename them into "tmp",
           do whatever they want, and either discard them or move into
           "errors" directory
     """
@@ -76,7 +82,7 @@ class SafeDir(object):
         self.path = self._ensure_path(p)
         self.incoming = self._ensure_path(path.join(self.path, "incoming"))
         self.errors = self._ensure_path(path.join(self.path, "errors"))
-        self.temp = self._ensure_path(path.join(self.path, "temp"))
+        self.temp = self._ensure_path(path.join(self.path, "tmp"))
         self.hostname = socket.gethostname()
         self.pid = os.getpid()
 
@@ -116,7 +122,7 @@ class SafeDir(object):
         #       which checked uniqueness among all directories by atomic
         #       links.
 
-        # First find and open name unique within temp
+        # First find and open name unique within tmp
         tmpname = None
         while not tmpname:
             tmpname = self._get_new_name()
@@ -138,46 +144,73 @@ class SafeDir(object):
         return [NamedFile(self.incoming, n) for n in os.listdir(self.incoming)]
 
 
+    def get_incoming_cnt(self):
+        """Get number of files in the incoming directory"""
+        return len(os.listdir(self.incoming))
+
+
 
 def receiver(config, wclient, sdir, oneshot):
     poll_time = config.get("poll_time", 5)
     node = config.get("node", None)
     conf_filt = config.get("filter", {})
+    file_limit = config.get("file_limit", None)
+    wait_time = config.get("limit_wait_time", 5)
     filt = {}
     # Extract filter explicitly to be sure we have right param names for getEvents
     for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"):
         filt[s] = conf_filt.get(s, None)
 
     while running_flag:
-        events = wclient.getEvents(**filt)
         count_ok = count_err = 0
-        while events:
-            for event in events:
-                if node:
-                    nodelist = event.setdefault("Node", [])
-                    nodelist.insert(0, node)
-                try:
-                    nf = None
-                    nf = sdir.newfile()
-                    with nf.f as f:
-                        data = json.dumps(event)
-                        f.write(data)
-                    nf.moveto(sdir.incoming)
-                    count_ok += 1
-                except Exception as e:
-                    Error(message="Error saving event", exc=sys.exc_info(), file=str(nf),
-                          event_ids=[event.get("ID")], sdir=sdir.path).log(wclient.logger)
-                    count_err += 1
+        limit_reached = False
+        if file_limit:
+            cnt_files = sdir.get_incoming_cnt() # Count files in 'incoming' dir
+            remain_to_limit = file_limit - cnt_files
+            # Query server, but not for more events than what can fit into limit
+            if remain_to_limit > 0:
+                events = wclient.getEvents(count=remain_to_limit, **filt)
+            else:
+                events = []
+            # Check whether limit was reached
+            if len(events) >= remain_to_limit:
+                limit_reached = True
+        else:
+            events = wclient.getEvents(**filt)
+
+        for event in events:
+            if node:
+                nodelist = event.setdefault("Node", [])
+                nodelist.insert(0, node)
+            try:
+                nf = None
+                nf = sdir.newfile()
+                with nf.f as f:
+                    data = json.dumps(event)
+                    f.write(data.encode('utf-8'))
+                nf.moveto(sdir.incoming)
+                count_ok += 1
+            except Exception as e:
+                Error(message="Error saving event", exc=sys.exc_info(), file=str(nf),
+                      event_ids=[event.get("ID")], sdir=sdir.path).log(wclient.logger)
+                count_err += 1
+
+        if events:
             wclient.logger.info(
                 "warden_filer: received %d, errors %d"
                 % (count_ok, count_err))
-            events = wclient.getEvents(**filt)
-            count_ok = count_err = 0
+
+        if limit_reached:
+            wclient.logger.info("Limit on number of files in 'incoming' dir reached.")
+
         if oneshot:
-            if not events:
+            if not events or limit_reached:
                 terminate_me(None, None)
         else:
-            time.sleep(poll_time)
+            if limit_reached:
+                time.sleep(wait_time)
+            elif not events:
+                time.sleep(poll_time)
 
 
 
@@ -212,73 +245,101 @@ def match_event(event, cat=None, nocat=None, tag=None, notag=None, group=None, n
 
 
 
+def get_dir_list(sdir, owait_poll_time, owait_timeout, nfchunk, oneshot):
+    nflist = sdir.get_incoming()
+    if oneshot and not nflist:
+        terminate_me(None, None)
+    timeout = time.time() + owait_timeout
+    while len(nflist)<nfchunk and time.time()<timeout and running_flag:
+        time.sleep(owait_poll_time)
+        nflist = sdir.get_incoming()
+    return nflist
+
+
+
 def sender(config, wclient, sdir, oneshot):
-    send_events_limit = config.get("send_events_limit", 500)
     poll_time = config.get("poll_time", 5)
+    owait_poll_time = config.get("owait_poll_time", 1)
+    owait_timeout = config.get("owait_timeout", poll_time)
     node = config.get("node", None)
+    done_dir = config.get("done_dir", None)
     conf_filt = config.get("filter", {})
     filt = {}
     # Extract filter explicitly to be sure we have right param names for match_event
     for s in ("cat", "nocat", "tag", "notag", "group", "nogroup"):
         filt[s] = conf_filt.get(s, None)
 
+    nfchunk = wclient.send_events_limit
     while running_flag:
-        nflist = sdir.get_incoming()
-        if oneshot:
+        nflist = get_dir_list(sdir, owait_poll_time, owait_timeout, nfchunk, oneshot)
+        if oneshot and not nflist:
             terminate_me(None, None)
         while running_flag and not nflist:
+            # No new files, wait and try again
             time.sleep(poll_time)
-            nflist = sdir.get_incoming()
-        # count chunk iterations rounded up
-        count = len(nflist)
-
-        events = []
-        nf_sent = []
-        count_ok = count_err = count_unmatched = 0
-        for nf in nflist:
-            # prepare event array from files
-            try:
-                nf.moveto(sdir.temp)
-            except Exception:
-                continue    # Silently go to next filename, somebody else might have interfered
-            try:
-                with nf.open("rb") as fd:
-                    data = fd.read()
-                    event = json.loads(data)
-                    if not match_event(event, **filt):
-                        wclient.logger.debug("Unmatched event: %s" % data)
-                        count_unmatched += 1
-                        nf.remove()
-                        continue
-                    if node:
-                        nodelist = event.setdefault("Node", [])
-                        nodelist.insert(0, node)
-                    events.append(event)
-                    nf_sent.append(nf)
-            except Exception as e:
-                Error(message="Error loading event", exc=sys.exc_info(), file=str(nf),
-                      sdir=sdir.path).log(wclient.logger)
-                nf.moveto(sdir.errors)
-
-        res = wclient.sendEvents(events)
-
-        if isinstance(res, Error):
-            for e in res.errors:
-                errno = e["error"]
-                evlist = e.get("events", range(len(nf_sent)))  # None means all
-                for i in evlist:
-                    if nf_sent[i]:
-                        nf_sent[i].moveto(sdir.errors)
-                        nf_sent[i] = None
-                        count_err += 1
-
-        # Cleanup rest - succesfully sent events
-        for name in nf_sent:
-            if name:
-                name.remove()
-                count_ok += 1
-        wclient.logger.info(
-            "warden_filer: saved %d, errors %d, unmatched %d" % (count_ok, count_err, count_unmatched))
+            nflist = get_dir_list(sdir, owait_poll_time, owait_timeout, nfchunk, oneshot)
+
+        # Loop over all chunks. However:
+        # - omit the last loop, if there is less data than the optimal window;
+        #   next get_dir_list will still get it again, possibly together with
+        #   new files, which may have appeared meanwhile
+        # - unless it's the sole loop (so that at least _something_ gets sent)
+        nfindex = 0
+        while nfindex<len(nflist) and ((len(nflist)-nfindex>=nfchunk) or not nfindex):
+            events = []
+            nf_sent = []
+            count_ok = count_err = count_unmatched = count_local = 0
+            for nf in nflist[nfindex:nfindex+nfchunk]:
+                # prepare event array from files
+                try:
+                    nf.moveto(sdir.temp)
+                except Exception:
+                    continue    # Silently go to next filename, somebody else might have interfered
+                try:
+                    with nf.open("rb") as fd:
+                        data = fd.read().decode('utf-8')
+                        event = json.loads(data)
+                        if not match_event(event, **filt):
+                            wclient.logger.debug("Unmatched event: %s" % data)
+                            count_unmatched += 1
+                            nf.remove()
+                            continue
+                        if node:
+                            nodelist = event.setdefault("Node", [])
+                            nodelist.insert(0, node)
+                        events.append(event)
+                        nf_sent.append(nf)
+                except Exception as e:
+                    Error(message="Error loading event", exc=sys.exc_info(), file=str(nf),
+                          sdir=sdir.path).log(wclient.logger)
+                    nf.moveto(sdir.errors)
+                    count_local += 1
+
+            res = wclient.sendEvents(events)
+
+            if isinstance(res, Error):
+                for e in res.errors:
+                    errno = e["error"]
+                    evlist = e.get("events", range(len(nf_sent)))  # None means all
+                    for i in evlist:
+                        if nf_sent[i]:
+                            nf_sent[i].moveto(sdir.errors)
+                            nf_sent[i] = None
+                            count_err += 1
+
+            # Cleanup rest - the succesfully sent events
+            for name in nf_sent:
+                if name:
+                    if done_dir:
+                        name.moveto(done_dir)
+                    else:
+                        name.remove()
+                    count_ok += 1
+            wclient.logger.info(
+                "warden_filer: saved %d, warden errors %d, local errors %d, unmatched %d" % (count_ok, count_err, count_local, count_unmatched))
+
+            nfindex += nfchunk  # skip to next chunk of files
+            nfchunk = wclient.send_events_limit # might get changed by server
 
 
 
@@ -294,6 +355,61 @@ def get_logger_files(logger):
 
 
 
+def daemonize(
+        work_dir = None, chroot_dir = None,
+        umask = None, uid = None, gid = None,
+        pidfile = None, files_preserve = [], signals = {}):
+    # Dirs, limits, users
+    if chroot_dir is not None:
+        os.chdir(chroot_dir)
+        os.chroot(chroot_dir)
+    if umask is not None:
+        os.umask(umask)
+    if work_dir is not None:
+        os.chdir(work_dir)
+    if gid is not None:
+        os.setgid(gid)
+    if uid is not None:
+        os.setuid(uid)
+    # Doublefork, split session
+    if os.fork()>0:
+        os._exit(0)
+    os.setsid()
+    if os.fork()>0:
+        os._exit(0)
+    # Setup signal handlers
+    for (signum, handler) in signals.items():
+        signal.signal(signum, handler)
+    # Close descriptors
+    descr_preserve = set(f.fileno() for f in files_preserve)
+    maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
+    if maxfd==resource.RLIM_INFINITY:
+        maxfd = 65535
+    for fd in range(maxfd, 3, -1):  # 3 means omit stdin, stdout, stderr
+        if fd not in descr_preserve:
+            try:
+                os.close(fd)
+            except Exception:
+                pass
+    # Redirect stdin, stdout, stderr to /dev/null
+    devnull = os.open(os.devnull, os.O_RDWR)
+    for fd in range(3):
+        os.dup2(devnull, fd)
+    # PID file
+    if pidfile is not None:
+        pidd = os.open(pidfile, os.O_RDWR|os.O_CREAT|os.O_EXCL|os.O_TRUNC)
+        os.write(pidd, (str(os.getpid())+"\n").encode())
+        os.close(pidd)
+        # Define and setup atexit closure
+        @atexit.register
+        def unlink_pid():
+            try:
+                os.unlink(pidfile)
+            except Exception:
+                pass
+
+
+
 running_flag = True     # Daemon cleanly exits when set to False
 
 def terminate_me(signum, frame):
@@ -321,11 +437,21 @@ def get_args():
         default=path.splitext(__file__)[0]+".cfg",
         dest="config",
         help="configuration file path")
-    argp.add_argument('--oneshot',
+    argp.add_argument("-o", "--oneshot",
         default=False,
         dest="oneshot",
         action="store_true",
         help="don't daemonise, run just once")
+    argp.add_argument("-d", "--daemon",
+        default=False,
+        dest="daemon",
+        action="store_true",
+        help="daemonize")
+    argp.add_argument("-p", "--pid_file",
+        default=None,
+        dest="pid_file",
+        action="store",
+        help="create PID file with this name")
     return argp.parse_args()
 
 
@@ -344,6 +470,18 @@ def get_configs():
 
 
 
+def get_uid_gid(str_id, get_nam_func):
+    if str_id:
+        try:
+            id = int(str_id)
+        except ValueError:
+            id = get_nam_func(str_id)[2]
+    else:
+        id = None
+    return id
+
+
+
 if __name__ == "__main__":
 
     args = get_args()
@@ -352,43 +490,34 @@ if __name__ == "__main__":
 
     wconfig, fconfig = get_configs()
 
-    oneshot = args.oneshot
-    safe_dir = SafeDir(fconfig.get("dir", args.func))
-
     wclient = Client(**wconfig)
 
-    if oneshot:
-        daemon = DummyContext()
-    else:
-        work_dir = fconfig.get("work_dir", ".")
-        chroot_dir = fconfig.get("chroot_dir")
-        umask = fconfig.get("umask", 0)
-        pid_file = fconfig.get("pid_file", "/var/run/warden_filer.pid")
-        uid = fconfig.get("uid")
-        gid = fconfig.get("gid")
-    
-        daemon = DaemonContext(
-            working_directory = work_dir,
-            chroot_directory = chroot_dir,
-            umask = umask,
-            pidfile = TimeoutPIDLockFile(pid_file, acquire_timeout=0),
-            uid = uid,
-            gid = gid,
-            files_preserve = get_logger_files(wclient.logger),
-            signal_map = {
-                signal.SIGTERM: terminate_me,
-                signal.SIGINT: terminate_me,
-                signal.SIGHUP: None
-            }
-        )
-        
     try:
-        with daemon:
-            wclient.logger.info("Starting %s" % args.func)
-            function(fconfig, wclient, safe_dir, oneshot)
-            wclient.logger.info("Exiting %s" % args.func)
-    except lockfile.Error as e:
-        wclient.logger.critical("Error acquiring lockfile %s (%s)"
-            % (daemon.pidfile.lock_file, type(e).__name__))
-    except Exception:
-        wclient.logger.critical("%s daemon error" % args.func, exc_info=sys.exc_info())
+        if args.daemon:
+            from pwd import getpwnam
+            from grp import getgrnam
+            uid = get_uid_gid(fconfig.get("uid"), getpwnam)
+            gid = get_uid_gid(fconfig.get("gid"), getgrnam)
+
+            daemonize(
+                work_dir = fconfig.get("work_dir", "."),
+                chroot_dir = fconfig.get("chroot_dir"),
+                umask = fconfig.get("umask"),
+                uid = uid,
+                gid = gid,
+                pidfile = args.pid_file,
+                files_preserve = get_logger_files(wclient.logger),
+                signals = {
+                    signal.SIGTERM: terminate_me,
+                    signal.SIGINT: terminate_me,
+                    signal.SIGHUP: signal.SIG_IGN,
+                    signal.SIGTTIN: signal.SIG_IGN,
+                    signal.SIGTTOU: signal.SIG_IGN})
+
+        safe_dir = SafeDir(fconfig.get("dir", args.func))
+        wclient.logger.info("Starting %s" % args.func)
+        function(fconfig, wclient, safe_dir, args.oneshot)
+        wclient.logger.info("Exiting %s" % args.func)
+
+    except Exception as e:
+        Error(message="%s daemon error" % args.func, exc=sys.exc_info()).log(wclient.logger)

warden_filer/warden_filer_logrotate.dist

+# You may want to review and/or change the logfile path, user/group in
+# 'create' and daemon to restart in 'postrotate'
+
+/var/log/warden_filer.log
+{
+	rotate 52
+	weekly
+	missingok
+	notifempty
+	compress
+	delaycompress
+	dateext
+	create 640 mentat mentat
+	postrotate
+		/etc/init.d/warden_filer_sender restart
+		/etc/init.d/warden_filer_receiver restart
+	endscript
+}

warden3/contrib/warden_filer/warden_filer_receiver → warden_filer/warden_filer_receiver

-#!/bin/sh
+#!/bin/bash
 #
 ### BEGIN INIT INFO
 # Provides:          warden_filer_receiver
@@ -18,15 +18,27 @@ SERVICE_NAME="${DAEMON_NAME}_${FUNC}"
 PID=/var/run/"$DAEMON_NAME"/"$FUNC".pid
 CONFIG=/etc/"$DAEMON_NAME".cfg
 
-test -f /etc/default/"$SERVICE_NAME" && . /etc/default/"$SERVICE_NAME"
-. /lib/lsb/init-functions
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$SERVICE_NAME" ] && . /etc/$n/"$SERVICE_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
 
 ACTION="$1"
 
 case "$ACTION" in
 	start)
+		mkdir -p "${PID%/*}"
 		log_daemon_msg "Starting $SERVICE_NAME" "$SERVICE_NAME"
-		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" "$FUNC"
+		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" --pid_file "$PID" --daemon "$FUNC"
 		log_end_msg $?
 		;;
 	stop)

warden3/contrib/warden_filer/warden_filer_sender → warden_filer/warden_filer_sender

-#!/bin/sh
+#!/bin/bash
 #
 ### BEGIN INIT INFO
 # Provides:          warden_filer_sender
@@ -18,15 +18,27 @@ SERVICE_NAME="${DAEMON_NAME}_${FUNC}"
 PID=/var/run/"$DAEMON_NAME"/"$FUNC".pid
 CONFIG=/etc/"$DAEMON_NAME".cfg
 
-test -f /etc/default/"$SERVICE_NAME" && . /etc/default/"$SERVICE_NAME"
-. /lib/lsb/init-functions
+# Try Debian & Fedora/RHEL/Suse sysconfig
+for n in default sysconfig; do
+	[ -f /etc/$n/"$SERVICE_NAME" ] && . /etc/$n/"$SERVICE_NAME"
+done
+
+# Fallback
+function log_daemon_msg () { echo -n "$@"; }
+function log_end_msg () { [ $1 -eq 0 ] && echo " OK" || echo " Failed"; }
+function status_of_proc () { [ -f "$PID" ] && ps u -p $(<"$PID") || echo "$PID not found."; }
+function start_daemon () { shift; shift; $* ; }
+function killproc () { kill $(cat $PID) ; }
+
+[ -f /lib/lsb/init-functions ] && . /lib/lsb/init-functions
 
 ACTION="$1"
 
 case "$ACTION" in
 	start)
+		mkdir -p "${PID%/*}"
 		log_daemon_msg "Starting $SERVICE_NAME" "$SERVICE_NAME"
-		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" "$FUNC"
+		start_daemon -p "$PID" "$DAEMON_PATH" -c "$CONFIG" --pid_file "$PID" --daemon "$FUNC"
 		log_end_msg $?
 		;;
 	stop)

warden_ra/LICENSE

+Copyright (c) 2011-2016 Cesnet z.s.p.o <warden-info@cesnet.cz>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\ No newline at end of file

warden_ra/README

+Warden Registration Authority for Warden 3.X
+============================================
+
+Introduction
+------------
+
+Warden RA is a certificate registration authority for Warden_ server.
+It is meant to support the client registration process and simplification
+of the credential transport.
+
+As Warden clients are authenticated by X509 certificate, the usual certificate
+generation process can be used - local key and certificate request gets
+generated, the request is submitted to registration authority, and after
+review, certificate is issued and delivered back.
+
+However in centralised Warden setup, it is useful to be able to preallow
+certificate for particular client during negotiation phase (thus removing
+another round-trip).
+
+This is done by issuing 'token' by Warden registration officer during client
+registration, which is just a oneshot password, allowing sending the request
+and getting new certificate in one step through web API.
+
+Password is meant only for completely new clients or unusual situations,
+however RA also allows prolongation - generating new certificate by using
+old certificate (which must be still valid, of course) instead of password.
+
+The application script, which can be distributed to newly registered clients,
+is also included.
+
+Dependencies
+------------
+
+ 1. Platform
+
+    Python 2.7+
+    Apache 2.2+
+    mod_wsgi 3.3+
+
+
+Registration process
+--------------------
+
+New client credentials
+``````````````````````
+
+After succesful negotiation of new Warden client parameters, the registration
+officer enables new certificate generation by issuing (on the server side):
+
+    warden_ra.py applicant --name org.example.warden.client
+
+The tool generates and sets one time password on the registration authority
+side, and this password can be sent (preferably through the secured channel)
+to the new client administrator along with other setup information.
+
+The client administrator runs the application script with application
+password:
+
+    warden_apply.sh https://warden.example.org/warden-ra/ org.example.warden.client P4SSW0RD
+
+The script creates new X509 key, CSR certificate request and makes call to
+the Warden RA web service, where it obtains the new complete certificate.
+
+
+Prolonging existing client credentials
+``````````````````````````````````````
+
+The client administrator runs the application script with his existing valid
+Warden credentials, which he needs to prolong:
+
+    warden_apply.sh https://warden.example.org/warden-ra/ org.example.warden.client cert.pem key.pem
+
+The script creates new X509 key, CSR certificate request and makes call to
+the Warden RA web service, where it obtains the new complete certificate.
+
+
+Installation
+------------
+
+This depends heavily on your distribution and Apache configuration.
+Basically you need to create and include apache.conf:
+
+  Include /opt/warden_ra/apache22.conf
+
+or paste the contents into whichever Directory, Location or VirtualHost
+you dedicate for Warden RA. Note that you have to use different host than
+the one for Warden server, as Warden RA needs different Apache options
+for SSL authentication.
+
+You may need to change paths to certificate/key/ca material, path to
+warden_server.wsgi and web path alias.
+
+Note that warden_ra itself is NOT thread safe, so included configuration
+expects Apache with mpm-prefork worker, or you will have to configure
+mod_wsgi as separate process with threads=1.
+
+Also, for warden_server.wsgi, you can use warden_server.wsgi.dist as
+a template. You will possibly need to change at least configuration
+file path.
+
+ * Now install and/or configure RA backend (see README.openssl or README.ejbca)
+
+ * Configure Warden RA (see next chapter)
+
+ * Reload Apache
+
+
+Configuration
+-------------
+
+   Configuration is JSON object in file (warden_server.cfg by default),
+however, lines starting with "#" or "//" are allowed and will be ignored as
+comments. File must contain valid JSON object, containing configuration. See
+also warden_server.cfg.dist as example.
+
+   Top level JSON object contains subsections, which configure particular
+aspect of the server. Each subsection can contain "type" key, which chooses
+particular implementation object of the aspect, for example type of logger
+(file, syslog), such as:
+
+   {
+      "Log": {
+         "type": "SysLogger",
+         ...
+      },
+      "DB": { ... }
+   }
+
+Sections and their "type" objects can be:
+
+    Log: FileLogger, SysLogger
+    Auth: OptionalAuthenticator
+    Registry: EjbcaRegistry, OpenSSLRegistry
+    Handler: CertHandler
+
+"type" keyword is not mandatory, if not specified, first implementation
+object from particular section list is used ("FileLogger" for example).
+
+Object function and configuration keys are as follows:
+
+   FileLogger: logging into file on the filesystem
+      filename: name of the file, defaults to "warden_ra.log" at
+                installation directory
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+   SysLogger: logging into unix syslog
+      socket: path to syslog socket, defaults to "/dev/log"
+      facility: syslog facility, defaults to "daemon"
+      level: least log level (CRITICAL, ERROR, WARNING, INFO, DEBUG)
+
+    OptionalAuthenticator: authenticate based on X509 certificate, or
+        signal the password auth for the registry
+
+    CertHandler: the main certificate requestor implementation
+
+    For OpenSSLRegistry or EJBCARegistry configuration please see
+    README.openssl or README.ejbca respectively.
+
+
+Command line
+------------
+
+When run from the command line, RA allows for client and request management.
+
+
+    warden_ra.py [--help] [-c CONFIG] [-v]
+                        {list,register,applicant,request,gencert} ...
+
+    Warden server certificate registry
+    
+    arguments:
+      --help                show this help message and exit
+      -c CONFIG, --config CONFIG
+                            path to configuration file
+      -v, --verbose         be more chatty
+    
+    commands:
+      {list,register,applicant,request,gencert}
+        list                list clients
+        register            register client
+        applicant           allow for certificate application
+        request             generate CSR
+        gencert             get new certificate
+
+
+    warden_ra.py list [--help] [--name NAME]
+    
+    List registered clients.
+    
+    arguments:
+      --help       show this help message and exit
+      --name NAME  client name
+
+
+    warden_ra.py register [--help] --name NAME --admins
+                                 [ADMINS [ADMINS ...]]
+    
+    Add client registration entry.
+    
+    arguments:
+      --help                show this help message and exit
+      --name NAME           client name
+      --admins [ADMINS [ADMINS ...]]
+                            administrator list
+
+
+    warden_ra.py applicant [--help] --name NAME [--password PASSWORD]
+    
+    Set client into certificate application mode and set its password
+    
+    optional arguments:
+      --help               show this help message and exit
+      --name NAME          client name
+      --password PASSWORD  password for application (will be autogenerated if not
+                           set)
+
+    
+.. _Warden: https://warden.cesnet.cz/
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o

warden_ra/README.ejbca

+EJBCA backend for Warden 3.# Registration Authority
+===================================================
+
+Introduction
+------------
+
+EJBCA_ is an open source CA management software. To use this backend
+with Warden RA, you need to have it already installed and running.
+Tested with EJBCA_ 3.9.
+
+.. _EJBCA: https://www.ejbca.org/
+
+
+Configuration
+-------------
+
+    Options for "Registry: EjbcaRegistry" section.
+
+    url: EJBCA API URL, for example "https://ejbca.example.org/ejbca/ejbcaws/ejbcaws?wsdl"
+    cert: certificate for authentication to EJBCA, defaults to "warden_ra.cert.pem"
+    key: key for authentication to EJBCA, defaults to "warden_ra.key.pem"
+    ca_name: name of the CA, dedicated for Warden, defaults to "Example CA"
+    certificate_profile_name: name of the EJBCA certificate profile, defaults to "Example"
+    end_entity_profile_name: name of the EJBCA entity profile, defaults to "Example EE"
+    subject_dn_template: template for the DN generation, defaults to "DC=cz,DC=example-ca,DC=warden,CN=%s"
+    username_suffix: suffix, which will be added to EJBCA entities, defaults to "@warden"
+
+
+------------------------------------------------------------------------------
+
+Copyright (C) 2017 Cesnet z.s.p.o