Simplify data download and unnecessary config, add CESNET banners and legend,...

Simplify data download and unnecessary config, add CESNET banners and legend, more interesting arcs animation

contrib/map/README

@@ -36,14 +36,11 @@ B. Configuration
 
   4. Setup backend call (warden-map.py) in a crontab.
      NOTE: Please make sure you will have stored warden-map.json file
-           in the fronted folder.
-     EXAMPLE: ./warden-map.py --events 100 \
-                              --client cz.cesnet.warden.map \
+           in the frontend folder.
+     EXAMPLE: ./warden-map.py --client cz.cesnet.warden.map \
                               --key certs/key.pem \
                               --cert certs/cert.pem \
-                              --cacert certs/cacert.pem \
-                              --secret SeCreT \
-                              --output ../fronted/
+                              --output ../frontend/
 
   5. Enjoy your map.
 

contrib/map/backend/warden-map.py

@@ -6,47 +6,45 @@
 # Copyright (C) 2016 Cesnet z.s.p.o
 # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file.
 
-import GeoIP
+import json
 import codecs
 import time
+import argparse
+import GeoIP
+import requests
 
-def getLastEvents(events, client, key, cert, cacert, secret):
+def getLastEvents(client, key, cert):
 
-  ses = Session()
-  #req = Request('POST', 'https://warden-hub.cesnet.cz/warden3/getEvents?client='+ client + ('&secret='+ secret if secret else "")+'&count=' + events + '&id=0')
-  req = Request('POST', 'https://warden-hub.cesnet.cz/warden3/getEvents?client='+ client + ('&secret='+ secret if secret else "")+'&count=' + events)
-  #req = Request('POST', 'https://warden-hub.cesnet.cz/warden3/getEvents?nocat=Other&client='+ client + ('&secret='+ secret if secret else "")+'&count=' + events)
-  pre = req.prepare()
-  res = ses.send(pre, cert = (cert, key), verify=cacert)
+  res = requests.post(
+      'https://warden-hub.cesnet.cz/warden3/getEvents?client=%s' % (client,),
+      cert=(cert, key)
+  )
 
   data = res.json()
   i = 0
   eventsList = []
   for p in data['events']:
     event = {}
-    if i < events:
-      for key, value in { 'event': 'Category', 'time': 'DetectTime', 'origin': 'Source', 'destination': 'Target'}.iteritems():
-        if value in p:
-          if (key == 'origin') or (key == 'destination'):
-            event[key] = {}
-            if 'IP4' in p[value][0]:
-              event[key]['ip'] = p[value][0]['IP4'][0]
-            else:
-              event[key] = {}
-          elif (key == 'event'):
-            event[key] = ', '.join(p[value])
+    for key, value in { 'event': 'Category', 'time': 'DetectTime', 'origin': 'Source', 'destination': 'Target'}.items():
+      if value in p:
+        if (key == 'origin') or (key == 'destination'):
+          event[key] = {}
+          if 'IP4' in p[value][0]:
+            event[key]['ip'] = p[value][0]['IP4'][0]
           else:
-            event[key] = p[value]
-        else:
-          if (key == 'origin') or (key == 'destination'):
             event[key] = {}
-          else:
-            event[key] = {}
-      if 'ip' in event['origin']:
-        eventsList.append(event)
-        i += 1
-    else:
-      break
+        elif (key == 'event'):
+          event[key] = ', '.join(p[value])
+        else:
+          event[key] = p[value]
+      else:
+        if (key == 'origin') or (key == 'destination'):
+          event[key] = {}
+        else:
+          event[key] = {}
+    if 'ip' in event['origin']:
+      eventsList.append(event)
+      i += 1
 
   return eventsList
 
@@ -60,18 +58,15 @@ def getGeolocation(ip, db):
     return {
       'latitude': data['latitude'], 
       'longitude': data['longitude'], 
-      'country_name': unicode(data['country_name'], "utf-8") if data['country_name'] else None, 
-      'city': unicode(data['city'], "utf-8") if data['city'] else None
+      'country_name': data['country_name'] if data['country_name'] else None, 
+      'city': data['city'] if data['city'] else None
     }
 
 def main(args):
 
-  events = args.events[0]
   client = args.client[0]
   key    = args.key[0]
   cert   = args.cert[0]
-  cacert = args.cacert[0]
-  secret = args.secret[0]
 
   if args.output is not None:
     path = args.output[0] + 'warden-map.json'
@@ -81,7 +76,7 @@ def main(args):
   db = GeoIP.open("GeoLiteCity.dat", GeoIP.GEOIP_MEMORY_CACHE)
   db.set_charset(GeoIP.GEOIP_CHARSET_UTF8)
 
-  wardenEvents = getLastEvents(events, client, key, cert, cacert, secret)
+  wardenEvents = getLastEvents(client, key, cert)
 
   for p in wardenEvents:
     for target in {'origin', 'destination'}:
@@ -106,18 +101,11 @@ def main(args):
 
   wardenEvents.append(int(time.time()));
 
-  with codecs.open(path, 'w', encoding="utf-8") as outfile:
-    json.dump(wardenEvents, outfile, ensure_ascii = False)
+  with open(path, 'w') as outfile:
+    json.dump(wardenEvents, outfile)
 
-  return 0
 
 if __name__ == '__main__':
-  import sys
-  import json
-  from requests import Request, Session
-  import requests
-  import argparse
-
   parser = argparse.ArgumentParser(description='Creates warden-map.json for warden-map.html frontend.',
                                   formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=30))
 
@@ -126,18 +114,12 @@ if __name__ == '__main__':
 
   requiredNamed = parser.add_argument_group('required arguments')
 
-  requiredNamed.add_argument('--events', metavar='<number>', type=str, required=True,
-                    nargs=1, help='count of events for a map')
   requiredNamed.add_argument('--client', metavar='<org.ex.cl>', type=str, required=True,
                     nargs=1, help='client name')
   requiredNamed.add_argument('--key', metavar='path/key.pem', type=str, required=True,
                     nargs=1, help='SSL key for a client')
   requiredNamed.add_argument('--cert', metavar='path/cert.pem', type=str, required=True,
                     nargs=1, help='SSL cert for a client')
-  requiredNamed.add_argument('--cacert', metavar='path/cacert.pem', type=str, required=True,
-                    nargs=1, help='SSL cacert for a client')
-  requiredNamed.add_argument('--secret', metavar='<SeCreT>', type=str, required=True,
-                    nargs=1, help='secret key for a client')
 
   args = parser.parse_args()
   main(args)

contrib/map/frontend/css/warden-map.css

@@ -11,6 +11,10 @@
 
 body {
   font-family: 'Oswald', sans-serif;
+  background: #00253D;
+  border: 0px;
+  padding: 0px;
+  margin: 0px;
 }
 
 h2 {
@@ -22,7 +26,7 @@ h2 {
 }
 
 #country {
-  color: #0062a2;
+  color: #0062a2;	/* Cesnet blue */
   font-weight: bold;
 }
 
@@ -35,25 +39,29 @@ table {
 }
 
 table th {
-  color: #0062a2;
+  color: #0062a2;	/* Cesnet blue */
   padding: 0;
 }
 
 table td {
-  color: #4b4d4a;
+  color: #4b4d4a;	/* Greenish gray */
   padding: 0;
 }
 
 #container {
   overflow: hidden;
-  border: 2px solid #0062a2;
-  border-radius: 5px;
-  background: white;
+/*  border: 2px solid #0062a2;
+  border: 0px;
+  padding: 0px;
+  margin: 0px;
+  border-radius: 5px;*/
   position: relative;
-  width: 1280px;
-  height: 720px;
+/*  width: 1280px;
+  height: 720px;*/
   max-width: 100%;
   max-height: 100%
+  width: 100%;
+  height: 100vh;*/
 }
 
 .zoom-button {
@@ -77,3 +85,54 @@ table td {
   padding: 10px;
   color: #0062a2;
 }
+
+#warden-logo {
+  position: absolute;
+  top: 30px;
+  left: 30px;
+  background: white;
+  padding: 10px;
+  border-radius: 10px;
+  width: 240px;
+  height: 92px;
+  text-align: center;
+}
+
+#cesnet-logo {
+  position: absolute;
+  top: 30px;
+  right: 30px;
+  background: white;
+  padding: 10px;
+  border-radius: 10px;
+  width: 240px;
+  height: 92px;
+  text-align: center;
+}
+
+#legend-box {
+  position: absolute;
+  bottom: 30px;
+  left: 30px;
+  background-color: rgba(0,0,0,0.3);
+  color: white;
+  padding: 10px;
+  border-radius: 10px;
+  /*width: 240px;
+  height: 92px;
+  text-align: center;*/
+}
+
+#heading {
+  position: absolute;
+  top: 30px;
+  left: 50%;
+  width: 40em;
+  height: 92px;
+  margin-left: -20em;
+  font-size: xx-large;
+  color: white;
+  text-align: center;
+  vertical-align: middle;
+  line-height: 92px;
+}

contrib/map/frontend/js/warden-map.js

@@ -205,20 +205,197 @@ Zoom.prototype._getNextScale = function(direction) {
   return scaleSet[shift];
 };
 
+
+ function defaults(obj) {
+    Array.prototype.slice.call(arguments, 1).forEach(function(source) {
+      if (source) {
+        for (var prop in source) {
+          // Deep copy if property not set
+          if (obj[prop] == null) {
+            if (typeof source[prop] == 'function') {
+              obj[prop] = source[prop];
+            }
+            else {
+              obj[prop] = JSON.parse(JSON.stringify(source[prop]));
+            }
+          }
+        }
+      }
+    });
+    return obj;
+  }
+
+function val( datumValue, optionsValue, context ) {
+    if ( typeof context === 'undefined' ) {
+      context = optionsValue;
+      optionsValues = undefined;
+    }
+    var value = typeof datumValue !== 'undefined' ? datumValue : optionsValue;
+
+    if (typeof value === 'undefined') {
+      return  null;
+    }
+
+    if ( typeof value === 'function' ) {
+      var fnContext = [context];
+      if ( context.geography ) {
+        fnContext = [context.geography, context.data];
+      }
+      return value.apply(null, fnContext);
+    }
+    else {
+      return value;
+    }
+  }
+  
+var cat_color = {
+  "Abusive": "MediumPurple",
+  "Malware": "Red",
+  "Recon": "LightSlateGray",
+  "Attempt": "GhostWhite",
+  "Intrusion": "DarkTurquoise",
+  "Availability": "HotPink",
+  "Information": "PaleTurquoise",
+  "Fraud": "Yellow",
+  "Vulnerable": "DarkGoldenRod",
+  "Anomaly": "Brown",
+  "Other": "Green"
+}
+
+var cat_desc = {
+  "Abusive": "spam",
+  "Malware": "virus, worm, trojan, malware",
+  "Recon": "scanning, sniffing",
+  "Attempt": "bruteforce, exploitation attempt",
+  "Intrusion": "botnet, successful exploit",
+  "Availability": "(D)DOS",
+  "Information": "wiretapping, spoofing, hijacking",
+  "Fraud": "phishing, scam",
+  "Vulnerable": "open for abuse",
+  "Anomaly": "unusual traffic",
+  "Other": "unknown/unidentified"
+}
+
+ function handleArcs (layer, data, options) {
+    var self = this,
+        svg = this.svg;
+
+    if ( !data || (data && !data.slice) ) {
+      throw "Datamaps Error - arcs must be an array";
+    }
+
+    // For some reason arc options were put in an `options` object instead of the parent arc
+    // I don't like this, so to match bubbles and other plugins I'm moving it
+    // This is to keep backwards compatability
+    for ( var i = 0; i < data.length; i++ ) {
+      data[i] = defaults(data[i], data[i].options);
+      delete data[i].options;
+    }
+
+    if ( typeof options === "undefined" ) {
+      options = defaultOptions.arcConfig;
+    }
+
+    var arcs = layer.selectAll('path.datamaps-arc').data( data, JSON.stringify );
+
+    var path = d3.geo.path()
+        .projection(self.projection);
+
+    arcs
+      .enter()
+        .append('svg:path')
+        .attr('class', 'datamaps-arc')
+        .style('stroke-linecap', 'round')
+        .style('stroke', function(datum) {
+/*          return val(datum.strokeColor, options.strokeColor, datum);*/
+          for (cat in cat_color) {
+            if (datum.event.startsWith(cat)) {
+              return cat_color[cat];
+            }
+          }
+          return "Green";
+        })
+        .style('fill', 'none')
+        .style('stroke-width', function(datum) {
+            return val(datum.strokeWidth, options.strokeWidth, datum);
+        })
+        .attr('d', function(datum) {
+
+            var originXY, destXY;
+
+            originXY = self.latLngToXY(val(datum.origin.latitude, datum), val(datum.origin.longitude, datum))
+
+            destXY = self.latLngToXY(val(datum.destination.latitude, datum), val(datum.destination.longitude, datum));
+
+            var midXY = [ (originXY[0] + destXY[0]) / 2, (originXY[1] + destXY[1]) / 2];
+            if (options.greatArc) {
+                  // TODO: Move this to inside `if` clause when setting attr `d`
+              var greatArc = d3.geo.greatArc()
+                  .source(function(d) { return [val(d.origin.longitude, d), val(d.origin.latitude, d)]; })
+                  .target(function(d) { return [val(d.destination.longitude, d), val(d.destination.latitude, d)]; });
+
+              return path(greatArc(datum))
+            }
+            var sharpness = val(datum.arcSharpness, options.arcSharpness, datum);
+            return "M" + originXY[0] + ',' + originXY[1] + "S" + (midXY[0] + (50 * sharpness)) + "," + (midXY[1] - (75 * sharpness)) + "," + destXY[0] + "," + destXY[1];
+        })
+        .attr('data-info', function(datum) {
+          return JSON.stringify(datum);
+        })
+        .on('mouseover', function ( datum ) {
+          var $this = d3.select(this);
+
+          if (options.popupOnHover) {
+            self.updatePopup($this, datum, options, svg);
+          }
+        })
+        .on('mouseout', function ( datum ) {
+          var $this = d3.select(this);
+
+          d3.selectAll('.datamaps-hoverover').style('display', 'none');
+        })
+        .transition()
+          .style('fill', function(datum, i) {
+            /*
+              Thank you Jake Archibald, this is awesome.
+              Source: http://jakearchibald.com/2013/animated-line-drawing-svg/
+            */
+            var length = this.getTotalLength();
+            this.style.transition = this.style.WebkitTransition = 'none';
+            this.style.strokeDasharray = length + ' ' + length;
+            this.style.strokeDashoffset = length;
+            this.getBoundingClientRect();
+            this.style.transition = this.style.WebkitTransition = 'stroke-dashoffset ' + val(datum.animationSpeed, options.animationSpeed, datum) + 'ms ' + datum.delay*1000 + 'ms ease-out';
+            this.style.strokeDashoffset = '0';
+            return 'none';
+          });
+
+    arcs.exit()
+      .transition()
+      .duration(1000)
+      .style('opacity', 0)
+      .remove();
+  }
+
+var main_data = [];
+var prev_data = 0;
+
 // Configuration of datamap canvas
 // Futher reading can be found at https://datamaps.github.io/
 function Datamap() {
   this.$container = $("#container");
-  this.instance = new Datamaps({
+  instance = this.instance = new Datamaps({
     scope: 'world',
     element: this.$container.get(0),
     done: this._handleMapReady.bind(this),
     projection: 'mercator',
         fills: {
-          defaultFill: '#dcdcda'
+          /*defaultFill: '#454545'*/
+          defaultFill: 'black'
         },
         geographyConfig: {
-          borderColor: '#fdfdfd',
+          hideAntarctica: true,
+          borderColor: '#0062a2',
           highlightFillColor: '#4b4d4a',
           highlightBorderColor: '#fdfdfd',
           popupOnHover: true,
@@ -226,18 +403,18 @@ function Datamap() {
           return '<div class="hoverinfo" id="country">' + geography.properties.name + '</div>';
         },
         },
-        arcConfig: {
+        ph_arcConfig: {
           strokeColor: '#0062a2',
-          strokeWidth: 1,
-          arcSharpness: 5,
-          animationSpeed: 4000, // Milliseconds
+          strokeWidth: 2,
+          arcSharpness: 2, /* 5 */
+          animationSpeed: 3000, // Milliseconds
           popupOnHover: true,
           // Case with latitude and longitude
           popupTemplate: function(geography, data) {
             if ( ( data.origin && data.destination ) && data.origin.latitude && data.origin.longitude && data.destination.latitude && data.destination.longitude ) {
               // Content of info table
               str = '<div class="hoverinfo"><table id="event"><tr><th>Warden Event</th></tr><tr><td>Type</td><td>'+ JSON.stringify(data.event) +'</td></tr><tr><td>Detect Time</td><td>'+ JSON.stringify(data.time) +'</td></tr><tr><th>Event origin</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.origin.ip) +  '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.origin.city) + ',&nbsp;' + JSON.stringify(data.origin.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.origin.latitude) + ',&nbsp;' + JSON.stringify(data.origin.longitude) + '</td></tr><tr><th>Event Destination</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.destination.ip) + '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.destination.city) + ',&nbsp;' + JSON.stringify(data.destination.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.destination.latitude) + ',&nbsp;' + JSON.stringify(data.destination.longitude) + '</td></tr></table></div>';
-              return str.replace(/"/g,"");
+              return str.replace(/&quot;/g,"");
             }
             // Missing information
             else {
@@ -246,15 +423,48 @@ function Datamap() {
           }
         }
       });
-  var instance = this.instance;
-  // Link to a json file with information for drawing.
-  // NOTE: Change path if you separate backend and fronend!
-  d3.json("./warden-map.json", function(error, data) {
-      instance.arc(data);
-  });
-};
 
+  legend_data = d3.select("#legend")
+    .selectAll("li")
+    .data(Object.keys(cat_color).sort())
+    .enter()
+      .append("li")
+        .append("span")
+          .style("color", function(datum) { return cat_color[datum]})
+          .text(function(datum) { return datum; })
+          .append("span")
+          .text(function(datum) { return " – " + cat_desc[datum]})
+          .style("color", "white");
+  
+    instance.addPlugin('ph_arc', handleArcs);
+
+  setInterval(function(){
+      d3.json("./warden-map.json", function(error, data) {
+          if (data) {
+              var cur_data = data.pop()
+              var cur_time = new Date().getTime();
+              if (cur_data != prev_data) {
+                  prev_data = cur_data;
+                  for (var i=0; i<data.length; i++) {
+                      data[i].arrivalTime = cur_time;
+                      data[i].delay = i/data.length;
+                  }
+                  main_data = main_data.concat(data);
+              }
+          }
+          var trimmed_data = [];
+          for (var i=0; i<main_data.length; i++) {
+              if (main_data[i].arrivalTime + 3500 > cur_time) {
+                  trimmed_data.push(main_data[i]);
+              }
+          }
+          main_data = trimmed_data;
+          trimmed_data = cur_time = cur_data = error = data = null;
+          instance.ph_arc(main_data);
+      });
+  }, 1000);
 
+};
 
 Datamap.prototype._handleMapReady = function(datamap) {
   this.zoom = new Zoom({

contrib/map/frontend/warden-map.html

@@ -23,6 +23,7 @@
 <script src="./js/datamaps.world.hires.min.js"></script>
 <script src="./js/warden-map.js"></script>
 
+<!--
 <h2>Warden Map</h2>
 <div id="tools">
   <button class="zoom-button" data-zoom="reset">&#x2302</button>
@@ -30,7 +31,15 @@
   <button class="zoom-button" data-zoom="in">+</button>
   <div id="zoom-info"></div>
 </div>
+-->
 <div id="container"></div>
+<div id="heading">Attacks, detected in CESNET network<br/>
+SABU - Sharing and Analysis of Security Events
+</div>
+<div id="legend-box">
+	<p><b>Reported to Warden right <i>now</i>.</b></p>
+	<ul id="legend"></ul>
+</div>
 
 <!-- Draw datamap into id="container" -->
 <script>new Datamap();</script>