From c51564de4ca81841af3a7ef74b7fa54523a0eac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20K=C3=A1cha?= <ph@cesnet.cz> Date: Fri, 1 Jul 2022 18:05:09 +0200 Subject: [PATCH] Simplify data download and unnecessary config, add CESNET banners and legend, more interesting arcs animation --- contrib/map/README | 9 +- contrib/map/backend/warden-map.py | 82 ++++---- contrib/map/frontend/css/warden-map.css | 75 +++++++- contrib/map/frontend/js/warden-map.js | 240 ++++++++++++++++++++++-- contrib/map/frontend/warden-map.html | 9 + 5 files changed, 336 insertions(+), 79 deletions(-) diff --git a/contrib/map/README b/contrib/map/README index b69f63e..0f9147b 100644 --- a/contrib/map/README +++ b/contrib/map/README @@ -36,14 +36,11 @@ B. Configuration 4. Setup backend call (warden-map.py) in a crontab. NOTE: Please make sure you will have stored warden-map.json file - in the fronted folder. - EXAMPLE: ./warden-map.py --events 100 \ - --client cz.cesnet.warden.map \ + in the frontend folder. + EXAMPLE: ./warden-map.py --client cz.cesnet.warden.map \ --key certs/key.pem \ --cert certs/cert.pem \ - --cacert certs/cacert.pem \ - --secret SeCreT \ - --output ../fronted/ + --output ../frontend/ 5. Enjoy your map. diff --git a/contrib/map/backend/warden-map.py b/contrib/map/backend/warden-map.py index c742452..a6a2ac8 100755 --- a/contrib/map/backend/warden-map.py +++ b/contrib/map/backend/warden-map.py @@ -6,47 +6,45 @@ # Copyright (C) 2016 Cesnet z.s.p.o # Use of this source is governed by a 3-clause BSD-style license, see LICENSE file. -import GeoIP +import json import codecs import time +import argparse +import GeoIP +import requests -def getLastEvents(events, client, key, cert, cacert, secret): +def getLastEvents(client, key, cert): - ses = Session() - #req = Request('POST', 'https://warden-hub.cesnet.cz/warden3/getEvents?client='+ client + ('&secret='+ secret if secret else "")+'&count=' + events + '&id=0') - req = Request('POST', 'https://warden-hub.cesnet.cz/warden3/getEvents?client='+ client + ('&secret='+ secret if secret else "")+'&count=' + events) - #req = Request('POST', 'https://warden-hub.cesnet.cz/warden3/getEvents?nocat=Other&client='+ client + ('&secret='+ secret if secret else "")+'&count=' + events) - pre = req.prepare() - res = ses.send(pre, cert = (cert, key), verify=cacert) + res = requests.post( + 'https://warden-hub.cesnet.cz/warden3/getEvents?client=%s' % (client,), + cert=(cert, key) + ) data = res.json() i = 0 eventsList = [] for p in data['events']: event = {} - if i < events: - for key, value in { 'event': 'Category', 'time': 'DetectTime', 'origin': 'Source', 'destination': 'Target'}.iteritems(): - if value in p: - if (key == 'origin') or (key == 'destination'): - event[key] = {} - if 'IP4' in p[value][0]: - event[key]['ip'] = p[value][0]['IP4'][0] - else: - event[key] = {} - elif (key == 'event'): - event[key] = ', '.join(p[value]) + for key, value in { 'event': 'Category', 'time': 'DetectTime', 'origin': 'Source', 'destination': 'Target'}.items(): + if value in p: + if (key == 'origin') or (key == 'destination'): + event[key] = {} + if 'IP4' in p[value][0]: + event[key]['ip'] = p[value][0]['IP4'][0] else: - event[key] = p[value] - else: - if (key == 'origin') or (key == 'destination'): event[key] = {} - else: - event[key] = {} - if 'ip' in event['origin']: - eventsList.append(event) - i += 1 - else: - break + elif (key == 'event'): + event[key] = ', '.join(p[value]) + else: + event[key] = p[value] + else: + if (key == 'origin') or (key == 'destination'): + event[key] = {} + else: + event[key] = {} + if 'ip' in event['origin']: + eventsList.append(event) + i += 1 return eventsList @@ -60,18 +58,15 @@ def getGeolocation(ip, db): return { 'latitude': data['latitude'], 'longitude': data['longitude'], - 'country_name': unicode(data['country_name'], "utf-8") if data['country_name'] else None, - 'city': unicode(data['city'], "utf-8") if data['city'] else None + 'country_name': data['country_name'] if data['country_name'] else None, + 'city': data['city'] if data['city'] else None } def main(args): - events = args.events[0] client = args.client[0] key = args.key[0] cert = args.cert[0] - cacert = args.cacert[0] - secret = args.secret[0] if args.output is not None: path = args.output[0] + 'warden-map.json' @@ -81,7 +76,7 @@ def main(args): db = GeoIP.open("GeoLiteCity.dat", GeoIP.GEOIP_MEMORY_CACHE) db.set_charset(GeoIP.GEOIP_CHARSET_UTF8) - wardenEvents = getLastEvents(events, client, key, cert, cacert, secret) + wardenEvents = getLastEvents(client, key, cert) for p in wardenEvents: for target in {'origin', 'destination'}: @@ -106,18 +101,11 @@ def main(args): wardenEvents.append(int(time.time())); - with codecs.open(path, 'w', encoding="utf-8") as outfile: - json.dump(wardenEvents, outfile, ensure_ascii = False) + with open(path, 'w') as outfile: + json.dump(wardenEvents, outfile) - return 0 if __name__ == '__main__': - import sys - import json - from requests import Request, Session - import requests - import argparse - parser = argparse.ArgumentParser(description='Creates warden-map.json for warden-map.html frontend.', formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=30)) @@ -126,18 +114,12 @@ if __name__ == '__main__': requiredNamed = parser.add_argument_group('required arguments') - requiredNamed.add_argument('--events', metavar='<number>', type=str, required=True, - nargs=1, help='count of events for a map') requiredNamed.add_argument('--client', metavar='<org.ex.cl>', type=str, required=True, nargs=1, help='client name') requiredNamed.add_argument('--key', metavar='path/key.pem', type=str, required=True, nargs=1, help='SSL key for a client') requiredNamed.add_argument('--cert', metavar='path/cert.pem', type=str, required=True, nargs=1, help='SSL cert for a client') - requiredNamed.add_argument('--cacert', metavar='path/cacert.pem', type=str, required=True, - nargs=1, help='SSL cacert for a client') - requiredNamed.add_argument('--secret', metavar='<SeCreT>', type=str, required=True, - nargs=1, help='secret key for a client') args = parser.parse_args() main(args) diff --git a/contrib/map/frontend/css/warden-map.css b/contrib/map/frontend/css/warden-map.css index a1b53d9..ed1435d 100644 --- a/contrib/map/frontend/css/warden-map.css +++ b/contrib/map/frontend/css/warden-map.css @@ -11,6 +11,10 @@ body { font-family: 'Oswald', sans-serif; + background: #00253D; + border: 0px; + padding: 0px; + margin: 0px; } h2 { @@ -22,7 +26,7 @@ h2 { } #country { - color: #0062a2; + color: #0062a2; /* Cesnet blue */ font-weight: bold; } @@ -35,25 +39,29 @@ table { } table th { - color: #0062a2; + color: #0062a2; /* Cesnet blue */ padding: 0; } table td { - color: #4b4d4a; + color: #4b4d4a; /* Greenish gray */ padding: 0; } #container { overflow: hidden; - border: 2px solid #0062a2; - border-radius: 5px; - background: white; +/* border: 2px solid #0062a2; + border: 0px; + padding: 0px; + margin: 0px; + border-radius: 5px;*/ position: relative; - width: 1280px; - height: 720px; +/* width: 1280px; + height: 720px;*/ max-width: 100%; max-height: 100% + width: 100%; + height: 100vh;*/ } .zoom-button { @@ -77,3 +85,54 @@ table td { padding: 10px; color: #0062a2; } + +#warden-logo { + position: absolute; + top: 30px; + left: 30px; + background: white; + padding: 10px; + border-radius: 10px; + width: 240px; + height: 92px; + text-align: center; +} + +#cesnet-logo { + position: absolute; + top: 30px; + right: 30px; + background: white; + padding: 10px; + border-radius: 10px; + width: 240px; + height: 92px; + text-align: center; +} + +#legend-box { + position: absolute; + bottom: 30px; + left: 30px; + background-color: rgba(0,0,0,0.3); + color: white; + padding: 10px; + border-radius: 10px; + /*width: 240px; + height: 92px; + text-align: center;*/ +} + +#heading { + position: absolute; + top: 30px; + left: 50%; + width: 40em; + height: 92px; + margin-left: -20em; + font-size: xx-large; + color: white; + text-align: center; + vertical-align: middle; + line-height: 92px; +} diff --git a/contrib/map/frontend/js/warden-map.js b/contrib/map/frontend/js/warden-map.js index e4610f5..bda4790 100644 --- a/contrib/map/frontend/js/warden-map.js +++ b/contrib/map/frontend/js/warden-map.js @@ -205,20 +205,197 @@ Zoom.prototype._getNextScale = function(direction) { return scaleSet[shift]; }; + + function defaults(obj) { + Array.prototype.slice.call(arguments, 1).forEach(function(source) { + if (source) { + for (var prop in source) { + // Deep copy if property not set + if (obj[prop] == null) { + if (typeof source[prop] == 'function') { + obj[prop] = source[prop]; + } + else { + obj[prop] = JSON.parse(JSON.stringify(source[prop])); + } + } + } + } + }); + return obj; + } + +function val( datumValue, optionsValue, context ) { + if ( typeof context === 'undefined' ) { + context = optionsValue; + optionsValues = undefined; + } + var value = typeof datumValue !== 'undefined' ? datumValue : optionsValue; + + if (typeof value === 'undefined') { + return null; + } + + if ( typeof value === 'function' ) { + var fnContext = [context]; + if ( context.geography ) { + fnContext = [context.geography, context.data]; + } + return value.apply(null, fnContext); + } + else { + return value; + } + } + +var cat_color = { + "Abusive": "MediumPurple", + "Malware": "Red", + "Recon": "LightSlateGray", + "Attempt": "GhostWhite", + "Intrusion": "DarkTurquoise", + "Availability": "HotPink", + "Information": "PaleTurquoise", + "Fraud": "Yellow", + "Vulnerable": "DarkGoldenRod", + "Anomaly": "Brown", + "Other": "Green" +} + +var cat_desc = { + "Abusive": "spam", + "Malware": "virus, worm, trojan, malware", + "Recon": "scanning, sniffing", + "Attempt": "bruteforce, exploitation attempt", + "Intrusion": "botnet, successful exploit", + "Availability": "(D)DOS", + "Information": "wiretapping, spoofing, hijacking", + "Fraud": "phishing, scam", + "Vulnerable": "open for abuse", + "Anomaly": "unusual traffic", + "Other": "unknown/unidentified" +} + + function handleArcs (layer, data, options) { + var self = this, + svg = this.svg; + + if ( !data || (data && !data.slice) ) { + throw "Datamaps Error - arcs must be an array"; + } + + // For some reason arc options were put in an `options` object instead of the parent arc + // I don't like this, so to match bubbles and other plugins I'm moving it + // This is to keep backwards compatability + for ( var i = 0; i < data.length; i++ ) { + data[i] = defaults(data[i], data[i].options); + delete data[i].options; + } + + if ( typeof options === "undefined" ) { + options = defaultOptions.arcConfig; + } + + var arcs = layer.selectAll('path.datamaps-arc').data( data, JSON.stringify ); + + var path = d3.geo.path() + .projection(self.projection); + + arcs + .enter() + .append('svg:path') + .attr('class', 'datamaps-arc') + .style('stroke-linecap', 'round') + .style('stroke', function(datum) { +/* return val(datum.strokeColor, options.strokeColor, datum);*/ + for (cat in cat_color) { + if (datum.event.startsWith(cat)) { + return cat_color[cat]; + } + } + return "Green"; + }) + .style('fill', 'none') + .style('stroke-width', function(datum) { + return val(datum.strokeWidth, options.strokeWidth, datum); + }) + .attr('d', function(datum) { + + var originXY, destXY; + + originXY = self.latLngToXY(val(datum.origin.latitude, datum), val(datum.origin.longitude, datum)) + + destXY = self.latLngToXY(val(datum.destination.latitude, datum), val(datum.destination.longitude, datum)); + + var midXY = [ (originXY[0] + destXY[0]) / 2, (originXY[1] + destXY[1]) / 2]; + if (options.greatArc) { + // TODO: Move this to inside `if` clause when setting attr `d` + var greatArc = d3.geo.greatArc() + .source(function(d) { return [val(d.origin.longitude, d), val(d.origin.latitude, d)]; }) + .target(function(d) { return [val(d.destination.longitude, d), val(d.destination.latitude, d)]; }); + + return path(greatArc(datum)) + } + var sharpness = val(datum.arcSharpness, options.arcSharpness, datum); + return "M" + originXY[0] + ',' + originXY[1] + "S" + (midXY[0] + (50 * sharpness)) + "," + (midXY[1] - (75 * sharpness)) + "," + destXY[0] + "," + destXY[1]; + }) + .attr('data-info', function(datum) { + return JSON.stringify(datum); + }) + .on('mouseover', function ( datum ) { + var $this = d3.select(this); + + if (options.popupOnHover) { + self.updatePopup($this, datum, options, svg); + } + }) + .on('mouseout', function ( datum ) { + var $this = d3.select(this); + + d3.selectAll('.datamaps-hoverover').style('display', 'none'); + }) + .transition() + .style('fill', function(datum, i) { + /* + Thank you Jake Archibald, this is awesome. + Source: http://jakearchibald.com/2013/animated-line-drawing-svg/ + */ + var length = this.getTotalLength(); + this.style.transition = this.style.WebkitTransition = 'none'; + this.style.strokeDasharray = length + ' ' + length; + this.style.strokeDashoffset = length; + this.getBoundingClientRect(); + this.style.transition = this.style.WebkitTransition = 'stroke-dashoffset ' + val(datum.animationSpeed, options.animationSpeed, datum) + 'ms ' + datum.delay*1000 + 'ms ease-out'; + this.style.strokeDashoffset = '0'; + return 'none'; + }); + + arcs.exit() + .transition() + .duration(1000) + .style('opacity', 0) + .remove(); + } + +var main_data = []; +var prev_data = 0; + // Configuration of datamap canvas // Futher reading can be found at https://datamaps.github.io/ function Datamap() { this.$container = $("#container"); - this.instance = new Datamaps({ + instance = this.instance = new Datamaps({ scope: 'world', element: this.$container.get(0), done: this._handleMapReady.bind(this), projection: 'mercator', fills: { - defaultFill: '#dcdcda' + /*defaultFill: '#454545'*/ + defaultFill: 'black' }, geographyConfig: { - borderColor: '#fdfdfd', + hideAntarctica: true, + borderColor: '#0062a2', highlightFillColor: '#4b4d4a', highlightBorderColor: '#fdfdfd', popupOnHover: true, @@ -226,18 +403,18 @@ function Datamap() { return '<div class="hoverinfo" id="country">' + geography.properties.name + '</div>'; }, }, - arcConfig: { + ph_arcConfig: { strokeColor: '#0062a2', - strokeWidth: 1, - arcSharpness: 5, - animationSpeed: 4000, // Milliseconds + strokeWidth: 2, + arcSharpness: 2, /* 5 */ + animationSpeed: 3000, // Milliseconds popupOnHover: true, // Case with latitude and longitude popupTemplate: function(geography, data) { if ( ( data.origin && data.destination ) && data.origin.latitude && data.origin.longitude && data.destination.latitude && data.destination.longitude ) { // Content of info table str = '<div class="hoverinfo"><table id="event"><tr><th>Warden Event</th></tr><tr><td>Type</td><td>'+ JSON.stringify(data.event) +'</td></tr><tr><td>Detect Time</td><td>'+ JSON.stringify(data.time) +'</td></tr><tr><th>Event origin</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.origin.ip) + '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.origin.city) + ', ' + JSON.stringify(data.origin.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.origin.latitude) + ', ' + JSON.stringify(data.origin.longitude) + '</td></tr><tr><th>Event Destination</th></tr><tr><td>IP</td><td>' + JSON.stringify(data.destination.ip) + '</td></tr><tr><td>City & Country</td><td>' + JSON.stringify(data.destination.city) + ', ' + JSON.stringify(data.destination.country_name) + '</td></tr><tr><td>GPS</td><td>' + JSON.stringify(data.destination.latitude) + ', ' + JSON.stringify(data.destination.longitude) + '</td></tr></table></div>'; - return str.replace(/"/g,""); + return str.replace(/"/g,""); } // Missing information else { @@ -246,15 +423,48 @@ function Datamap() { } } }); - var instance = this.instance; - // Link to a json file with information for drawing. - // NOTE: Change path if you separate backend and fronend! - d3.json("./warden-map.json", function(error, data) { - instance.arc(data); - }); -}; + legend_data = d3.select("#legend") + .selectAll("li") + .data(Object.keys(cat_color).sort()) + .enter() + .append("li") + .append("span") + .style("color", function(datum) { return cat_color[datum]}) + .text(function(datum) { return datum; }) + .append("span") + .text(function(datum) { return " – " + cat_desc[datum]}) + .style("color", "white"); + + instance.addPlugin('ph_arc', handleArcs); + + setInterval(function(){ + d3.json("./warden-map.json", function(error, data) { + if (data) { + var cur_data = data.pop() + var cur_time = new Date().getTime(); + if (cur_data != prev_data) { + prev_data = cur_data; + for (var i=0; i<data.length; i++) { + data[i].arrivalTime = cur_time; + data[i].delay = i/data.length; + } + main_data = main_data.concat(data); + } + } + var trimmed_data = []; + for (var i=0; i<main_data.length; i++) { + if (main_data[i].arrivalTime + 3500 > cur_time) { + trimmed_data.push(main_data[i]); + } + } + main_data = trimmed_data; + trimmed_data = cur_time = cur_data = error = data = null; + instance.ph_arc(main_data); + }); + }, 1000); +}; Datamap.prototype._handleMapReady = function(datamap) { this.zoom = new Zoom({ diff --git a/contrib/map/frontend/warden-map.html b/contrib/map/frontend/warden-map.html index 09b9593..82eded3 100644 --- a/contrib/map/frontend/warden-map.html +++ b/contrib/map/frontend/warden-map.html @@ -23,6 +23,7 @@ <script src="./js/datamaps.world.hires.min.js"></script> <script src="./js/warden-map.js"></script> +<!-- <h2>Warden Map</h2> <div id="tools"> <button class="zoom-button" data-zoom="reset">⌂</button> @@ -30,7 +31,15 @@ <button class="zoom-button" data-zoom="in">+</button> <div id="zoom-info"></div> </div> +--> <div id="container"></div> +<div id="heading">Attacks, detected in CESNET network<br/> +SABU - Sharing and Analysis of Security Events +</div> +<div id="legend-box"> + <p><b>Reported to Warden right <i>now</i>.</b></p> + <ul id="legend"></ul> +</div> <!-- Draw datamap into id="container" --> <script>new Datamap();</script> -- GitLab