diff --git a/Gruntfile.js b/Gruntfile.js
index 2d5f36f076d2cf9d29d6310250626b73304e724d..b19343abd76f1b7cc054cff8eaf7794a6be3c5f1 100644
--- a/Gruntfile.js
+++ b/Gruntfile.js
@@ -263,6 +263,29 @@ module.exports = function(grunt) {
src: './*',
dest: '<%= project_paths.web_static_dir %>vendor/bootstrap/js/'
},
+ // ----- D3
+ {
+ expand: true,
+ flatten: true,
+ cwd: 'node_modules/d3/',
+ src: './d3*.js',
+ dest: '<%= project_paths.web_static_dir %>vendor/d3/js/'
+ },
+ // ----- NVD3
+ {
+ expand: true,
+ flatten: true,
+ cwd: 'node_modules/nvd3/build/',
+ src: './*.js*',
+ dest: '<%= project_paths.web_static_dir %>vendor/nvd3/js/'
+ },
+ {
+ expand: true,
+ flatten: true,
+ cwd: 'node_modules/nvd3/build/',
+ src: './*.css*',
+ dest: '<%= project_paths.web_static_dir %>vendor/nvd3/css/'
+ },
// ----- moment
{
expand: true,
diff --git a/conf/core/statistics.json.conf b/conf/core/statistics.json.conf
new file mode 100644
index 0000000000000000000000000000000000000000..d9fe0201f27f3dcc4976061653e00bbaa7c1d172
--- /dev/null
+++ b/conf/core/statistics.json.conf
@@ -0,0 +1,9 @@
+{
+ #
+ # Definitions of core configurations for event statistics.
+ #
+ "__core__statistics": {
+ "rrds_dir": "/var/mentat/rrds",
+ "reports_dir": "/var/mentat/reports/statistician"
+ }
+}
diff --git a/lib/hawat/app.py b/lib/hawat/app.py
index 2a949ffe447222f989cbeac39446ac8c32a84a2b..48c5c1a8e06803c91ac067d2a2a52cf6752d8ec9 100644
--- a/lib/hawat/app.py
+++ b/lib/hawat/app.py
@@ -385,7 +385,8 @@ def _setup_app_babel(app):
return dict(
babel_format_datetime = flask_babel.format_datetime,
- babel_format_timedelta = flask_babel.format_timedelta
+ babel_format_timedelta = flask_babel.format_timedelta,
+ babel_format_decimal = flask_babel.format_decimal
)
return app
diff --git a/lib/hawat/base.py b/lib/hawat/base.py
index 0605adc0b03179c88e9e91a3b6c31af236dbf5ac..345f07a71ca86837b0fccad1151eed1a96d197b0 100644
--- a/lib/hawat/base.py
+++ b/lib/hawat/base.py
@@ -205,14 +205,18 @@ class HawatBaseView(flask.views.View):
class HawatFileView(HawatBaseView):
"""
- Base class for simple views. These are the most, well, simple views, that are
- rendering single template file. In most use cases, it should be enough to just
- enhance the default implementation :py:func:`hawat.base.HawatBaseView.get_template_context`
- to inject additional variables into the template.
+ Base class for indirrect file access views. These views can be used to access
+ and serve files from arbitrary filesystem directories (that are accessible to
+ application process). This can be very usefull for serving files like charts,
+ that are periodically generated into configurable and changeable location.
"""
@staticmethod
def get_directory_path():
+ """
+ *Hook method*. Must return full path to the directory, that will be used
+ as a base path for serving files.
+ """
raise NotImplementedError()
def dispatch_request(self, filename):
@@ -281,6 +285,47 @@ class HawatDbmodelView(HawatBaseView):
return
+class HawatSearchView(HawatDbmodelView):
+ """
+ Base class for search views.
+ """
+ @staticmethod
+ def get_search_form(args):
+ """
+ *Hook method*. Must return instance of :py:mod:`flask_wtf.FlaskForm`
+ appropriate for given search type.
+ """
+ raise NotImplementedError()
+
+ def search(self, query, model, args, context):
+ """
+ *Hook method*.
+ """
+ raise NotImplementedError()
+
+ def dispatch_request(self):
+ """
+ Mandatory interface required by the :py:func:`flask.views.View.dispatch_request`.
+ Will be called by the *Flask* framework to service the request.
+ """
+ context = self.get_template_context()
+
+ form = self.get_search_form(flask.request.args)
+
+ if hawat.const.HAWAT_FORM_ACTION_SUBMIT in flask.request.args:
+ if form.validate():
+ items = self.search(self.dbquery, self.dbmodel, flask.request.args, context)
+ context.update(
+ items = items,
+ items_count = len(items),
+ args = flask.request.args
+ )
+ self.do_before_render(items, context)
+
+ context.update(search_form = form)
+ return flask.render_template(self.get_view_template(), **context)
+
+
class HawatItemListView(HawatDbmodelView):
"""
Base class for item *list* views. These views provide quick and simple access
diff --git a/lib/hawat/blueprints/dashboards/__init__.py b/lib/hawat/blueprints/dashboards/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0761e018c8814859f6c73ae9a1124c14509f6e2
--- /dev/null
+++ b/lib/hawat/blueprints/dashboards/__init__.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+Hawat pluggable module: *dashboards*
+
+Description
+^^^^^^^^^^^
+
+This pluggable module provides access to event statistic dashboards.
+
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import collections
+
+import flask_login
+from flask_babel import lazy_gettext
+
+import mentat.stats.idea
+import hawat.base
+
+from mentat.datatype.sqldb import EventStatisticsModel
+from hawat.blueprints.dashboards.forms import OverallDashboardForm
+
+
+class OverallView(hawat.base.HawatSearchView):
+
+ decorators = [flask_login.login_required]
+
+ @staticmethod
+ def get_view_name():
+ return 'overall'
+
+ @staticmethod
+ def get_view_title():
+ return lazy_gettext('Overall event dashboards')
+
+ @staticmethod
+ def get_view_template():
+ return 'dashboards/overall.html'
+
+ @staticmethod
+ def get_menu_icon():
+ return 'dashboards'
+
+ @staticmethod
+ def get_menu_title():
+ return lazy_gettext('Overall')
+
+ @property
+ def dbmodel(self):
+ """
+ *Hook property*. Implementation of :py:func:`hawat.base.HawatDbmodelView.dbmodel` interface.
+ """
+ return EventStatisticsModel
+
+ @staticmethod
+ def get_search_form(args):
+ return OverallDashboardForm(args, meta = {'csrf': False})
+
+ def search(self, query, model, args, context):
+ if 'dt_from' in args and args['dt_from']:
+ query = query.filter(model.dt_from >= args['dt_from'])
+ if 'dt_to' in args and args['dt_to']:
+ query = query.filter(model.dt_to <= args['dt_to'])
+ return query.order_by(model.interval).all()
+
+ def do_before_render(self, items, context):
+ """
+ *Hook method*. Will be called before rendering the template.
+ """
+ context.update(
+ statistics = mentat.stats.idea.truncate_evaluations(
+ mentat.stats.idea.aggregate_stat_groups(items)
+ )
+ )
+
+
+#-------------------------------------------------------------------------------
+
+
+class DashboardsBlueprint(hawat.base.HawatBlueprint):
+
+ @staticmethod
+ def get_module_title():
+ return lazy_gettext('Event statistic dashboards')
+
+ def register_app(self, app):
+ app.menu_main.add_entry('dashboards.overall', content = self.hawat_view_list['overall'], position = 10)
+
+
+#-------------------------------------------------------------------------------
+
+
+def get_blueprint():
+ """
+ Mandatory interface and factory function. This function must return a valid
+ instance of :py:class:`hawat.base.HawatBlueprint` or :py:class:`flask.Blueprint`.
+ """
+
+ bp = DashboardsBlueprint(
+ 'dashboards',
+ __name__,
+ template_folder = 'templates',
+ url_prefix = '/dashboards')
+
+ bp.register_view_class(OverallView, '/overall')
+
+ return bp
diff --git a/lib/hawat/blueprints/dashboards/forms.py b/lib/hawat/blueprints/dashboards/forms.py
new file mode 100644
index 0000000000000000000000000000000000000000..2db6ca0011e329a23126a1b237b3605af257a317
--- /dev/null
+++ b/lib/hawat/blueprints/dashboards/forms.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This module contains custom internal geoip search form for Hawat.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import time
+import datetime
+import ipranges
+import wtforms
+
+import flask_wtf
+from flask_babel import lazy_gettext, gettext
+
+import hawat.forms
+
+class SimpleOverallDashboardForm(flask_wtf.FlaskForm):
+ """
+ Class representing simple overall dashboard search form.
+ """
+ dt_from = hawat.forms.DateTimeLocalField(
+ lazy_gettext('From:'),
+ validators = [
+ wtforms.validators.Optional()
+ ],
+ format = '%Y-%m-%d'
+ )
+ interval = wtforms.SelectField(
+ lazy_gettext('Interval:'),
+ validators = [
+ wtforms.validators.Optional()
+ ],
+ choices = [
+ ('1day', lazy_gettext('one day')),
+ ('1week', lazy_gettext('one week')),
+ ('4weeks', lazy_gettext('four weeks'))
+ ],
+ filters = [lambda x: x or None]
+ )
+ submit = wtforms.SubmitField(
+ lazy_gettext('Calculate')
+ )
+
+class OverallDashboardForm(flask_wtf.FlaskForm):
+ """
+ Class representing overall dashboard search form.
+ """
+ dt_from = hawat.forms.DateTimeLocalField(
+ lazy_gettext('From:'),
+ validators = [
+ wtforms.validators.Optional()
+ ],
+ format = '%Y-%m-%d %H:%M',
+ default = datetime.datetime.fromtimestamp(int(time.time() - (time.time() % 86400)))
+ )
+ dt_to = hawat.forms.DateTimeLocalField(
+ lazy_gettext('To:'),
+ validators = [
+ wtforms.validators.Optional()
+ ],
+ format = '%Y-%m-%d %H:%M'
+ )
+ submit = wtforms.SubmitField(
+ lazy_gettext('Calculate')
+ )
diff --git a/lib/hawat/blueprints/dashboards/templates/dashboards/overall.html b/lib/hawat/blueprints/dashboards/templates/dashboards/overall.html
new file mode 100644
index 0000000000000000000000000000000000000000..d819cd1632bc22da8545feb5605a5e30bcd207b3
--- /dev/null
+++ b/lib/hawat/blueprints/dashboards/templates/dashboards/overall.html
@@ -0,0 +1,209 @@
+{% extends "_layout.html" %}
+
+{% block content %}
+
+ <div class="row">
+ <div class="col-lg-12">
+
+ <div class="jumbotron" style="margin-top: 1em;">
+ <h2>{{ hawat_view_title }}</h2>
+ <hr>
+ <form method="GET" class="form-inline" action="{{ url_for('dashboards.overall') }}">
+ {{ macros_site.render_form_item_datetime(search_form.dt_from, 'datetimepicker-hm-from', False) }}
+
+ {{ macros_site.render_form_item_datetime(search_form.dt_to, 'datetimepicker-hm-to', False) }}
+
+ {{ search_form.submit(class_='btn btn-primary') }}
+ </form>
+ {%- if search_form.dt_from.errors %}
+ <div>
+ {{ macros_site.form_errors(search_form.dt_from.errors) }}
+ </div>
+ {%- endif %}
+ {%- if search_form.dt_to.errors %}
+ <div>
+ {{ macros_site.form_errors(search_form.dt_to.errors) }}
+ </div>
+ {%- endif %}
+ </div> <!-- /.jumbotron -->
+
+ </div> <!-- /.col-lg-12 -->
+ </div> <!-- /.row -->
+
+ {%- if items %}
+
+ <div class="row">
+ <div class="col-lg-12">
+
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <h3 class="panel-title">{{ gettext('Result summary') }}</h3>
+ </div>
+ <table class="table">
+ <tr>
+ <th>{{ gettext('Number of statistical records') }}:</th>
+ <td>{{ babel_format_decimal(items_count) }}</td>
+ </tr>
+ <tr>
+ <th>{{ gettext('Number of events') }}:</th>
+ <td>{{ babel_format_decimal(statistics['count']) }}</td>
+ </tr>
+ <tr>
+ <th>{{ gettext('Time interval') }}:</th>
+ <td>{{ babel_format_datetime(statistics['dt_from']) }} - {{ babel_format_datetime(statistics['dt_to']) }}</td>
+ </tr>
+ <tr>
+ <th>{{ gettext('Time interval delta') }}:</th>
+ <td>{{ babel_format_timedelta(statistics['dt_to'] - statistics['dt_from']) }}</td>
+ </tr>
+ </table>
+ </div>
+
+ </div> <!-- /.col-lg-12 -->
+ </div> <!-- /.row -->
+
+ <!-- Nav tabs -->
+ <ul class="nav nav-tabs" role="tablist">
+ <li role="presentation" class="active">
+ <a href="#tab-stats-overall" aria-controls="tab-stats-overall" role="tab" data-toggle="tab">
+ <strong>{{ gettext('Overall processing statistics') }}</strong>
+ </a>
+ </li>
+ <li role="presentation">
+ <a href="#tab-stats-internal" aria-controls="tab-stats-internal" role="tab" data-toggle="tab">
+ <strong>{{ gettext('Internal processing statistics') }}</strong>
+ </a>
+ </li>
+ <li role="presentation">
+ <a href="#tab-stats-external" aria-controls="tab-stats-external" role="tab" data-toggle="tab">
+ <strong>{{ gettext('External processing statistics') }}</strong>
+ </a>
+ </li>
+ </ul>
+
+ <!-- Tab panes -->
+ <div class="tab-content">
+
+ <div role="tabpanel" class="tab-pane fade in active" id="tab-stats-overall">
+ <ul class="nav nav-tabs nav-tabs-tooltipped" role="tablist">
+ {%- for chsection in ('abuses', 'analyzers', 'asns', 'categories', 'category_sets', 'countries', 'detectors', 'detectorsws', 'ips') %}
+ <li role="presentation" class="text-center{% if loop.first %} active{% endif %}">
+ <a role="tab" data-toggle="tab" href="#tab-stats-overall-{{ chsection }}" class="chart-tab">
+ # {{ chsection }}
+ </a>
+ </li>
+ {%- endfor %}
+ </ul>
+ <div class="tab-content">
+ {%- for chsection in ('abuses', 'analyzers', 'asns', 'categories', 'category_sets', 'countries', 'detectors', 'detectorsws', 'ips') %}
+ <div role="tabpanel" class="tab-pane fade{% if loop.first %} in active{% endif %}" id="tab-stats-overall-{{ chsection }}">
+ {{ macros_site.render_chart_pie(
+ 'stats_overall',
+ statistics['stats_overall'],
+ chsection,
+ 'Number of events per ' + chsection,
+ )
+ }}
+ </div>
+ {%- endfor %}
+ </div>
+ </div>
+
+ <div role="tabpanel" class="tab-pane fade" id="tab-stats-internal">
+ <ul class="nav nav-tabs nav-tabs-tooltipped" role="tablist">
+ {%- for chsection in ('abuses', 'analyzers', 'asns', 'categories', 'category_sets', 'countries', 'detectors', 'detectorsws', 'ips') %}
+ <li role="presentation" class="text-center{% if loop.first %} active{% endif %}">
+ <a role="tab" data-toggle="tab" href="#tab-stats-internal-{{ chsection }}" class="chart-tab">
+ # {{ chsection }}
+ </a>
+ </li>
+ {%- endfor %}
+ </ul>
+ <div class="tab-content">
+ {%- for chsection in ('abuses', 'analyzers', 'asns', 'categories', 'category_sets', 'countries', 'detectors', 'detectorsws', 'ips') %}
+ <div role="tabpanel" class="tab-pane fade{% if loop.first %} in active{% endif %}" id="tab-stats-internal-{{ chsection }}">
+ {{ macros_site.render_chart_pie(
+ 'stats_internal',
+ statistics['stats_internal'],
+ chsection,
+ 'Number of events per ' + chsection,
+ )
+ }}
+ </div>
+ {%- endfor %}
+ </div>
+ </div>
+
+ <div role="tabpanel" class="tab-pane fade" id="tab-stats-external">
+ <ul class="nav nav-tabs nav-tabs-tooltipped" role="tablist">
+ {%- for chsection in ('analyzers', 'asns', 'categories', 'category_sets', 'countries', 'detectors', 'detectorsws', 'ips') %}
+ <li role="presentation" class="text-center{% if loop.first %} active{% endif %}">
+ <a role="tab" data-toggle="tab" href="#tab-stats-external-{{ chsection }}" class="chart-tab">
+ # {{ chsection }}
+ </a>
+ </li>
+ {%- endfor %}
+ </ul>
+ <div class="tab-content">
+ {%- for chsection in ('abuses', 'analyzers', 'asns', 'categories', 'category_sets', 'countries', 'detectors', 'detectorsws', 'ips') %}
+ <div role="tabpanel" class="tab-pane fade{% if loop.first %} in active{% endif %}" id="tab-stats-external-{{ chsection }}">
+ {{ macros_site.render_chart_pie(
+ 'stats_external',
+ statistics['stats_external'],
+ chsection,
+ 'Number of events per ' + chsection,
+ )
+ }}
+ </div>
+ {%- endfor %}
+ </div>
+ </div>
+
+ </div>
+
+ {%- if permission_can('developer') %}
+ {%- if item %}
+
+ <hr>
+
+{{ macros_site.render_raw_item_view(item) }}
+
+ {%- endif %}
+ {%- endif %}
+
+ {%- else %}
+ <div class="alert alert-info">{{ gettext('No data matches your search criteria.') }}</div>
+ {%- endif %}
+
+{%- endblock content %}
+
+
+{%- block css %}
+{{ super() }}
+ <link rel="stylesheet" href="{{ url_for('design.static', filename='vendor/nvd3/css/nv.d3.min.css') }}">
+{%- endblock css %}
+
+
+{%- block headjs %}
+{{ super() }}
+ <script src="{{ url_for('design.static', filename='vendor/d3/js/d3.min.js') }}"></script>
+ <script src="{{ url_for('design.static', filename='vendor/nvd3/js/nv.d3.min.js') }}"></script>
+{%- endblock headjs %}
+
+
+{%- block js %}
+{{ super() }}
+ <script src="{{ url_for('design.static', filename='vendor/datatables/js/jquery.dataTables.js') }}"></script>
+ <script src="{{ url_for('design.static', filename='vendor/datatables/js/dataTables.bootstrap.js') }}"></script>
+
+ <script>
+ $(document).ready(function() {
+ $('.hawat-datatable-nopaging').DataTable({
+ "responsive": true,
+ "pageLength": 50,
+ "paging": false,
+ "ordering": true
+ });
+ });
+ </script>
+{%- endblock js %}
diff --git a/lib/hawat/blueprints/design/static/css/hawat.css b/lib/hawat/blueprints/design/static/css/hawat.css
index 6c35acd94ba9bb49f38fe0bb8ec27f0671b25889..2342a7d5b53f10f98625b267843f6ef5a6032d88 100644
--- a/lib/hawat/blueprints/design/static/css/hawat.css
+++ b/lib/hawat/blueprints/design/static/css/hawat.css
@@ -75,3 +75,10 @@ body {
.rrd-button-group {
padding-left: 1em !important;
}
+
+/*
+ * Fix for wrong indentation of search field in datatables.
+ * */
+#DataTables_Table_0_filter {
+ text-align: right;
+}
diff --git a/lib/hawat/blueprints/design/static/js/hawat-common.js b/lib/hawat/blueprints/design/static/js/hawat-common.js
index 80feb22bd9a6b07908c0518b2d924ffa262ab13d..6bb56b3ce89bb5b8b19aaf01f4b84b751a255d02 100644
--- a/lib/hawat/blueprints/design/static/js/hawat-common.js
+++ b/lib/hawat/blueprints/design/static/js/hawat-common.js
@@ -39,6 +39,35 @@ $(function() {
$('#datetimepicker-from').data("DateTimePicker").maxDate(e.date);
});
+ // Initialize linked date and datetime picker components.
+ $('#datetimepicker-hm-from').datetimepicker({
+ locale: GLOBAL_LOCALE,
+ format: 'YYYY-MM-DD HH:mm',
+ icons: {
+ time: "fa fa-fw fa-clock-o",
+ date: "fa fa-fw fa-calendar",
+ up: "fa fa-fw fa-arrow-up",
+ down: "fa fa-fw fa-arrow-down"
+ }
+ });
+ $('#datetimepicker-hm-to').datetimepicker({
+ useCurrent: false, //Important! See issue #1075
+ locale: GLOBAL_LOCALE,
+ format: 'YYYY-MM-DD HH:mm',
+ icons: {
+ time: "fa fa-fw fa-clock-o",
+ date: "fa fa-fw fa-calendar",
+ up: "fa fa-fw fa-arrow-up",
+ down: "fa fa-fw fa-arrow-down"
+ }
+ });
+ $("#datetimepicker-hm-from").on("dp.change", function (e) {
+ $('#datetimepicker-hm-to').data("DateTimePicker").minDate(e.date);
+ });
+ $("#datetimepicker-hm-to").on("dp.change", function (e) {
+ $('#datetimepicker-hm-from').data("DateTimePicker").maxDate(e.date);
+ });
+
// Initialize select pickers.
//$('.selectpicker').selectpicker();
diff --git a/lib/hawat/blueprints/design/templates/_layout.html b/lib/hawat/blueprints/design/templates/_layout.html
index 15257653914a5fa0749746302b31d7dbdfff7f18..5430088ae0c70b7de2e2ebd4bd8eab85cbad7041 100644
--- a/lib/hawat/blueprints/design/templates/_layout.html
+++ b/lib/hawat/blueprints/design/templates/_layout.html
@@ -20,6 +20,10 @@
<link rel="stylesheet" href="{{ url_for('design.static', filename='vendor/bootstrap-datetimepicker/css/bootstrap-datetimepicker.min.css') }}">
<link rel="stylesheet" href="{{ url_for('design.static', filename='css/hawat.css') }}">
{%- endblock css %}
+ {%- block headjs %}
+
+ <!-- Head JS -->
+ {%- endblock headjs %}
<!-- Favicon -->
<link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}">
diff --git a/lib/hawat/blueprints/design/templates/_macros_site.html b/lib/hawat/blueprints/design/templates/_macros_site.html
index d82d22d07a73c006c06066224a9647c4a9589d43..ee21c782b20785cc7eca8b83e6b9d15643051db2 100644
--- a/lib/hawat/blueprints/design/templates/_macros_site.html
+++ b/lib/hawat/blueprints/design/templates/_macros_site.html
@@ -1,4 +1,4 @@
-{#- ----------------------------------------------------------------------------
+;{#- ----------------------------------------------------------------------------
Macro for rendering the main menu of the application.
@@ -166,13 +166,15 @@
----------------------------------------------------------------------------- #}
-{%- macro render_form_item_default(form_item) %}
+{%- macro render_form_item_default(form_item, with_errors = True) %}
<div class="form-group{% if form_item.errors %}{{ ' has-error' }}{% endif %}">
{{ form_item.label }}
{{ form_item(class_='form-control') }}
- {%- for err in form_item.errors %}
+ {%- if with_errors %}
+ {%- for err in form_item.errors %}
<span class="help-block form-error">{{ get_fa_icon('form-error') }} {{ err }}</span>{%- if not loop.last %}<br>{%- endif %}
- {%- endfor %}
+ {%- endfor %}
+ {%- endif %}
</div>
{%- endmacro %}
@@ -183,13 +185,15 @@
----------------------------------------------------------------------------- #}
-{%- macro render_form_item_select(form_item) %}
+{%- macro render_form_item_select(form_item, with_errors = True) %}
<div class="form-group{% if form_item.errors %}{{ ' has-error' }}{% endif %}">
{{ form_item.label }}
{{ form_item(class_='form-control selectpicker',**{'data-live-search':'true', 'data-size': '10', 'data-selected-text-format': 'count > 3'}) }}
- {%- for err in form_item.errors %}
+ {%- if with_errors %}
+ {%- for err in form_item.errors %}
<span class="help-block form-error">{{ get_fa_icon('form-error') }} {{ err }}</span>{%- if not loop.last %}<br>{%- endif %}
- {%- endfor %}
+ {%- endfor %}
+ {%- endif %}
</div>
{%- endmacro %}
@@ -200,7 +204,7 @@
----------------------------------------------------------------------------- #}
-{%- macro render_form_item_datetime(form_item, ident) %}
+{%- macro render_form_item_datetime(form_item, ident, with_errors = True) %}
<div class="form-group{% if form_item.errors %}{{ ' has-error' }}{% endif %}">
{{ form_item.label }}
<div id="{{ ident }}" class="input-group date">
@@ -209,9 +213,11 @@
<i class="fa fa-fw fa-calendar"></i>
</span>
</div>
- {%- for err in form_item.errors %}
+ {%- if with_errors %}
+ {%- for err in form_item.errors %}
<span class="help-block form-error">{{ get_fa_icon('form-error') }} {{ err }}</span>{%- if not loop.last %}<br>{%- endif %}
- {%- endfor %}
+ {%- endfor %}
+ {%- endif %}
</div>
{%- endmacro %}
@@ -236,7 +242,7 @@
----------------------------------------------------------------------------- #}
-{%- macro render_form_item_radiobutton(form_item) %}
+{%- macro render_form_item_radiobutton(form_item, with_errors = True) %}
<div class="form-group{% if form_item.errors %}{{ ' has-error' }}{% endif %}">
<div>
{{ form_item.label }}
@@ -247,10 +253,13 @@
{{ subfield }} {{ subfield.label }}
</span>
{%- endfor %}
+
</div>
- {%- for err in form_item.errors %}
+ {%- if with_errors %}
+ {%- for err in form_item.errors %}
<span class="help-block form-error">{{ get_fa_icon('form-error') }} {{ err }}</span>{%- if not loop.last %}<br>{%- endif %}
- {%- endfor %}
+ {%- endfor %}
+ {%- endif %}
</div>
{%- endmacro %}
@@ -297,3 +306,119 @@
<pre>{{ var | pprint }}</pre>
</div> <!-- /.well -->
{%- endmacro %}
+
+{#- ----------------------------------------------------------------------------
+
+ Macros for rendering NVD3 charts.
+
+----------------------------------------------------------------------------- #}
+
+{%- macro render_chart_pie(chidprefix, chstats, chstatkey, chtitle) %}
+{%- set chid = '' + chidprefix + '_' + chstatkey %}
+{%- set chdata = chstats[chstatkey] %}
+{%- set chkeys = chstats['list_' + chstatkey] %}
+ <h4>{{ chtitle }}</h4>
+ <div class="row">
+ <div class="col-md-6">
+
+ <div id="nvd3_chart_pie_{{ chid }}" class="span4">
+ <svg style='height:600px;width:600px' class='center-block'></svg>
+ </div>
+
+ <script>
+ nv.addGraph(function() {
+ var nvd3_data_pie_{{ chid }} = [
+ {%- for chitemkey in chkeys %}
+ { label: "{{ chitemkey }}", value: {{ chdata[chitemkey] }} },
+ {%- endfor %}
+ ];
+
+ var chart = nv.models.pieChart()
+ .x(function(d) { return d.label })
+ .y(function(d) { return d.value })
+ .showLabels(true)
+ .showLegend(true)
+ .labelThreshold(.05) //Configure the minimum slice size for labels to show up
+ .labelType("key") //Configure what type of data to show in the label. Can be "key", "value" or "percent"
+ .donut(true) //Turn on Donut mode. Makes pie chart look tasty!
+ .donutRatio(0.25) //Configure how big you want the donut hole size to be.
+ ;
+
+ d3.select("#nvd3_chart_pie_{{ chid }} svg")
+ .datum(nvd3_data_pie_{{ chid }})
+ .transition().duration(350)
+ .call(chart);
+
+ return chart;
+ });
+ </script>
+
+ </div>
+ <div class="col-md-6">
+
+ <table class="table table-bordered table-striped table-condensed hawat-datatable-nopaging">
+ <thead>
+ <tr>
+ <th> </th>
+ <th>Name</th>
+ <th>#</th>
+ <th>%</th>
+ </tr>
+ </thead>
+ <tbody>
+ {%- for chitemkey in chkeys %}
+ <tr>
+ <td style="text-align: right; font-weight: bold; background-color: white;">
+ {{ loop.index }}
+ </td>
+ <td>
+ {{ chitemkey }}
+ </td>
+ <td class="col-value">
+ {{ babel_format_decimal(chdata[chitemkey]) }}
+ </td>
+ <td>
+
+ </td>
+ </tr>
+ {%- endfor %}
+ </tbody>
+ <tfoot>
+ <tr>
+ <th> </th>
+ <th>Sum</th>
+ <th>{{ babel_format_decimal(chstats['sum_' + chstatkey]) }}</th>
+ <th></th>
+ </tr>
+ </tfoot>
+ </table>
+
+ <hr>
+
+ <table class="table table-bordered table-striped table-condensed">
+ <tr>
+ <th>{{ get_fa_icon('min') }} Min</th>
+ <td class="col-value">{{ babel_format_decimal(chstats['min_' + chstatkey]) }}</td>
+ </tr>
+ <tr>
+ <th>{{ get_fa_icon('max') }} Max</th>
+ <td class="col-value">{{ babel_format_decimal(chstats['max_' + chstatkey]) }}</td>
+ </tr>
+ <tr>
+ <th>{{ get_fa_icon('sum') }} Sum</th>
+ <td class="col-value">{{ babel_format_decimal(chstats['sum_' + chstatkey]) }}</td>
+ </tr>
+ <tr>
+ <th>{{ get_fa_icon('cnt') }} Cnt</th>
+ <td class="col-value">{{ babel_format_decimal(chstats['cnt_' + chstatkey]) }}</td>
+ </tr>
+ <tr>
+ <th>{{ get_fa_icon('avg') }} Avg</th>
+ <td class="col-value">{{ babel_format_decimal(chstats['avg_' + chstatkey]) }}</td>
+ </tr>
+ </table>
+
+ </div>
+ </div>
+
+{%- endmacro %}
diff --git a/lib/hawat/config.py b/lib/hawat/config.py
index 457c099e5cdccf38b2c4a4198ee03821eb304611..cd80aff078c185db4b20a84c2c8a0007f54f3882 100644
--- a/lib/hawat/config.py
+++ b/lib/hawat/config.py
@@ -112,6 +112,7 @@ class Config:
ENABLED_BLUEPRINTS = [
'hawat.blueprints.auth_env',
'hawat.blueprints.design',
+ 'hawat.blueprints.dashboards',
'hawat.blueprints.reports',
'hawat.blueprints.events',
'hawat.blueprints.geoip',
@@ -132,7 +133,7 @@ class Config:
HAWAT_MENU_SKELETON = [
{
- 'ident': 'boards',
+ 'ident': 'dashboards',
'content': {'type': 'submenu', 'title': lazy_gettext('Dashboards'), 'icon': 'dashboards', 'tooltip': lazy_gettext('Various dashboards')},
'position': 100
},
@@ -197,6 +198,7 @@ class DevelopmentConfig(Config):
'hawat.blueprints.auth_env',
'hawat.blueprints.auth_dev',
'hawat.blueprints.design',
+ 'hawat.blueprints.dashboards',
'hawat.blueprints.reports',
'hawat.blueprints.events',
'hawat.blueprints.geoip',
diff --git a/lib/mentat/module/statistician.py b/lib/mentat/module/statistician.py
index 736e15bc83ba59db7334b73f1d9fa11f2a16c69b..53efb2b76f3d080ad4d55fb2eb8fe1f57288ad7f 100644
--- a/lib/mentat/module/statistician.py
+++ b/lib/mentat/module/statistician.py
@@ -41,33 +41,6 @@ Available script commands
Calculate statistics for messages stored into database within configured
time interval thresholds.
-
-Custom configuration
---------------------
-
-Custom command line options
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-``--database-stats db-name``
- Name of the database for storing IDEA message statistics.
-
- *Type:* ``string``
-
-``--collection-stats col-name``
- Name of the collection for storing IDEA message statistics.
-
- *Type:* ``string``
-
-``--rrds-dir dir-name``
- Name of the directory for the RRD database files.
-
- *Type:* ``string``, *default:* ``/var/mentat/rrds``
-
-``--reports-dir dir-name``
- Name of the directory for RRD generated charts.
-
- *Type:* ``string``, *default:* ``/var/mentat/reports/statistician``
-
"""
@@ -84,6 +57,8 @@ import mentat.script.fetcher
import mentat.const
import mentat.stats.idea
import mentat.stats.rrd
+import mentat.plugin.app.sqlstorage
+import mentat.datatype.sqldb
# Current time (second precission)
@@ -101,9 +76,7 @@ class MentatStatisticianScript(mentat.script.fetcher.FetcherScript):
#
# List of configuration keys.
- CONFIG_DATABASE_STATS = 'database_stats'
- CONFIG_COLLECTION_STATS = 'collection_stats'
-
+ CORECFG_STATISTICS = '__core__statistics'
CONFIG_RRDS_DIR = 'rrds_dir'
CONFIG_REPORTS_DIR = 'reports_dir'
@@ -115,6 +88,9 @@ class MentatStatisticianScript(mentat.script.fetcher.FetcherScript):
and it aims to even more simplify the script object creation by providing
configuration values for parent contructor.
"""
+ # Declare private attributes.
+ self.sqlservice = None
+
super().__init__(
description = 'mentat-statistician.py - Mentat system statistical script',
@@ -131,78 +107,15 @@ class MentatStatisticianScript(mentat.script.fetcher.FetcherScript):
#
# Override default configurations.
#
- default_config_dir = '/etc/mentat/core'
- )
-
- def _init_argparser(self, **kwargs):
- """
- Initialize script command line argument parser. This method overrides the
- base implementation in :py:func:`pyzenkit.zenscript.ZenScript._init_argparser`
- and it must return valid :py:class:`argparse.ArgumentParser` object. It
- appends additional command line options custom for this script object.
-
- This method is called from the main constructor in :py:func:`pyzenkit.baseapp.BaseApp.__init__`
- as a part of the **__init__** stage of application`s life cycle.
-
- :param kwargs: Various additional parameters passed down from object constructor.
- :return: Valid argument parser object.
- :rtype: argparse.ArgumentParser
- """
- argparser = super()._init_argparser(**kwargs)
+ default_config_dir = '/etc/mentat/core',
- #
- # Create and populate options group for custom script arguments.
- #
- arggroup_script = argparser.add_argument_group('custom script arguments')
-
- arggroup_script.add_argument('--database-stats', type = str, default = None, help = 'name of the database for storing IDEA message statistics')
- arggroup_script.add_argument('--collection-stats', type = str, default = None, help = 'name of the collection for storing IDEA message statistics')
- arggroup_script.add_argument('--rrds-dir', type = str, default = None, help = 'name of the directory for the RRD database files')
- arggroup_script.add_argument('--reports-dir', type = str, default = None, help = 'name of the directory for RRD generated charts')
-
- return argparser
-
- def _init_config(self, cfgs, **kwargs):
- """
- Initialize default script configurations. This method overrides the base
- implementation in :py:func:`pyzenkit.zenscript.ZenScript._init_config`
- and it appends additional configurations via ``cfgs`` parameter.
-
- This method is called from the main constructor in :py:func:`pyzenkit.baseapp.BaseApp.__init__`
- as a part of the **__init__** stage of application`s life cycle.
-
- :param list cfgs: Additional set of configurations.
- :param kwargs: Various additional parameters passed down from constructor.
- :return: Default configuration structure.
- :rtype: dict
- """
- cfgs = (
- (self.CONFIG_DATABASE_STATS, None),
- (self.CONFIG_COLLECTION_STATS, None),
- (self.CONFIG_RRDS_DIR, '/var/mentat/rrds'),
- (self.CONFIG_REPORTS_DIR, '/var/mentat/reports/statistician')
- ) + cfgs
- return super()._init_config(cfgs, **kwargs)
-
- def _configure_postprocess(self):
- """
- Perform postprocessing of configuration values and calculate *core* configurations.
-
- This method is called from the :py:func:`pyzenkit.baseapp.BaseApp._stage_setup_configuration`
- as a port of the **setup** stage of application`s life cycle.
- """
- super()._configure_postprocess()
-
- # Prepare shortcuts for core database settings.
- db_settings = self.c(mentat.const.CKEY_CORE_DATABASE)
- db_config = db_settings[mentat.const.CKEY_CORE_DATABASE_CONFIG]
-
- # Configure undefined database settings from core settings.
- for cfg in (
- (self.CONFIG_DATABASE_STATS, db_config['db_stats']),
- (self.CONFIG_COLLECTION_STATS, db_config['col_stats_alerts'])):
- if self.config[cfg[0]] is None:
- self.config[cfg[0]] = cfg[1]
+ #
+ # Load additional application-level plugins.
+ #
+ plugins = [
+ mentat.plugin.app.sqlstorage.SQLStoragePlugin()
+ ]
+ )
def _sub_stage_init(self, **kwargs):
"""
@@ -226,14 +139,10 @@ class MentatStatisticianScript(mentat.script.fetcher.FetcherScript):
This method is called from the main setup method :py:func:`pyzenkit.baseapp.BaseApp._stage_setup`
as a part of the **setup** stage of application`s life cycle.
"""
- self.database_stats_name = self.c(self.CONFIG_DATABASE_STATS)
- self.collection_stats_name = self.c(self.CONFIG_COLLECTION_STATS)
-
- self.database_stats = self.storage.database(self.database_stats_name)
- self.collection_stats = self.database_stats.collection(self.collection_stats_name)
- self.logger.info("Connected to stats database collection '%s':'%s'", self.database_stats_name, self.collection_stats_name)
-
- self.stats_rrd = mentat.stats.rrd.RrdStats(rrds_dir = self.c(self.CONFIG_RRDS_DIR), reports_dir = self.c(self.CONFIG_REPORTS_DIR))
+ self.stats_rrd = mentat.stats.rrd.RrdStats(
+ rrds_dir = self.config[self.CORECFG_STATISTICS][self.CONFIG_RRDS_DIR],
+ reports_dir = self.config[self.CORECFG_STATISTICS][self.CONFIG_REPORTS_DIR]
+ )
#---------------------------------------------------------------------------
@@ -264,27 +173,26 @@ class MentatStatisticianScript(mentat.script.fetcher.FetcherScript):
interval = self.c(self.CONFIG_INTERVAL),
adjust = self.c(self.CONFIG_REGULAR)
)
- self.logger.info("Lower statistics calculation time interval threshold: %s (%s)", time_low.strftime('%Y-%m-%d %H:%M'), time_low.timestamp())
- self.logger.info("Upper statistics calculation time interval threshold: %s (%s)", time_high.strftime('%Y-%m-%d %H:%M'), time_high.timestamp())
+ self.logger.info("Lower statistics calculation time interval threshold: %s (%s)", time_low.strftime('%FT%T'), time_low.timestamp())
+ self.logger.info("Upper statistics calculation time interval threshold: %s (%s)", time_high.strftime('%FT%T'), time_high.timestamp())
- result['ts_from_s'] = str(time_low)
- result['ts_to_s'] = str(time_high)
+ result['ts_from_s'] = time_low.strftime('%FT%T')
+ result['ts_to_s'] = time_high.strftime('%FT%T')
result['ts_from'] = int(time_low.timestamp())
result['ts_to'] = int(time_high.timestamp())
- result['_id'] = '{}_{}'.format(result['ts_from'], result['ts_to'])
+ result['interval'] = '{}_{}'.format(result['ts_from_s'], result['ts_to_s'])
messages = self.fetch_messages(time_low, time_high)
- result = mentat.stats.idea.evaluate_messages_full(messages, result)
+ result = mentat.stats.idea.evaluate_event_groups(messages, result)
self._update_rrds(result, time_high)
self._generate_charts(time_high)
- result = mentat.stats.idea.brief_stats_full(result)
- result = mentat.stats.idea.escape_stats_full(result)
+ result = mentat.stats.idea.truncate_evaluations(result)
- result = self._update_db(result)
+ result = self._update_db(result, time_low, time_high)
return result
@@ -361,19 +269,32 @@ class MentatStatisticianScript(mentat.script.fetcher.FetcherScript):
self.runlog['generated_files'] = result
- def _update_db(self, stats):
+ def _update_db(self, stats, time_low, time_high):
"""
- Store given message statistics into database.
+ Store given event statistics into database.
- :param dict stats: Message statistics to store.
+ :param dict stats: Event statistics to store.
"""
stats['ts'] = int(time.time())
try:
- db_id = self.collection_stats.insert(stats)
- self.logger.info("Stored statistics log to database document '%s'", db_id)
+ sql_stats = mentat.datatype.sqldb.EventStatisticsModel(
+ dt_from = time_low,
+ dt_to = time_high,
+ count = stats[mentat.stats.idea.ST_SKEY_COUNT],
+ stats_overall = stats.get(mentat.stats.idea.ST_OVERALL, {}),
+ stats_internal = stats.get(mentat.stats.idea.ST_INTERNAL, {}),
+ stats_external = stats.get(mentat.stats.idea.ST_EXTERNAL, {})
+ )
+ sql_stats.calculate_interval()
+ sql_stats.calculate_delta()
+
+ self.sqlservice.session.add(sql_stats)
+ self.sqlservice.session.commit()
+
+ self.logger.info("Stored event statistics log to database document '%s'", sql_stats.id)
stats['flag_stored'] = True
- stats['db_id'] = db_id
+ stats['db_id'] = sql_stats.id
except Exception as exc:
self.logger.error(str(exc))
diff --git a/lib/mentat/stats/idea.py b/lib/mentat/stats/idea.py
index 8c98f0ae71e64a1cecb24ed9f92dc2795e246f11..7e870aff4c34a309601c6d36081a8cf4ebd10696 100644
--- a/lib/mentat/stats/idea.py
+++ b/lib/mentat/stats/idea.py
@@ -17,18 +17,18 @@ __author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-from mentat.stats import escape_dict, unescape_dict
from pynspect.jpath import jpath_values
-
+from mentat.stats import escape_dict, unescape_dict
+from mentat.datatype.sqldb import EventStatisticsModel
KEY_UNKNOWN = '__unknown__'
#
# Literals for keywords of statistical categories
#
-ST_INTERNAL = 'internal'
-ST_EXTERNAL = 'external'
-ST_OVERALL = 'overall'
+ST_INTERNAL = 'stats_internal'
+ST_EXTERNAL = 'stats_external'
+ST_OVERALL = 'stats_overall'
#
# Literals for keywords of calculated statistics
@@ -44,17 +44,11 @@ ST_SKEY_DETECTORSWS = 'detectorsws'
ST_SKEY_ABUSES = 'abuses'
ST_SKEY_ASNS = 'asns'
ST_SKEY_COUNTRIES = 'countries'
-
-ST_SKEY_REST = '__REST__'
-
+ST_SKEY_LIST_IDS = 'list_ids'
ST_SKEY_CNT_ALERTS = 'cnt_alerts'
-
-ST_LIST_IDS = 'list_ids'
-ST_LIST_IPS = 'list_ips'
-ST_LIST_IP4S = 'list_ip4s'
-ST_LIST_IP6S = 'list_ip6s'
-ST_LIST_ASNS = 'list_asns'
-ST_LIST_COUNTRIES = 'list_countries'
+ST_SKEY_COUNT = 'count'
+ST_SKEY_HISTOGRAM = 'histogram'
+ST_SKEY_REST = '__REST__'
LIST_CALCSTAT_KEYS = (
ST_SKEY_IPS,
@@ -88,43 +82,6 @@ LIST_AGGREGATIONS = (
#-------------------------------------------------------------------------------
-def _counter_inc(stats, stat, key, increment = 1):
- """
- Helper for incrementing given statistical parameter within given statistical
- bundle.
-
- :param dict stats: Structure containing all statistics.
- :param str stat: Name of the statistic category.
- :param str key: Name of the statistic key.
- :param int increment: Counter increment.
- :return: Updated structure containing statistics.
- :rtype: dict
- """
- if not stat in stats:
- stats[stat] = {}
- stats[stat][str(key)] = stats[stat].get(str(key), 0) + increment
- return stats
-
-
-def escape_stats(stats):
- """
- Escape dots ``.`` in statistic keyword names with ``(dot)`` (because ``.`` is reserved
- in MongoDB).
-
- .. todo::
-
- Perhaps move this feature into :py:mod:`mentat.datatype.mongodb` library.
-
- :param dict stats: Structure containing single statistic category.
- :return: Updated structure containing statistics.
- :rtype: dict
- """
- for key in LIST_CALCSTAT_KEYS:
- if key in stats:
- stats[key] = escape_dict(stats[key])
- return stats
-
-
def unescape_stats(stats):
"""
Unescape ``(dot)``s in statistic keyword names with ``.`` (because ``.`` is reserved
@@ -143,25 +100,6 @@ def unescape_stats(stats):
stats[key] = unescape_dict(stats[key])
return stats
-
-def escape_stats_full(stats):
- """
- Escape all statistic categories with :py:func:`escape_stats`.
-
- .. todo::
-
- Perhaps move this feature into :py:mod:`mentat.datatype.mongodb` library.
-
- :param dict stats: Structure containing single statistic category.
- :return: Updated structure containing statistics.
- :rtype: dict
- """
- for key in LIST_STAT_GROUPS:
- if key in stats:
- stats[key] = escape_stats(stats[key])
- return stats
-
-
def unescape_stats_full(stats):
"""
Unescape all statistic categories with :py:func:`unescape_stats`.
@@ -174,13 +112,12 @@ def unescape_stats_full(stats):
:return: Updated structure containing statistics.
:rtype: dict
"""
- for key in LIST_STAT_GROUPS:
+ for key in ('internal', 'external', 'overall'):
if key in stats:
stats[key] = unescape_stats(stats[key])
return stats
-
-def brief_stats(stats, top_threshold = 20):
+def truncate_stats(stats, top_threshold = 20):
"""
Make statistics more brief.
@@ -190,18 +127,19 @@ def brief_stats(stats, top_threshold = 20):
:rtype: dict
"""
if stats[ST_SKEY_CNT_ALERTS] > 0:
- del stats[ST_LIST_IDS]
+ if ST_SKEY_LIST_IDS in stats:
+ del stats[ST_SKEY_LIST_IDS]
- stats = _make_toplist(stats, ST_SKEY_IPS, ST_LIST_IPS, top_threshold)
- stats = _make_toplist(stats, ST_SKEY_ASNS, ST_LIST_ASNS, top_threshold)
- stats = _make_toplist(stats, ST_SKEY_COUNTRIES, ST_LIST_COUNTRIES, top_threshold)
+ for key in LIST_CALCSTAT_KEYS:
+ if key in stats:
+ stats = _make_toplist(stats, key, top_threshold)
return stats
-def brief_stats_full(stats, top_threshold = 20):
+def truncate_evaluations(stats, top_threshold = 20):
"""
- Make all statistic categories more brief with :py:func:`brief_stats`.
+ Make all statistic categories more brief with :py:func:`truncate_stats`.
:param dict stats: Structure containing single statistic category.
:param int top_threshold: Toplist threshold size.
@@ -209,22 +147,23 @@ def brief_stats_full(stats, top_threshold = 20):
:rtype: dict
"""
for key in LIST_STAT_GROUPS:
- stats[key] = brief_stats(stats[key], top_threshold)
+ if key in stats:
+ stats[key] = truncate_stats(stats[key], top_threshold)
return stats
-def group_messages(messages):
+def group_events(events):
"""
Group mesages according to the presence of ``_CESNET.ResolvedAbuses`` key.
Each message will be added to group ``overall`` and then to either ``internal``,
or ``external`` based on the presence of the key mentioned above.
- :param list messages: List of messages to be grouped.
- :return: Structure containing message groups ``overall``, ``internal`` and ``external``.
+ :param list events: List of IDEA events to be grouped.
+ :return: Structure containing message groups ``stats_overall``, ``stats_internal`` and ``stats_external``.
:rtype: dict
"""
result = {ST_OVERALL: [], ST_INTERNAL: [], ST_EXTERNAL: []}
- for msg in messages:
+ for msg in events:
result[ST_OVERALL].append(msg)
values = jpath_values(msg, '_CESNET.ResolvedAbuses')
if values:
@@ -237,11 +176,11 @@ def group_messages(messages):
#-------------------------------------------------------------------------------
-def evaluate_messages(messages, stats = None):
+def evaluate_events(events, stats = None):
"""
- Evaluate statistics for given list of IDEA messages.
+ Evaluate statistics for given list of IDEA events.
- :param list messages: List of messages to be evaluated.
+ :param list events: List of IDEA events to be evaluated.
:param dict stats: Data structure to which to append calculated statistics.
:return: Structure containing evaluated message statistics.
:rtype: dict
@@ -249,18 +188,18 @@ def evaluate_messages(messages, stats = None):
if stats is None:
stats = dict()
- stats[ST_SKEY_CNT_ALERTS] = len(messages)
+ stats[ST_SKEY_CNT_ALERTS] = len(events)
# Do not calculate anything for empty message list.
if not stats[ST_SKEY_CNT_ALERTS]:
return stats
# Prepare structure for storing IDEA message identifiers.
- if ST_LIST_IDS not in stats:
- stats[ST_LIST_IDS] = []
+ if ST_SKEY_LIST_IDS not in stats:
+ stats[ST_SKEY_LIST_IDS] = []
- for msg in messages:
- stats[ST_LIST_IDS].append(msg['ID'])
+ for msg in events:
+ stats[ST_SKEY_LIST_IDS].append(msg['ID'])
reg = {}
@@ -293,53 +232,163 @@ def evaluate_messages(messages, stats = None):
# Calculate secondary statistics (cnt, min, max, sum, avg).
for key in LIST_CALCSTAT_KEYS:
if key in stats:
- stats['cnt_{}'.format(key)] = len(stats[key])
- stats['sum_{}'.format(key)] = sum(stats[key].values())
- stats['min_{}'.format(key)] = min(stats[key].values())
- stats['max_{}'.format(key)] = max(stats[key].values())
- stats['avg_{}'.format(key)] = stats['sum_{}'.format(key)]/stats['cnt_{}'.format(key)]
- stats['list_{}'.format(key)] = list(sorted(stats[key].keys()))
+ stats = _calculate_substats(stats, key)
return stats
-def evaluate_messages_full(messages, stats = None):
+def evaluate_event_groups(events, stats = None):
"""
- Evaluate full statistics for given list of IDEA messages. Messages will be
- gruped using :py:func:`group_messages` first and the statistics will be
- evaluated separatelly for each of message groups ``overall``, ``internal``
- and ``external``.
-
- :param list messages: List of messages to be evaluated.
- :param dict stats: Data structure to which to append calculated statistics.
- :return: Structure containing evaluated message statistics.
+ Evaluate full statistics for given list of IDEA events. Events will be
+ grouped using :py:func:`group_events` first and the statistics will be
+ evaluated separatelly for each of message groups ``stats_overall``,
+ ``stats_internal`` and ``external``.
+
+ :param list events: List of IDEA events to be evaluated.
+ :param dict stats: Optional dictionary structure to populate with statistics.
+ :return: Structure containing evaluated event statistics.
:rtype: dict
"""
if stats is None:
stats = dict()
+ stats[ST_SKEY_COUNT] = len(events)
- msg_groups = group_messages(messages)
+ msg_groups = group_events(events)
for grp_key in LIST_STAT_GROUPS:
- stats[grp_key] = evaluate_messages(msg_groups.get(grp_key, []))
-
+ stats[grp_key] = evaluate_events(msg_groups.get(grp_key, []))
return stats
+def aggregate_stats(stats, interval, dt_from, result = None):
+ """
+ :param dict stats: Optional dictionary structure to populate with statistics.
+ :param str interval: String label describing time interval.
+ :param datetime.Datetime dt_from: Timestamp of the time interval beginning.
+ :param dict result: Structure for aggregated result.
+ :return: Structure containing aggregated event statistics.
+ :rtype: dict
+ """
+ if not result:
+ result = {
+ ST_SKEY_CNT_ALERTS: 0,
+ ST_SKEY_HISTOGRAM: []
+ }
+
+ result[ST_SKEY_CNT_ALERTS] += stats[ST_SKEY_CNT_ALERTS]
+ result[ST_SKEY_HISTOGRAM].append((
+ interval,
+ dt_from,
+ stats[ST_SKEY_CNT_ALERTS]
+ ))
+
+ for key in LIST_CALCSTAT_KEYS:
+ if key in stats:
+ for subkey, subval in stats[key].items():
+ result = _counter_inc(result, key, subkey, subval)
+
+ for key in LIST_CALCSTAT_KEYS:
+ if key in result:
+ result = _calculate_substats(result, key)
+
+ return result
+
+def aggregate_stat_groups(stats_list, result = None):
+ """
+
+ """
+ if result is None:
+ result = dict()
+ result[ST_SKEY_COUNT] = 0
+ result[ST_SKEY_HISTOGRAM] = []
+
+ for stat in stats_list:
+ result[ST_SKEY_COUNT] += stat.count
+ result[ST_SKEY_HISTOGRAM].append((
+ stat.interval,
+ stat.dt_from,
+ stat.count
+ ))
+
+ if 'dt_from' in result:
+ result['dt_from'] = min(result['dt_from'], stat.dt_from)
+ else:
+ result['dt_from'] = stat.dt_from
+ if 'dt_to' in result:
+ result['dt_to'] = max(result['dt_to'], stat.dt_to)
+ else:
+ result['dt_to'] = stat.dt_to
+
+ if not stat.count:
+ continue
+
+ for grp_key in LIST_STAT_GROUPS:
+ result[grp_key] = aggregate_stats(
+ getattr(stat, grp_key),
+ stat.interval,
+ stat.dt_from,
+ result.get(grp_key, {})
+ )
+
+ return result
+
+
#-------------------------------------------------------------------------------
-def _make_toplist(stats, dict_key, list_key, top_threshold):
+def _counter_inc(stats, stat, key, increment = 1):
+ """
+ Helper for incrementing given statistical parameter within given statistical
+ bundle.
+
+ :param dict stats: Structure containing all statistics.
+ :param str stat: Name of the statistic category.
+ :param str key: Name of the statistic key.
+ :param int increment: Counter increment.
+ :return: Updated structure containing statistics.
+ :rtype: dict
+ """
+ if not stat in stats:
+ stats[stat] = {}
+ stats[stat][str(key)] = stats[stat].get(str(key), 0) + increment
+ return stats
+
+def _calculate_substats(stats, key):
+ """
+ Calculate substatistics for given key within given statistics.
+
+ :param dict stats: Structure containing event statistics.
+ :param str key: Designated key within structure.
+ :return: Enriched event statistics.
+ :rtype: dict
+ """
+ stats['cnt_{}'.format(key)] = len(stats[key])
+ stats['sum_{}'.format(key)] = sum(stats[key].values())
+ stats['min_{}'.format(key)] = min(stats[key].values())
+ stats['max_{}'.format(key)] = max(stats[key].values())
+ stats['avg_{}'.format(key)] = stats['sum_{}'.format(key)]/stats['cnt_{}'.format(key)]
+ stats['list_{}'.format(key)] = list(sorted(stats[key].keys()))
+ return stats
+
+def _make_toplist(stats, dict_key, top_threshold):
"""
Produce only toplist of given statistical keys.
:param dict stats: Calculated statistics.
:param str dict_key: Name of the dictionary key within statistics containing values.
- :param str dict_key: Name of the list key within statistics containing list of keys.
:param int top_threshold: Number of desired items in toplist.
:return: Updated statistics structure.
:rtype: dict
"""
+ list_key = 'list_{}'.format(dict_key)
+
+ top_threshold -= 1
+
+ rest = None
+ if ST_SKEY_REST in stats[dict_key]:
+ rest = stats[dict_key][ST_SKEY_REST]
+ del stats[dict_key][ST_SKEY_REST]
+
# Produce list of dictionary keys sorted in reverse order by their values.
sorted_key_list = sorted(sorted(stats[dict_key].keys()), key=lambda x: stats[dict_key][x], reverse=True)
sorted_key_list_keep = sorted_key_list[:top_threshold]
@@ -353,8 +402,14 @@ def _make_toplist(stats, dict_key, list_key, top_threshold):
if sorted_key_list_throw:
tmp[ST_SKEY_REST] = sum([stats[dict_key][key] for key in sorted_key_list_throw])
+ if rest:
+ tmp[ST_SKEY_REST] = tmp.get(ST_SKEY_REST, 0) + rest
+
# Put everything back into original statistics.
stats[dict_key] = tmp
- stats[list_key] = sorted(sorted_key_list_keep)
+ stats[list_key] = sorted_key_list_keep
+
+ if ST_SKEY_REST in stats[dict_key]:
+ stats[list_key].append(ST_SKEY_REST)
return stats
diff --git a/lib/mentat/stats/test_idea.py b/lib/mentat/stats/test_idea.py
index 9e730729baae349260f42ac484ebfb9a740938bc..44948893cc0ad18d1df0ccc023ab83647c42562e 100644
--- a/lib/mentat/stats/test_idea.py
+++ b/lib/mentat/stats/test_idea.py
@@ -15,12 +15,10 @@ import os
import sys
import time
import shutil
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../lib'))
-sys.path.insert(0, lib)
+import datetime
import mentat.stats.idea
+import mentat.datatype.sqldb
class TestMentatStatsIdea(unittest.TestCase):
@@ -186,7 +184,7 @@ class TestMentatStatsIdea(unittest.TestCase):
def test_01_counter_inc(self):
"""
- Perform the basic operativity tests.
+ Test counter incrementation utility.
"""
self.maxDiff = None
@@ -197,213 +195,125 @@ class TestMentatStatsIdea(unittest.TestCase):
self.assertEqual(mentat.stats.idea._counter_inc(test, 'x', 'a'), {'x': {'a': 3}})
self.assertEqual(mentat.stats.idea._counter_inc(test, 'x', 'a', 5), {'x': {'a': 8}})
- def test_02_evaluate_messages(self):
+ def test_02_make_toplist(self):
"""
- Perform the message evaluation tests.
+ Test toplist creation utility.
"""
self.maxDiff = None
- self.assertEqual(mentat.stats.idea.evaluate_messages(self.ideas_raw), {
- 'abuses': {'__unknown__': 2, 'abuse@cesnet.cz': 4},
- 'analyzers': {'Beekeeper': 1, 'Dionaea': 1, 'Kippo': 3, 'LaBrea': 1},
- 'asns': {'__unknown__': 6},
- 'avg_abuses': 3.0,
- 'avg_analyzers': 1.5,
- 'avg_asns': 6.0,
- 'avg_categories': 2.0,
- 'avg_category_sets': 2.0,
- 'avg_countries': 6.0,
- 'avg_detectors': 1.5,
- 'avg_detectorsws': 1.2,
- 'avg_ips': 1.5,
- 'categories': {'Exploit': 2, 'Fraud.Phishing': 3, 'Spam': 1},
- 'category_sets': {'Exploit': 2, 'Fraud.Phishing': 3, 'Spam': 1},
- 'cnt_abuses': 2,
- 'cnt_alerts': 6,
- 'cnt_analyzers': 4,
- 'cnt_asns': 1,
- 'cnt_categories': 3,
- 'cnt_category_sets': 3,
- 'cnt_countries': 1,
- 'cnt_detectors': 4,
- 'cnt_detectorsws': 5,
- 'cnt_ips': 8,
- 'countries': {'__unknown__': 6},
- 'detectors': {'cz.cesnet.holly': 1,
- 'org.example.dionaea': 2,
- 'org.example.kippo': 2,
- 'org.example.labrea': 1},
- 'detectorsws': {'cz.cesnet.holly/Beekeeper': 1,
- 'org.example.dionaea/Dionaea': 1,
- 'org.example.dionaea/Kippo': 1,
- 'org.example.kippo/Kippo': 2,
- 'org.example.labrea/LaBrea': 1},
- 'ips': {'192.168.0.0/25': 3,
- '192.168.0.100': 1,
- '192.168.0.105': 1,
- '192.168.0.109': 1,
- '192.168.0.2-192.168.0.5': 3,
- '192.168.0.200': 1,
- '192.172.0.109': 1,
- '192.172.0.200': 1},
- 'list_abuses': ['__unknown__', 'abuse@cesnet.cz'],
- 'list_analyzers': ['Beekeeper', 'Dionaea', 'Kippo', 'LaBrea'],
- 'list_asns': ['__unknown__'],
- 'list_categories': ['Exploit', 'Fraud.Phishing', 'Spam'],
- 'list_category_sets': ['Exploit', 'Fraud.Phishing', 'Spam'],
- 'list_countries': ['__unknown__'],
- 'list_detectors': ['cz.cesnet.holly',
- 'org.example.dionaea',
- 'org.example.kippo',
- 'org.example.labrea'],
- 'list_detectorsws': ['cz.cesnet.holly/Beekeeper',
- 'org.example.dionaea/Dionaea',
- 'org.example.dionaea/Kippo',
- 'org.example.kippo/Kippo',
- 'org.example.labrea/LaBrea'],
- 'list_ids': ['msg01', 'msg02', 'msg03', 'msg04', 'msg05', 'msg06'],
- 'list_ips': ['192.168.0.0/25',
- '192.168.0.100',
- '192.168.0.105',
- '192.168.0.109',
- '192.168.0.2-192.168.0.5',
- '192.168.0.200',
- '192.172.0.109',
- '192.172.0.200'],
- 'max_abuses': 4,
- 'max_analyzers': 3,
- 'max_asns': 6,
- 'max_categories': 3,
- 'max_category_sets': 3,
- 'max_countries': 6,
- 'max_detectors': 2,
- 'max_detectorsws': 2,
- 'max_ips': 3,
- 'min_abuses': 2,
- 'min_analyzers': 1,
- 'min_asns': 6,
- 'min_categories': 1,
- 'min_category_sets': 1,
- 'min_countries': 6,
- 'min_detectors': 1,
- 'min_detectorsws': 1,
- 'min_ips': 1,
- 'sum_abuses': 6,
- 'sum_analyzers': 6,
- 'sum_asns': 6,
- 'sum_categories': 6,
- 'sum_category_sets': 6,
- 'sum_countries': 6,
- 'sum_detectors': 6,
- 'sum_detectorsws': 6,
- 'sum_ips': 12
+ test1 = {
+ 'detectors': {
+ 'org.example.holly': 1,
+ 'org.example.rimmer': 5,
+ 'org.example.kryten': 10,
+ 'org.example.queeg': 20,
+ 'org.example.dionaea': 5,
+ 'org.example.kippo': 3,
+ 'org.example.labrea': 2
+ },
+ 'list_detectors': [
+ 'org.example.holly',
+ 'org.example.rimmer',
+ 'org.example.kryten',
+ 'org.example.queeg',
+ 'org.example.dionaea',
+ 'org.example.kippo',
+ 'org.example.labrea'
+ ]
+ }
+
+ self.assertEqual(mentat.stats.idea._make_toplist(test1, 'detectors', 5), {
+ 'detectors': {
+ '__REST__': 6,
+ 'org.example.dionaea': 5,
+ 'org.example.kryten': 10,
+ 'org.example.queeg': 20,
+ 'org.example.rimmer': 5
+ },
+ 'list_detectors': [
+ 'org.example.queeg',
+ 'org.example.kryten',
+ 'org.example.dionaea',
+ 'org.example.rimmer',
+ '__REST__'
+ ]
})
- def test_03_escape_stats(self):
+ test2 = {
+ 'detectors': {
+ '__REST__': 50,
+ 'org.example.rimmer': 5,
+ 'org.example.kryten': 10,
+ 'org.example.queeg': 20,
+ 'org.example.dionaea': 5,
+ 'org.example.kippo': 3,
+ 'org.example.labrea': 2
+ },
+ 'list_detectors': [
+ '__REST__',
+ 'org.example.rimmer',
+ 'org.example.kryten',
+ 'org.example.queeg',
+ 'org.example.dionaea',
+ 'org.example.kippo',
+ 'org.example.labrea'
+ ]
+ }
+
+ self.assertEqual(mentat.stats.idea._make_toplist(test2, 'detectors', 5), {
+ 'detectors': {
+ '__REST__': 55,
+ 'org.example.dionaea': 5,
+ 'org.example.kryten': 10,
+ 'org.example.queeg': 20,
+ 'org.example.rimmer': 5
+ },
+ 'list_detectors': [
+ 'org.example.queeg',
+ 'org.example.kryten',
+ 'org.example.dionaea',
+ 'org.example.rimmer',
+ '__REST__'
+ ]
+ })
+
+ test3 = {
+ 'detectors': {
+ '__REST__': 50,
+ 'org.example.rimmer': 5,
+ 'org.example.kryten': 10,
+ 'org.example.queeg': 20,
+ },
+ 'list_detectors': [
+ '__REST__',
+ 'org.example.rimmer',
+ 'org.example.kryten',
+ 'org.example.queeg'
+ ]
+ }
+
+ self.assertEqual(mentat.stats.idea._make_toplist(test3, 'detectors', 5), {
+ 'detectors': {
+ '__REST__': 50,
+ 'org.example.kryten': 10,
+ 'org.example.queeg': 20,
+ 'org.example.rimmer': 5
+ },
+ 'list_detectors': [
+ 'org.example.queeg',
+ 'org.example.kryten',
+ 'org.example.rimmer',
+ '__REST__'
+ ]
+ })
+
+ def test_03_evaluate_events(self):
"""
- Perform the basic operativity tests.
+ Perform the message evaluation tests.
"""
self.maxDiff = None
- result = mentat.stats.idea.escape_stats(mentat.stats.idea.evaluate_messages(self.ideas_raw))
- self.assertEqual(result, {
- 'abuses': {'__unknown__': 2, 'abuse@cesnet(dot)cz': 4},
- 'analyzers': {'Beekeeper': 1, 'Dionaea': 1, 'Kippo': 3, 'LaBrea': 1},
- 'asns': {'__unknown__': 6},
- 'avg_abuses': 3.0,
- 'avg_analyzers': 1.5,
- 'avg_asns': 6.0,
- 'avg_categories': 2.0,
- 'avg_category_sets': 2.0,
- 'avg_countries': 6.0,
- 'avg_detectors': 1.5,
- 'avg_detectorsws': 1.2,
- 'avg_ips': 1.5,
- 'categories': {'Exploit': 2, 'Fraud(dot)Phishing': 3, 'Spam': 1},
- 'category_sets': {'Exploit': 2, 'Fraud(dot)Phishing': 3, 'Spam': 1},
- 'cnt_abuses': 2,
- 'cnt_alerts': 6,
- 'cnt_analyzers': 4,
- 'cnt_asns': 1,
- 'cnt_categories': 3,
- 'cnt_category_sets': 3,
- 'cnt_countries': 1,
- 'cnt_detectors': 4,
- 'cnt_detectorsws': 5,
- 'cnt_ips': 8,
- 'countries': {'__unknown__': 6},
- 'detectors': {'cz(dot)cesnet(dot)holly': 1,
- 'org(dot)example(dot)dionaea': 2,
- 'org(dot)example(dot)kippo': 2,
- 'org(dot)example(dot)labrea': 1},
- 'detectorsws': {'cz(dot)cesnet(dot)holly/Beekeeper': 1,
- 'org(dot)example(dot)dionaea/Dionaea': 1,
- 'org(dot)example(dot)dionaea/Kippo': 1,
- 'org(dot)example(dot)kippo/Kippo': 2,
- 'org(dot)example(dot)labrea/LaBrea': 1},
- 'ips': {'192(dot)168(dot)0(dot)0/25': 3,
- '192(dot)168(dot)0(dot)100': 1,
- '192(dot)168(dot)0(dot)105': 1,
- '192(dot)168(dot)0(dot)109': 1,
- '192(dot)168(dot)0(dot)2-192(dot)168(dot)0(dot)5': 3,
- '192(dot)168(dot)0(dot)200': 1,
- '192(dot)172(dot)0(dot)109': 1,
- '192(dot)172(dot)0(dot)200': 1},
- 'list_abuses': ['__unknown__', 'abuse@cesnet.cz'],
- 'list_analyzers': ['Beekeeper', 'Dionaea', 'Kippo', 'LaBrea'],
- 'list_asns': ['__unknown__'],
- 'list_categories': ['Exploit', 'Fraud.Phishing', 'Spam'],
- 'list_category_sets': ['Exploit', 'Fraud.Phishing', 'Spam'],
- 'list_countries': ['__unknown__'],
- 'list_detectors': ['cz.cesnet.holly',
- 'org.example.dionaea',
- 'org.example.kippo',
- 'org.example.labrea'],
- 'list_detectorsws': ['cz.cesnet.holly/Beekeeper',
- 'org.example.dionaea/Dionaea',
- 'org.example.dionaea/Kippo',
- 'org.example.kippo/Kippo',
- 'org.example.labrea/LaBrea'],
- 'list_ids': ['msg01', 'msg02', 'msg03', 'msg04', 'msg05', 'msg06'],
- 'list_ips': ['192.168.0.0/25',
- '192.168.0.100',
- '192.168.0.105',
- '192.168.0.109',
- '192.168.0.2-192.168.0.5',
- '192.168.0.200',
- '192.172.0.109',
- '192.172.0.200'],
- 'max_abuses': 4,
- 'max_analyzers': 3,
- 'max_asns': 6,
- 'max_categories': 3,
- 'max_category_sets': 3,
- 'max_countries': 6,
- 'max_detectors': 2,
- 'max_detectorsws': 2,
- 'max_ips': 3,
- 'min_abuses': 2,
- 'min_analyzers': 1,
- 'min_asns': 6,
- 'min_categories': 1,
- 'min_category_sets': 1,
- 'min_countries': 6,
- 'min_detectors': 1,
- 'min_detectorsws': 1,
- 'min_ips': 1,
- 'sum_abuses': 6,
- 'sum_analyzers': 6,
- 'sum_asns': 6,
- 'sum_categories': 6,
- 'sum_category_sets': 6,
- 'sum_countries': 6,
- 'sum_detectors': 6,
- 'sum_detectorsws': 6,
- 'sum_ips': 12
- })
-
- result = mentat.stats.idea.unescape_stats(result)
- self.assertEqual(result, {
+ self.assertEqual(mentat.stats.idea.evaluate_events(self.ideas_raw), {
'abuses': {'__unknown__': 2, 'abuse@cesnet.cz': 4},
'analyzers': {'Beekeeper': 1, 'Dionaea': 1, 'Kippo': 3, 'LaBrea': 1},
'asns': {'__unknown__': 6},
@@ -499,15 +409,14 @@ class TestMentatStatsIdea(unittest.TestCase):
'sum_ips': 12
})
- def test_04_brief_stats(self):
+ def test_04_truncate_stats(self):
"""
Perform the basic operativity tests.
"""
self.maxDiff = None
- self.assertEqual(mentat.stats.idea.brief_stats(mentat.stats.idea.evaluate_messages(self.ideas_raw), 3), {
- 'abuses': {'__unknown__': 2, 'abuse@cesnet.cz': 4},
- 'analyzers': {'Beekeeper': 1, 'Dionaea': 1, 'Kippo': 3, 'LaBrea': 1},
+ self.assertEqual(mentat.stats.idea.truncate_stats(mentat.stats.idea.evaluate_events(self.ideas_raw), 3), {'abuses': {'__unknown__': 2, 'abuse@cesnet.cz': 4},
+ 'analyzers': {'Beekeeper': 1, 'Kippo': 3, '__REST__': 2},
'asns': {'__unknown__': 6},
'avg_abuses': 3.0,
'avg_analyzers': 1.5,
@@ -518,8 +427,8 @@ class TestMentatStatsIdea(unittest.TestCase):
'avg_detectors': 1.5,
'avg_detectorsws': 1.2,
'avg_ips': 1.5,
- 'categories': {'Exploit': 2, 'Fraud.Phishing': 3, 'Spam': 1},
- 'category_sets': {'Exploit': 2, 'Fraud.Phishing': 3, 'Spam': 1},
+ 'categories': {'Exploit': 2, 'Fraud.Phishing': 3, '__REST__': 1},
+ 'category_sets': {'Exploit': 2, 'Fraud.Phishing': 3, '__REST__': 1},
'cnt_abuses': 2,
'cnt_alerts': 6,
'cnt_analyzers': 4,
@@ -531,32 +440,22 @@ class TestMentatStatsIdea(unittest.TestCase):
'cnt_detectorsws': 5,
'cnt_ips': 8,
'countries': {'__unknown__': 6},
- 'detectors': {'cz.cesnet.holly': 1,
- 'org.example.dionaea': 2,
- 'org.example.kippo': 2,
- 'org.example.labrea': 1},
- 'detectorsws': {'cz.cesnet.holly/Beekeeper': 1,
- 'org.example.dionaea/Dionaea': 1,
- 'org.example.dionaea/Kippo': 1,
- 'org.example.kippo/Kippo': 2,
- 'org.example.labrea/LaBrea': 1},
- 'ips': {'192.168.0.0/25': 3, '192.168.0.100': 1, '192.168.0.2-192.168.0.5': 3, '__REST__': 5},
- 'list_abuses': ['__unknown__', 'abuse@cesnet.cz'],
- 'list_analyzers': ['Beekeeper', 'Dionaea', 'Kippo', 'LaBrea'],
+ 'detectors': {'__REST__': 2, 'org.example.dionaea': 2, 'org.example.kippo': 2},
+ 'detectorsws': {'__REST__': 3,
+ 'cz.cesnet.holly/Beekeeper': 1,
+ 'org.example.kippo/Kippo': 2},
+ 'ips': {'192.168.0.0/25': 3, '192.168.0.2-192.168.0.5': 3, '__REST__': 6},
+ 'list_abuses': ['abuse@cesnet.cz', '__unknown__'],
+ 'list_analyzers': ['Kippo', 'Beekeeper', '__REST__'],
'list_asns': ['__unknown__'],
- 'list_categories': ['Exploit', 'Fraud.Phishing', 'Spam'],
- 'list_category_sets': ['Exploit', 'Fraud.Phishing', 'Spam'],
+ 'list_categories': ['Fraud.Phishing', 'Exploit', '__REST__'],
+ 'list_category_sets': ['Fraud.Phishing', 'Exploit', '__REST__'],
'list_countries': ['__unknown__'],
- 'list_detectors': ['cz.cesnet.holly',
- 'org.example.dionaea',
- 'org.example.kippo',
- 'org.example.labrea'],
- 'list_detectorsws': ['cz.cesnet.holly/Beekeeper',
- 'org.example.dionaea/Dionaea',
- 'org.example.dionaea/Kippo',
- 'org.example.kippo/Kippo',
- 'org.example.labrea/LaBrea'],
- 'list_ips': ['192.168.0.0/25', '192.168.0.100', '192.168.0.2-192.168.0.5'],
+ 'list_detectors': ['org.example.dionaea', 'org.example.kippo', '__REST__'],
+ 'list_detectorsws': ['org.example.kippo/Kippo',
+ 'cz.cesnet.holly/Beekeeper',
+ '__REST__'],
+ 'list_ips': ['192.168.0.0/25', '192.168.0.2-192.168.0.5', '__REST__'],
'max_abuses': 4,
'max_analyzers': 3,
'max_asns': 6,
@@ -586,14 +485,14 @@ class TestMentatStatsIdea(unittest.TestCase):
'sum_ips': 12
})
- def test_05_group_messages(self):
+ def test_05_group_events(self):
"""
Perform the basic operativity tests.
"""
self.maxDiff = None
- self.assertEqual(mentat.stats.idea.group_messages(self.ideas_raw), {
- 'external': [{'Category': ['Spam'],
+ self.assertEqual(mentat.stats.idea.group_events(self.ideas_raw), {
+ 'stats_external': [{'Category': ['Spam'],
'CreateTime': '2012-11-03T15:00:02Z',
'DetectTime': '2012-11-03T15:00:07Z',
'Format': 'IDEA0',
@@ -614,7 +513,7 @@ class TestMentatStatsIdea(unittest.TestCase):
{'Name': 'cz.cesnet.holly', 'SW': ['Beekeeper']}],
'Source': [{'IP4': ['192.172.0.109', '192.172.0.200'],
'Type': ['Exploit']}]}],
- 'internal': [{'Category': ['Fraud.Phishing'],
+ 'stats_internal': [{'Category': ['Fraud.Phishing'],
'CreateTime': '2012-11-03T10:00:02Z',
'DetectTime': '2012-11-03T10:00:07Z',
'Format': 'IDEA0',
@@ -661,7 +560,7 @@ class TestMentatStatsIdea(unittest.TestCase):
'Source': [{'IP4': ['192.168.0.109', '192.168.0.200'],
'Type': ['Exploit']}],
'_CESNET': {'ResolvedAbuses': ['abuse@cesnet.cz']}}],
- 'overall': [{'Category': ['Fraud.Phishing'],
+ 'stats_overall': [{'Category': ['Fraud.Phishing'],
'CreateTime': '2012-11-03T10:00:02Z',
'DetectTime': '2012-11-03T10:00:07Z',
'Format': 'IDEA0',
@@ -731,27 +630,187 @@ class TestMentatStatsIdea(unittest.TestCase):
'Type': ['Exploit']}]}]
})
- def test_06_evaluate_messages_full(self):
+ def test_06_evaluate_event_groups(self):
"""
Perform the basic operativity tests.
"""
- result = mentat.stats.idea.evaluate_messages_full(self.ideas_raw)
+ result = mentat.stats.idea.evaluate_event_groups(self.ideas_raw)
if self.verbose:
- print('*** result = mentat.stats.idea.evaluate_messages_full(self.ideas_raw) ***')
+ print('*** result = mentat.stats.idea.evaluate_event_groups(self.ideas_raw) ***')
pprint(result)
self.assertTrue(result)
- result = mentat.stats.idea.escape_stats_full(mentat.stats.idea.brief_stats_full(result, 3))
+ result = mentat.stats.idea.truncate_evaluations(result, 3)
if self.verbose:
- print('*** result = mentat.stats.idea.escape_stats_full(mentat.stats.idea.brief_stats_full(result, 3)) ***')
+ print('*** result = mentat.stats.idea.truncate_evaluations(result, 3) ***')
pprint(result)
self.assertTrue(result)
- result = mentat.stats.idea.unescape_stats_full(result)
- if self.verbose:
- print('*** result = mentat.stats.idea.unescape_stats_full(result) ***')
- pprint(result)
+ def test_07_aggregate_stats(self):
+ """
+ Perform the statistics aggregation tests.
+ """
+ self.maxDiff = None
+
+ sts1 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+ sts2 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+ sts3 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+
+ result = mentat.stats.idea.aggregate_stats(sts1, 'interval1', 123456)
+ result = mentat.stats.idea.aggregate_stats(sts2, 'interval2', 123456, result)
+ result = mentat.stats.idea.aggregate_stats(sts3, 'interval3', 123456, result)
+
+ self.assertEqual(result, {'abuses': {'__unknown__': 6, 'abuse@cesnet.cz': 12},
+ 'analyzers': {'Beekeeper': 3, 'Dionaea': 3, 'Kippo': 9, 'LaBrea': 3},
+ 'asns': {'__unknown__': 18},
+ 'avg_abuses': 9.0,
+ 'avg_analyzers': 4.5,
+ 'avg_asns': 18.0,
+ 'avg_categories': 6.0,
+ 'avg_category_sets': 6.0,
+ 'avg_countries': 18.0,
+ 'avg_detectors': 4.5,
+ 'avg_detectorsws': 3.6,
+ 'avg_ips': 4.5,
+ 'categories': {'Exploit': 6, 'Fraud.Phishing': 9, 'Spam': 3},
+ 'category_sets': {'Exploit': 6, 'Fraud.Phishing': 9, 'Spam': 3},
+ 'cnt_abuses': 2,
+ 'cnt_alerts': 18,
+ 'cnt_analyzers': 4,
+ 'cnt_asns': 1,
+ 'cnt_categories': 3,
+ 'cnt_category_sets': 3,
+ 'cnt_countries': 1,
+ 'cnt_detectors': 4,
+ 'cnt_detectorsws': 5,
+ 'cnt_ips': 8,
+ 'countries': {'__unknown__': 18},
+ 'detectors': {'cz.cesnet.holly': 3,
+ 'org.example.dionaea': 6,
+ 'org.example.kippo': 6,
+ 'org.example.labrea': 3},
+ 'detectorsws': {'cz.cesnet.holly/Beekeeper': 3,
+ 'org.example.dionaea/Dionaea': 3,
+ 'org.example.dionaea/Kippo': 3,
+ 'org.example.kippo/Kippo': 6,
+ 'org.example.labrea/LaBrea': 3},
+ 'histogram': [('interval1', 123456, 6),
+ ('interval2', 123456, 6),
+ ('interval3', 123456, 6)],
+ 'ips': {'192.168.0.0/25': 9,
+ '192.168.0.100': 3,
+ '192.168.0.105': 3,
+ '192.168.0.109': 3,
+ '192.168.0.2-192.168.0.5': 9,
+ '192.168.0.200': 3,
+ '192.172.0.109': 3,
+ '192.172.0.200': 3},
+ 'list_abuses': ['__unknown__', 'abuse@cesnet.cz'],
+ 'list_analyzers': ['Beekeeper', 'Dionaea', 'Kippo', 'LaBrea'],
+ 'list_asns': ['__unknown__'],
+ 'list_categories': ['Exploit', 'Fraud.Phishing', 'Spam'],
+ 'list_category_sets': ['Exploit', 'Fraud.Phishing', 'Spam'],
+ 'list_countries': ['__unknown__'],
+ 'list_detectors': ['cz.cesnet.holly',
+ 'org.example.dionaea',
+ 'org.example.kippo',
+ 'org.example.labrea'],
+ 'list_detectorsws': ['cz.cesnet.holly/Beekeeper',
+ 'org.example.dionaea/Dionaea',
+ 'org.example.dionaea/Kippo',
+ 'org.example.kippo/Kippo',
+ 'org.example.labrea/LaBrea'],
+ 'list_ips': ['192.168.0.0/25',
+ '192.168.0.100',
+ '192.168.0.105',
+ '192.168.0.109',
+ '192.168.0.2-192.168.0.5',
+ '192.168.0.200',
+ '192.172.0.109',
+ '192.172.0.200'],
+ 'max_abuses': 12,
+ 'max_analyzers': 9,
+ 'max_asns': 18,
+ 'max_categories': 9,
+ 'max_category_sets': 9,
+ 'max_countries': 18,
+ 'max_detectors': 6,
+ 'max_detectorsws': 6,
+ 'max_ips': 9,
+ 'min_abuses': 6,
+ 'min_analyzers': 3,
+ 'min_asns': 18,
+ 'min_categories': 3,
+ 'min_category_sets': 3,
+ 'min_countries': 18,
+ 'min_detectors': 3,
+ 'min_detectorsws': 3,
+ 'min_ips': 3,
+ 'sum_abuses': 18,
+ 'sum_analyzers': 18,
+ 'sum_asns': 18,
+ 'sum_categories': 18,
+ 'sum_category_sets': 18,
+ 'sum_countries': 18,
+ 'sum_detectors': 18,
+ 'sum_detectorsws': 18,
+ 'sum_ips': 36
+ })
+
+ def test_08_aggregate_stat_groups(self):
+ """
+ Perform the statistic group aggregation tests.
+ """
+ self.maxDiff = None
+
+ timestamp = 1485993600
+
+ stse1 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+ stse2 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+ stse3 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+
+ stso1 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+ stso2 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+ stso3 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+
+ stsi1 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+ stsi2 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+ stsi3 = mentat.stats.idea.evaluate_events(self.ideas_raw)
+
+ sts1 = mentat.datatype.sqldb.EventStatisticsModel(
+ interval = 'interval1',
+ dt_from = datetime.datetime.fromtimestamp(timestamp),
+ dt_to = datetime.datetime.fromtimestamp(timestamp+300),
+ count = stso1[mentat.stats.idea.ST_SKEY_CNT_ALERTS],
+ stats_overall = stso1,
+ stats_internal = stsi1,
+ stats_external = stse1
+ )
+ sts2 = mentat.datatype.sqldb.EventStatisticsModel(
+ interval = 'interval2',
+ dt_from = datetime.datetime.fromtimestamp(timestamp+300),
+ dt_to = datetime.datetime.fromtimestamp(timestamp+600),
+ count = stso2[mentat.stats.idea.ST_SKEY_CNT_ALERTS],
+ stats_overall = stso2,
+ stats_internal = stsi2,
+ stats_external = stse2
+ )
+ sts3 = mentat.datatype.sqldb.EventStatisticsModel(
+ interval = 'interval3',
+ dt_from = datetime.datetime.fromtimestamp(timestamp+600),
+ dt_to = datetime.datetime.fromtimestamp(timestamp+900),
+ count = stso3[mentat.stats.idea.ST_SKEY_CNT_ALERTS],
+ stats_overall = stso3,
+ stats_internal = stsi3,
+ stats_external = stse3
+ )
+
+ result = mentat.stats.idea.aggregate_stat_groups([sts1, sts2, sts3])
+
self.assertTrue(result)
+ self.assertEqual(result['dt_from'], datetime.datetime.fromtimestamp(timestamp))
+ self.assertEqual(result['dt_to'], datetime.datetime.fromtimestamp(timestamp+900))
+
if __name__ == "__main__":
unittest.main()
diff --git a/package.json b/package.json
index c6ac83c3add2412cd3f04f36af5a5710f5fda3e7..2a5c730d98a8f449d7a74467d35ef8b6c45070e9 100644
--- a/package.json
+++ b/package.json
@@ -26,12 +26,14 @@
"bootstrap-datepicker": "^1.7.1",
"bootstrap-select": "^1.12.4",
"bootswatch": "^3.3.7",
- "d3": "^4.10.2",
+ "d3": "3.5.17",
"datatables.net": "^1.10.16",
"datatables.net-plugins": "^1.10.15",
"eonasdan-bootstrap-datetimepicker": "^4.17.47",
"font-awesome": "^4.7.0",
"jquery": "^3.2.1",
+ "moment": "^2.19.4",
+ "moment-timezone": "^0.5.14",
"nvd3": "^1.8.6"
}
}
diff --git a/yarn.lock b/yarn.lock
index 83491598c479ef9696742d84f1285eff51e7cdef..d26c07fff3793067c925e8e1a5db51d7d3b3c39b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -322,10 +322,6 @@ combined-stream@~0.0.4:
dependencies:
delayed-stream "0.0.5"
-commander@2:
- version "2.11.0"
- resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563"
-
concat-map@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
@@ -365,215 +361,9 @@ cycle@1.0.x:
version "1.0.3"
resolved "https://registry.yarnpkg.com/cycle/-/cycle-1.0.3.tgz#21e80b2be8580f98b468f379430662b046c34ad2"
-d3-array@1, d3-array@1.2.0, d3-array@^1.2.0:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.0.tgz#147d269720e174c4057a7f42be8b0f3f2ba53108"
-
-d3-axis@1.0.8:
- version "1.0.8"
- resolved "https://registry.yarnpkg.com/d3-axis/-/d3-axis-1.0.8.tgz#31a705a0b535e65759de14173a31933137f18efa"
-
-d3-brush@1.0.4:
- version "1.0.4"
- resolved "https://registry.yarnpkg.com/d3-brush/-/d3-brush-1.0.4.tgz#00c2f238019f24f6c0a194a26d41a1530ffe7bc4"
- dependencies:
- d3-dispatch "1"
- d3-drag "1"
- d3-interpolate "1"
- d3-selection "1"
- d3-transition "1"
-
-d3-chord@1.0.4:
- version "1.0.4"
- resolved "https://registry.yarnpkg.com/d3-chord/-/d3-chord-1.0.4.tgz#7dec4f0ba886f713fe111c45f763414f6f74ca2c"
- dependencies:
- d3-array "1"
- d3-path "1"
-
-d3-collection@1, d3-collection@1.0.4:
- version "1.0.4"
- resolved "https://registry.yarnpkg.com/d3-collection/-/d3-collection-1.0.4.tgz#342dfd12837c90974f33f1cc0a785aea570dcdc2"
-
-d3-color@1, d3-color@1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-1.0.3.tgz#bc7643fca8e53a8347e2fbdaffa236796b58509b"
-
-d3-dispatch@1, d3-dispatch@1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-1.0.3.tgz#46e1491eaa9b58c358fce5be4e8bed626e7871f8"
-
-d3-drag@1, d3-drag@1.1.1:
- version "1.1.1"
- resolved "https://registry.yarnpkg.com/d3-drag/-/d3-drag-1.1.1.tgz#b5155304433b18ba38726b2184d0098e820dc64b"
- dependencies:
- d3-dispatch "1"
- d3-selection "1"
-
-d3-dsv@1, d3-dsv@1.0.7:
- version "1.0.7"
- resolved "https://registry.yarnpkg.com/d3-dsv/-/d3-dsv-1.0.7.tgz#137076663f398428fc3d031ae65370522492b78f"
- dependencies:
- commander "2"
- iconv-lite "0.4"
- rw "1"
-
-d3-ease@1, d3-ease@1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/d3-ease/-/d3-ease-1.0.3.tgz#68bfbc349338a380c44d8acc4fbc3304aa2d8c0e"
-
-d3-force@1.0.6:
- version "1.0.6"
- resolved "https://registry.yarnpkg.com/d3-force/-/d3-force-1.0.6.tgz#ea7e1b7730e2664cd314f594d6718c57cc132b79"
- dependencies:
- d3-collection "1"
- d3-dispatch "1"
- d3-quadtree "1"
- d3-timer "1"
-
-d3-format@1, d3-format@1.2.0:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-1.2.0.tgz#6b480baa886885d4651dc248a8f4ac9da16db07a"
-
-d3-geo@1.6.4:
- version "1.6.4"
- resolved "https://registry.yarnpkg.com/d3-geo/-/d3-geo-1.6.4.tgz#f20e1e461cb1845f5a8be55ab6f876542a7e3199"
- dependencies:
- d3-array "1"
-
-d3-hierarchy@1.1.5:
- version "1.1.5"
- resolved "https://registry.yarnpkg.com/d3-hierarchy/-/d3-hierarchy-1.1.5.tgz#a1c845c42f84a206bcf1c01c01098ea4ddaa7a26"
-
-d3-interpolate@1, d3-interpolate@1.1.5:
- version "1.1.5"
- resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-1.1.5.tgz#69e099ff39214716e563c9aec3ea9d1ea4b8a79f"
- dependencies:
- d3-color "1"
-
-d3-path@1, d3-path@1.0.5:
- version "1.0.5"
- resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-1.0.5.tgz#241eb1849bd9e9e8021c0d0a799f8a0e8e441764"
-
-d3-polygon@1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/d3-polygon/-/d3-polygon-1.0.3.tgz#16888e9026460933f2b179652ad378224d382c62"
-
-d3-quadtree@1, d3-quadtree@1.0.3:
- version "1.0.3"
- resolved "https://registry.yarnpkg.com/d3-quadtree/-/d3-quadtree-1.0.3.tgz#ac7987e3e23fe805a990f28e1b50d38fcb822438"
-
-d3-queue@3.0.7:
- version "3.0.7"
- resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618"
-
-d3-random@1.1.0:
- version "1.1.0"
- resolved "https://registry.yarnpkg.com/d3-random/-/d3-random-1.1.0.tgz#6642e506c6fa3a648595d2b2469788a8d12529d3"
-
-d3-request@1.0.6:
- version "1.0.6"
- resolved "https://registry.yarnpkg.com/d3-request/-/d3-request-1.0.6.tgz#a1044a9ef4ec28c824171c9379fae6d79474b19f"
- dependencies:
- d3-collection "1"
- d3-dispatch "1"
- d3-dsv "1"
- xmlhttprequest "1"
-
-d3-scale@1.0.6:
- version "1.0.6"
- resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-1.0.6.tgz#bce19da80d3a0cf422c9543ae3322086220b34ed"
- dependencies:
- d3-array "^1.2.0"
- d3-collection "1"
- d3-color "1"
- d3-format "1"
- d3-interpolate "1"
- d3-time "1"
- d3-time-format "2"
-
-d3-selection@1, d3-selection@1.1.0, d3-selection@^1.1.0:
- version "1.1.0"
- resolved "https://registry.yarnpkg.com/d3-selection/-/d3-selection-1.1.0.tgz#1998684896488f839ca0372123da34f1d318809c"
-
-d3-shape@1.2.0:
- version "1.2.0"
- resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-1.2.0.tgz#45d01538f064bafd05ea3d6d2cb748fd8c41f777"
- dependencies:
- d3-path "1"
-
-d3-time-format@2, d3-time-format@2.0.5:
- version "2.0.5"
- resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-2.0.5.tgz#9d7780204f7c9119c9170b1a56db4de9a8af972e"
- dependencies:
- d3-time "1"
-
-d3-time@1, d3-time@1.0.7:
- version "1.0.7"
- resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-1.0.7.tgz#94caf6edbb7879bb809d0d1f7572bc48482f7270"
-
-d3-timer@1, d3-timer@1.0.7:
- version "1.0.7"
- resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-1.0.7.tgz#df9650ca587f6c96607ff4e60cc38229e8dd8531"
-
-d3-transition@1, d3-transition@1.1.0:
- version "1.1.0"
- resolved "https://registry.yarnpkg.com/d3-transition/-/d3-transition-1.1.0.tgz#cfc85c74e5239324290546623572990560c3966f"
- dependencies:
- d3-color "1"
- d3-dispatch "1"
- d3-ease "1"
- d3-interpolate "1"
- d3-selection "^1.1.0"
- d3-timer "1"
-
-d3-voronoi@1.1.2:
- version "1.1.2"
- resolved "https://registry.yarnpkg.com/d3-voronoi/-/d3-voronoi-1.1.2.tgz#1687667e8f13a2d158c80c1480c5a29cb0d8973c"
-
-d3-zoom@1.5.0:
- version "1.5.0"
- resolved "https://registry.yarnpkg.com/d3-zoom/-/d3-zoom-1.5.0.tgz#8417de9a077f98f9ce83b1998efb8ee12b4db26e"
- dependencies:
- d3-dispatch "1"
- d3-drag "1"
- d3-interpolate "1"
- d3-selection "1"
- d3-transition "1"
-
-d3@^4.10.2:
- version "4.10.2"
- resolved "https://registry.yarnpkg.com/d3/-/d3-4.10.2.tgz#d401b2bc0372a77e6822f278c0e4b4090206babd"
- dependencies:
- d3-array "1.2.0"
- d3-axis "1.0.8"
- d3-brush "1.0.4"
- d3-chord "1.0.4"
- d3-collection "1.0.4"
- d3-color "1.0.3"
- d3-dispatch "1.0.3"
- d3-drag "1.1.1"
- d3-dsv "1.0.7"
- d3-ease "1.0.3"
- d3-force "1.0.6"
- d3-format "1.2.0"
- d3-geo "1.6.4"
- d3-hierarchy "1.1.5"
- d3-interpolate "1.1.5"
- d3-path "1.0.5"
- d3-polygon "1.0.3"
- d3-quadtree "1.0.3"
- d3-queue "3.0.7"
- d3-random "1.1.0"
- d3-request "1.0.6"
- d3-scale "1.0.6"
- d3-selection "1.1.0"
- d3-shape "1.2.0"
- d3-time "1.0.7"
- d3-time-format "2.0.5"
- d3-timer "1.0.7"
- d3-transition "1.1.0"
- d3-voronoi "1.1.2"
- d3-zoom "1.5.0"
+d3@3.5.17:
+ version "3.5.17"
+ resolved "https://registry.yarnpkg.com/d3/-/d3-3.5.17.tgz#bc46748004378b21a360c9fc7cf5231790762fb8"
dashdash@^1.12.0:
version "1.14.1"
@@ -1117,10 +907,6 @@ i@0.3.x:
version "0.3.5"
resolved "https://registry.yarnpkg.com/i/-/i-0.3.5.tgz#1d2b854158ec8169113c6cb7f6b6801e99e211d5"
-iconv-lite@0.4:
- version "0.4.18"
- resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.18.tgz#23d8656b16aae6742ac29732ea8f0336a4789cf2"
-
iconv-lite@~0.2.11:
version "0.2.11"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.2.11.tgz#1ce60a3a57864a292d1321ff4609ca4bb965adc8"
@@ -1382,10 +1168,20 @@ moment-timezone@^0.4.0:
dependencies:
moment ">= 2.6.0"
+moment-timezone@^0.5.14:
+ version "0.5.14"
+ resolved "https://registry.yarnpkg.com/moment-timezone/-/moment-timezone-0.5.14.tgz#4eb38ff9538b80108ba467a458f3ed4268ccfcb1"
+ dependencies:
+ moment ">= 2.9.0"
+
"moment@>= 2.6.0", moment@^2.10:
version "2.18.1"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f"
+"moment@>= 2.9.0", moment@^2.19.4:
+ version "2.19.4"
+ resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.4.tgz#17e5e2c6ead8819c8ecfad83a0acccb312e94682"
+
ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -1904,10 +1700,6 @@ run-series@^1.0.2:
version "1.1.4"
resolved "https://registry.yarnpkg.com/run-series/-/run-series-1.1.4.tgz#89a73ddc5e75c9ef8ab6320c0a1600d6a41179b9"
-rw@1:
- version "1.3.3"
- resolved "https://registry.yarnpkg.com/rw/-/rw-1.3.3.tgz#3f862dfa91ab766b14885ef4d01124bfda074fb4"
-
safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
@@ -2225,10 +2017,6 @@ wrappy@1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
-xmlhttprequest@1:
- version "1.8.0"
- resolved "https://registry.yarnpkg.com/xmlhttprequest/-/xmlhttprequest-1.8.0.tgz#67fe075c5c24fef39f9d65f5f7b7fe75171968fc"
-
xtend@~2.1.1:
version "2.1.2"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-2.1.2.tgz#6efecc2a4dad8e6962c4901b337ce7ba87b5d28b"