Implemented prototype of reporting statistics dashboard.

Missing feature that needs to be implemented is filtering based on abuse groups. Currently any user can view dashboards for any abuse group reports. Aside from administrator user should be able to view dashboards only for groups he/she is member of. (Redmine issue: #3734)

lib/hawat/base.py

@@ -604,6 +604,7 @@ class HawatSearchView(HawatDbmodelView):
                 items = self.search(self.dbquery, self.dbmodel, form_data, context)
 
                 context.update(
+                    searched = True,
                     items = items,
                     items_count = len(items),
                     form_data = form_data

lib/hawat/blueprints/reports/__init__.py

@@ -30,7 +30,7 @@ import mentat.stats.idea
 from mentat.datatype.sqldb import EventReportModel
 
 import hawat.base
-from hawat.blueprints.reports.forms import EventReportSearchForm
+from hawat.blueprints.reports.forms import EventReportSearchForm, ReportingDashboardForm
 
 
 #
@@ -254,6 +254,70 @@ class DataView(hawat.base.HawatFileIdView):
             raise ValueError("Requested invalid data file type '{}'".format(filetype))
         return 'security_report_{}.{}'.format(fileid, fileext)
 
+
+class DashboardView(hawat.base.HawatSearchView):
+    """
+    View responsible for presenting reporting dashboard.
+    """
+    authentication = True
+
+    @classmethod
+    def get_view_name(cls):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_view_name`.
+        """
+        return 'dashboard'
+
+    @classmethod
+    def get_menu_icon(cls):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_menu_icon`.
+        """
+        return 'module-{}'.format(BLUEPRINT_NAME)
+
+    @classmethod
+    def get_menu_title(cls):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_menu_title`.
+        """
+        return lazy_gettext('Reporting')
+
+    @classmethod
+    def get_view_title(cls):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatRenderableView.get_view_title`.
+        """
+        return lazy_gettext('Event reporting dashboards')
+
+    @property
+    def dbmodel(self):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatDbmodelView.dbmodel`.
+        """
+        return EventReportModel
+
+    @staticmethod
+    def get_search_form(args):
+        return ReportingDashboardForm(args, meta = {'csrf': False})
+
+    def search(self, query, model, args, context):
+        if 'dt_from' in args and args['dt_from']:
+            query = query.filter(model.dt_from >= args['dt_from'])
+        if 'dt_to' in args and args['dt_to']:
+            query = query.filter(model.dt_to <= args['dt_to'])
+        return query.order_by(model.label).all()
+
+    def do_before_render(self, item, context):
+        """
+        *Hook method*. Will be called before rendering the template.
+        """
+        context.update(
+            statistics = mentat.stats.idea.truncate_evaluations(
+                mentat.stats.idea.aggregate_stats_reports(item)
+            )
+        )
+
+
 #-------------------------------------------------------------------------------
 
 
@@ -285,6 +349,12 @@ class ReportsBlueprint(hawat.base.HawatBlueprint):
             position = 120,
             view = SearchView
         )
+        app.menu_main.add_entry(
+            'view',
+            'dashboards.reporting',
+            position = 20,
+            view = DashboardView
+        )
 
 
 #-------------------------------------------------------------------------------
@@ -307,5 +377,6 @@ def get_blueprint():
     hbp.register_view_class(ShowView, '/<int:item_id>/show')
     hbp.register_view_class(UnauthShowView, '/<item_id>/unauth')
     hbp.register_view_class(DataView, '/data/<fileid>/<filetype>')
+    hbp.register_view_class(DashboardView, '/dashboard')
 
     return hbp

lib/hawat/blueprints/reports/forms.py

@@ -20,12 +20,21 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea
 import time
 import datetime
 import wtforms
+from wtforms.ext.sqlalchemy.fields import QuerySelectMultipleField
 
 import flask_wtf
 from flask_babel import lazy_gettext
 
 import hawat.const
 import hawat.forms
+from mentat.datatype.sqldb import GroupModel
+
+
+def get_available_groups():
+    """
+    Query the database for list of all available groups.
+    """
+    return hawat.db.db_query(GroupModel).order_by(GroupModel.name).all()
 
 
 class EventReportSearchForm(flask_wtf.FlaskForm):
@@ -38,6 +47,11 @@ class EventReportSearchForm(flask_wtf.FlaskForm):
             wtforms.validators.Optional()
         ]
     )
+    groups = QuerySelectMultipleField(
+        lazy_gettext('Groups:'),
+        query_factory = get_available_groups,
+        allow_blank = True
+    )
     dt_from = hawat.forms.DateTimeLocalField(
         lazy_gettext('From:'),
         validators = [
@@ -75,3 +89,63 @@ class EventReportSearchForm(flask_wtf.FlaskForm):
         filters = [int],
         default = 1
     )
+
+class SimpleReportingDashboardForm(flask_wtf.FlaskForm):
+    """
+    Class representing simple event reporting dashboard search form.
+    """
+    groups = QuerySelectMultipleField(
+        lazy_gettext('Groups:'),
+        query_factory = get_available_groups,
+        allow_blank = True
+    )
+    dt_from = hawat.forms.DateTimeLocalField(
+        lazy_gettext('From:'),
+        validators = [
+            wtforms.validators.Optional()
+        ],
+        format = '%Y-%m-%d'
+    )
+    interval = wtforms.SelectField(
+        lazy_gettext('Interval:'),
+        validators = [
+            wtforms.validators.Optional()
+        ],
+        choices = [
+            ('1day',   lazy_gettext('one day')),
+            ('1week',  lazy_gettext('one week')),
+            ('4weeks', lazy_gettext('four weeks'))
+        ],
+        filters = [lambda x: x or None]
+    )
+    submit = wtforms.SubmitField(
+        lazy_gettext('Calculate')
+    )
+
+class ReportingDashboardForm(flask_wtf.FlaskForm):
+    """
+    Class representing event reporting dashboard search form.
+    """
+    groups = QuerySelectMultipleField(
+        lazy_gettext('Groups:'),
+        query_factory = get_available_groups,
+        allow_blank = True
+    )
+    dt_from = hawat.forms.DateTimeLocalField(
+        lazy_gettext('From:'),
+        validators = [
+            wtforms.validators.Optional()
+        ],
+        format = '%Y-%m-%d %H:%M',
+        default = datetime.datetime.fromtimestamp(int(time.time() - (time.time() % 86400)))
+    )
+    dt_to = hawat.forms.DateTimeLocalField(
+        lazy_gettext('To:'),
+        validators = [
+            wtforms.validators.Optional()
+        ],
+        format = '%Y-%m-%d %H:%M'
+    )
+    submit = wtforms.SubmitField(
+        lazy_gettext('Calculate')
+    )

lib/hawat/blueprints/reports/templates/reports/dashboard.html

+{% extends "_layout.html" %}
+
+{% block content %}
+
+            <div class="row">
+                <div class="col-lg-12">
+
+                    <div class="jumbotron" style="margin-top: 1em;">
+                        <h2>{{ hawat_view_title }}</h2>
+                        <hr>
+                        <form method="GET" class="form-inline" action="{{ url_for('reports.dashboard') }}">
+                            {{ macros_site.render_form_item_datetime(search_form.dt_from, 'datetimepicker-hm-from', False) }}
+
+                            {{ macros_site.render_form_item_datetime(search_form.dt_to, 'datetimepicker-hm-to', False) }}
+
+                            {{ search_form.submit(class_='btn btn-primary') }}
+                        </form>
+                        {%- if search_form.groups.errors %}
+                        <div>
+                            {{ macros_site.form_errors(search_form.groups.errors) }}
+                        </div>
+                        {%- endif %}
+                        {%- if search_form.dt_from.errors %}
+                        <div>
+                            {{ macros_site.form_errors(search_form.dt_from.errors) }}
+                        </div>
+                        {%- endif %}
+                        {%- if search_form.dt_to.errors %}
+                        <div>
+                            {{ macros_site.form_errors(search_form.dt_to.errors) }}
+                        </div>
+                        {%- endif %}
+                    </div> <!-- /.jumbotron -->
+
+                </div> <!-- /.col-lg-12 -->
+            </div> <!-- /.row -->
+
+    {%- if searched %}
+        {%- if items %}
+
+            <div class="row">
+                <div class="col-lg-12">
+
+                    <div class="panel panel-default">
+                        <div class="panel-heading">
+                            <h3 class="panel-title">{{ gettext('Result summary') }}</h3>
+                        </div>
+                        <table class="table">
+                            <tr>
+                                <th>{{ gettext('Number of reports') }}:</th>
+                                <td>{{ babel_format_decimal(items_count) }}</td>
+                            </tr>
+                            <tr>
+                                <th>{{ gettext('Number of reported events') }}:</th>
+                                <td>{{ babel_format_decimal(statistics['count']) }}</td>
+                            </tr>
+                            <tr>
+                                <th>{{ gettext('Time interval') }}:</th>
+                                <td>{{ babel_format_datetime(statistics['dt_from']) }} - {{ babel_format_datetime(statistics['dt_to']) }}</td>
+                            </tr>
+                            <tr>
+                                <th>{{ gettext('Time interval delta') }}:</th>
+                                <td>{{ babel_format_timedelta(statistics['dt_to'] - statistics['dt_from']) }}</td>
+                            </tr>
+                        </table>
+                    </div>
+
+                </div> <!-- /.col-lg-12 -->
+            </div> <!-- /.row -->
+
+            <!-- Nav tabs -->
+            <ul class="nav nav-tabs nav-tabs-tooltipped" role="tablist">
+                {%- for chsection in ('abuses', 'analyzers', 'asns', 'categories', 'category_sets', 'countries', 'detectors', 'detectorsws', 'ips') %}
+                <li role="presentation" class="text-center{% if loop.first %} active{% endif %}">
+                    <a role="tab" data-toggle="tab" href="#tab-stats-overall-{{ chsection }}" class="chart-tab">
+                        # {{ chsection }}
+                    </a>
+                </li>
+                {%- endfor %}
+            </ul>
+
+            <!-- Tab panes -->
+            <div class="tab-content">
+                {%- for chsection in ('abuses', 'analyzers', 'asns', 'categories', 'category_sets', 'countries', 'detectors', 'detectorsws', 'ips') %}
+                <div role="tabpanel" class="tab-pane fade{% if loop.first %} in active{% endif %}" id="tab-stats-overall-{{ chsection }}">
+                    {{ macros_site.render_chart_pie(
+                            'reporting_stats',
+                            statistics,
+                            chsection,
+                            'Number of events per ' + chsection,
+                        )
+                    }}
+                </div>
+                {%- endfor %}
+            </div>
+
+            {%- if permission_can('developer') %}
+            <hr>
+            {{ macros_site.render_raw_var('items', items) }}
+            {{ macros_site.render_raw_var('statistics', statistics) }}
+            {%- endif %}
+
+        {%- else %}
+            {%- call macros_site.render_alert('info') %}
+                {{ gettext('No data matches your search criteria.') }}
+            {%- endcall %}
+        {%- endif %}
+    {%- endif %}
+
+{%- endblock content %}
+
+
+{%- block css %}
+{{ super() }}
+        <link rel="stylesheet" href="{{ url_for('design.static', filename='vendor/nvd3/css/nv.d3.min.css') }}">
+{%- endblock css %}
+
+
+{%- block headjs %}
+{{ super() }}
+        <script src="{{ url_for('design.static', filename='vendor/d3/js/d3.min.js') }}"></script>
+        <script src="{{ url_for('design.static', filename='vendor/nvd3/js/nv.d3.min.js') }}"></script>
+{%- endblock headjs %}
+
+
+{%- block js %}
+{{ super() }}
+        <script src="{{ url_for('design.static', filename='vendor/datatables/js/jquery.dataTables.js') }}"></script>
+        <script src="{{ url_for('design.static', filename='vendor/datatables/js/dataTables.bootstrap.js') }}"></script>
+
+        <script>
+            $(document).ready(function() {
+                $('.hawat-datatable-nopaging').DataTable({
+                    "responsive": true,
+                    "pageLength": 50,
+                    "paging": false,
+                    "ordering": true
+                });
+            });
+        </script>
+{%- endblock js %}

lib/hawat/blueprints/reports/templates/reports/show.html

@@ -46,7 +46,7 @@
                                 <a data-toggle="tooltip" role="button" class="btn btn-default btn-sm" href="{{ url_for('reports.show', item_id = item.id ) }}" title="{{ gettext('Delete report &quot;%(item)s&quot;', item = item.label) }}">{{ get_fa_icon('trash') }} {{ gettext('Delete') }}</a>
                                 <div class="btn-group" role="group">
                                     <a href="#" class="btn btn-default btn-sm dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
-                                        {{ get_fa_icon('more') }} {{ gettext('More')}} <span class="caret"></span>
+                                        {{ get_fa_icon('action-more') }} {{ gettext('More')}} <span class="caret"></span>
                                     </a>
                                     <ul class="dropdown-menu dropdown-menu-right">
                                         <li><a data-toggle="tooltip" href="{{ url_for('reports.data', fileid = item.label, filetype = 'json' ) }}" title="{{ gettext('Download data in JSON format for report &quot;%(item)s&quot;', item = item.label) }}">{{ get_fa_icon('save') }} {{ gettext('Download report data in JSON') }}</a></li>
@@ -178,6 +178,23 @@
                 </div>
             </div>
 
+    {%- if permission_can('developer') %}
+
+            <hr>
+
+<pre>
+{{ item | pprint }}
+</pre>
+
+<pre>
+{{ item.statistics | pprint }}
+</pre>
+
+
+
+        {%- endif %}
+
+
 {% endblock content %}
 
 {%- block css %}

lib/mentat/stats/idea.py

@@ -269,17 +269,18 @@ def aggregate_stats(stats, interval, dt_from, result = None):
     :rtype: dict
     """
     if not result:
-        result = {
-            ST_SKEY_CNT_ALERTS: 0,
-            ST_SKEY_HISTOGRAM: []
-        }
+        result = {}
+    if ST_SKEY_CNT_ALERTS not in result:
+        result[ST_SKEY_CNT_ALERTS] = 0
+    if ST_SKEY_HISTOGRAM not in result:
+        result[ST_SKEY_HISTOGRAM] = []
 
     result[ST_SKEY_CNT_ALERTS] += stats[ST_SKEY_CNT_ALERTS]
     result[ST_SKEY_HISTOGRAM].append((
-            interval,
-            dt_from,
-            stats[ST_SKEY_CNT_ALERTS]
-        ))
+        interval,
+        dt_from,
+        stats[ST_SKEY_CNT_ALERTS]
+    ))
 
     for key in LIST_CALCSTAT_KEYS:
         if key in stats:
@@ -331,6 +332,41 @@ def aggregate_stat_groups(stats_list, result = None):
 
     return result
 
+def aggregate_stats_reports(report_list, result = None):
+    """
+
+    """
+    if result is None:
+        result = dict()
+    result[ST_SKEY_COUNT] = 0
+
+    for report in report_list:
+        result[ST_SKEY_COUNT] += report.statistics[ST_SKEY_CNT_ALERTS]
+
+        if 'dt_from' in result:
+            result['dt_from'] = min(result['dt_from'], report.dt_from)
+        else:
+            result['dt_from'] = report.dt_from
+        if 'dt_to' in result:
+            result['dt_to'] = max(result['dt_to'], report.dt_to)
+        else:
+            result['dt_to'] = report.dt_to
+
+        if not report.statistics[ST_SKEY_CNT_ALERTS]:
+            continue
+
+        aggregate_stats(
+            report.statistics,
+            '{}_{}'.format(
+                report.dt_from.strftime('%FT%T'),
+                report.dt_to.strftime('%FT%T')
+            ),
+            report.dt_from,
+            result
+        )
+
+    return result
+
 
 #-------------------------------------------------------------------------------