diff --git a/conf/templates/reporter/default.extra.txt.j2 b/conf/templates/reporter/default.extra.txt.j2
index 4b362de87da006246d27a961121575d722d6fc32..cb5c3b8dedf9b4d1e4088853f079545901575779 100644
--- a/conf/templates/reporter/default.extra.txt.j2
+++ b/conf/templates/reporter/default.extra.txt.j2
@@ -105,19 +105,23 @@
{{ _('This message was generated by an automated system. For further communication please use the contact email address <{:s}> and for easier orientation please keep the identifier [{:s}] in email subject.').format(additional_vars['contact_email'], report.label) | wordwrap(width=text_width, break_long_words=False) }}
+{{ _('This report contains fraction of data from parent summary report [{:s}], that is also available in HTML format at URL address:').format(report.parent.label) | wordwrap(width=text_width, break_long_words=False) }}
+ {{ additional_vars['report_access_url'] }}{{ report.parent.handle }}/unauth
+
{{ _('Please report any possible issues with this automatic report to email address <{:s}>.').format(additional_vars['admin_email']) | wordwrap(width=text_width, break_long_words=False) }}
{{ _('Thank you in advance for your cooperation') }}
{{ _('Mentat System') }} ({{ _('https://mentat.cesnet.cz/en/index') }})
{{ _('CESNET-CERTS Computer Security Team') }} <certs@cesnet.cz> ({{ _('https://csirt.cesnet.cz/en/index') }})
-{{ _('CESNET, z.s.p.o.') }} ({{ _('http://www.ces.net/') }})
+{{ _('CESNET, a.l.e.') }} ({{ _('http://www.ces.net/') }})
===
{{ _('Extra report') }} {{ report.label }}
-{{ report.evcount_all }} {{ _('event(s)') }}
-{{ _('Generated for: ') }} {{ report.group.name }}
-{{ _('Generated at: ') }} {{ format_localdatetime(dt_c) }}
- ({{ format_datetime(dt_c) }} UTC)
-{{ _('Time period: ') }} {{ format_localdatetime(report.dt_from) }} - {{ format_localdatetime(report.dt_to) }}
- ({{ format_datetime(report.dt_from) }} - {{ format_datetime(report.dt_to) }} UTC)
+{{ report.evcount_rep }} {{ _('event(s)') }}
+{{ '{:20s}'.format(_('Parent report:')) }} {{ report.parent.label }}
+{{ '{:20s}'.format(_('Generated for:')) }} {{ report.group.name }}
+{{ '{:20s}'.format(_('Generated at:')) }} {{ format_localdatetime(dt_c) }}
+ ({{ format_datetime(dt_c) }} UTC)
+{{ '{:20s}'.format(_('Time period:')) }} {{ format_localdatetime(report.dt_from) }} - {{ format_localdatetime(report.dt_to) }}
+ ({{ format_datetime(report.dt_from) }} - {{ format_datetime(report.dt_to) }} UTC)
diff --git a/conf/templates/reporter/default.summary.txt.j2 b/conf/templates/reporter/default.summary.txt.j2
index 75f34098d1871128e39f78395e484887cc8cff37..ccf77281b582f40b7ea7d85b14c32587d5734d55 100644
--- a/conf/templates/reporter/default.summary.txt.j2
+++ b/conf/templates/reporter/default.summary.txt.j2
@@ -111,13 +111,13 @@
{{ _('Mentat System') }} ({{ _('https://mentat.cesnet.cz/en/index') }})
{{ _('CESNET-CERTS Computer Security Team') }} <certs@cesnet.cz> ({{ _('https://csirt.cesnet.cz/en/index') }})
-{{ _('CESNET, z.s.p.o.') }} ({{ _('http://www.ces.net/') }})
+{{ _('CESNET, a.l.e.') }} ({{ _('http://www.ces.net/') }})
===
{{ _('Summary report') }} {{ report.label }}
-{{ report.evcount_thr }} {{ _('event(s)') }}, {{ report.evcount_flt_blk }} {{ _('filtered out') }}, {{ report.evcount_thr_blk }} {{ _('thresholded') }}, {{ report.evcount_rlp }} {{ _('relapsed') }}
-{{ _('Generated for: ') }} {{ report.group.name }}
-{{ _('Generated at: ') }} {{ format_localdatetime(dt_c) }}
- ({{ format_datetime(dt_c) }} UTC)
-{{ _('Time period: ') }} {{ format_localdatetime(report.dt_from) }} - {{ format_localdatetime(report.dt_to) }}
- ({{ format_datetime(report.dt_from) }} - {{ format_datetime(report.dt_to) }} UTC)
+{{ report.evcount_rep }} {{ _('event(s)') }}, {{ report.evcount_flt_blk }} {{ _('filtered out') }}, {{ report.evcount_thr_blk }} {{ _('thresholded') }}, {{ report.evcount_rlp }} {{ _('relapsed') }}
+{{ '{:20s}'.format(_('Generated for:')) }} {{ report.group.name }}
+{{ '{:20s}'.format(_('Generated at:')) }} {{ format_localdatetime(dt_c) }}
+ ({{ format_datetime(dt_c) }} UTC)
+{{ '{:20s}'.format(_('Time period:')) }} {{ format_localdatetime(report.dt_from) }} - {{ format_localdatetime(report.dt_to) }}
+ ({{ format_datetime(report.dt_from) }} - {{ format_datetime(report.dt_to) }} UTC)
diff --git a/lib/hawat/blueprints/design/templates/_macros_site.html b/lib/hawat/blueprints/design/templates/_macros_site.html
index 9c307b480e3e0c989dfab5c32565593bf96b4b41..6291a6ab1a0c96e980613f676e6ed66c3615a6ec 100644
--- a/lib/hawat/blueprints/design/templates/_macros_site.html
+++ b/lib/hawat/blueprints/design/templates/_macros_site.html
@@ -602,26 +602,37 @@
----------------------------------------------------------------------------- #}
-{%- macro render_report_label_type(report) %}
+{%- macro render_report_label_type(report, with_label = False) %}
{%- if report.type == 'summary' %}
-<span class="label label-default" title="{{ gettext('Summary report') }}" data-toggle="tooltip">{{ get_fa_icon('r-t-summary') }}</span>
+<span class="label label-default" title="{{ gettext('Summary report') }}" data-toggle="tooltip">{{ get_fa_icon('r-t-summary') }}{% if with_label %} {{ gettext('summary') }}{% endif %}</span>
{%- elif report.type == 'extra' %}
-<span class="label label-default" title="{{ gettext('Extra report') }}" data-toggle="tooltip">{{ get_fa_icon('r-t-extra') }}</span>
+<span class="label label-default" title="{{ gettext('Extra report') }}" data-toggle="tooltip">{{ get_fa_icon('r-t-extra') }}{% if with_label %} {{ gettext('extra') }}{% endif %}</span>
{%- endif %}
{%- endmacro %}
-{%- macro render_report_label_severity(report) %}
+{%- macro render_report_label_severity(report, with_label = False) %}
{%- if report.severity == 'low' %}
-<span class="label label-info" title="{{ gettext('Low severity') }}" data-toggle="tooltip">{{ get_fa_icon('r-s-low') }}</span>
+<span class="label label-info" title="{{ gettext('Low severity') }}" data-toggle="tooltip">{{ get_fa_icon('r-s-low') }}{% if with_label %} {{ gettext('low') }}{% endif %}</span>
{%- elif report.severity == 'medium' %}
-<span class="label label-primary" title="{{ gettext('Medium severity') }}" data-toggle="tooltip">{{ get_fa_icon('r-s-medium') }}</span>
+<span class="label label-primary" title="{{ gettext('Medium severity') }}" data-toggle="tooltip">{{ get_fa_icon('r-s-medium') }}{% if with_label %} {{ gettext('medium') }}{% endif %}</span>
{%- elif report.severity == 'high' %}
-<span class="label label-warning" title="{{ gettext('High severity') }}" data-toggle="tooltip">{{ get_fa_icon('r-s-high') }}</span>
+<span class="label label-warning" title="{{ gettext('High severity') }}" data-toggle="tooltip">{{ get_fa_icon('r-s-high') }}{% if with_label %} {{ gettext('high') }}{% endif %}</span>
{%- elif report.severity == 'critical' %}
-<span class="label label-danger" title="{{ gettext('Critical severity') }}" data-toggle="tooltip">{{ get_fa_icon('r-s-critical') }}</span>
+<span class="label label-danger" title="{{ gettext('Critical severity') }}" data-toggle="tooltip">{{ get_fa_icon('r-s-critical') }}{% if with_label %} {{ gettext('critical') }}{% endif %}</span>
{%- endif %}
{%- endmacro %}
+{%- macro render_report_label_weight(report) %}
+ {%- if report.evcount_rep < 10 %}
+<span class="label label-info" title="{{ gettext('Low event count') }}" data-toggle="tooltip">{{ get_fa_icon('weight') }} <span class="badge">{{ report.evcount_rep}}</span></span>
+ {%- elif report.evcount_rep < 100 %}
+<span class="label label-primary" title="{{ gettext('Medium event count') }}" data-toggle="tooltip">{{ get_fa_icon('weight') }} <span class="badge">{{ report.evcount_rep}}</span></span>
+ {%- elif report.evcount_rep < 1000 %}
+<span class="label label-warning" title="{{ gettext('High event count') }}" data-toggle="tooltip">{{ get_fa_icon('weight') }} <span class="badge">{{ report.evcount_rep}}</span></span>
+ {%- else %}
+<span class="label label-danger" title="{{ gettext('Critical event count') }}" data-toggle="tooltip">{{ get_fa_icon('weight') }} <span class="badge">{{ report.evcount_rep}}</span></span>
+ {%- endif %}
+{%- endmacro %}
{#- ----------------------------------------------------------------------------
diff --git a/lib/hawat/blueprints/reports/__init__.py b/lib/hawat/blueprints/reports/__init__.py
index 64798dacaf014e5ee971f20c794ba2f303f96701..a3be32a82fee2ae0256bc875c72909d3c46cc34c 100644
--- a/lib/hawat/blueprints/reports/__init__.py
+++ b/lib/hawat/blueprints/reports/__init__.py
@@ -24,13 +24,14 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea
import flask
import flask_login
import flask_principal
-from flask_babel import lazy_gettext
+from flask_babel import gettext, lazy_gettext
import mentat.const
import mentat.stats.idea
from mentat.datatype.sqldb import EventReportModel, GroupModel, UserModel
import hawat.base
+import hawat.menu
from hawat.blueprints.reports.forms import EventReportSearchForm, ReportingDashboardForm
@@ -127,14 +128,20 @@ class SearchView(hawat.base.HawatSearchView):
# Adjust query based on group selection.
query = adjust_query_for_groups(query, args.get('groups', None))
# Adjust query based on text search string.
- if 'search' in args and args['search']:
- query = query.filter(model.label.like('%{}%'.format(args['search'])))
+ if 'label' in args and args['label']:
+ query = query.filter(model.label.like('%{}%'.format(args['label'])))
# Adjust query based on lower time boudary selection.
if 'dt_from' in args and args['dt_from']:
query = query.filter(model.dt_from >= args['dt_from'])
# Adjust query based on upper time boudary selection.
if 'dt_to' in args and args['dt_to']:
query = query.filter(model.dt_to <= args['dt_to'])
+ # Adjust query based on report severity selection.
+ if 'severities' in args and args['severities']:
+ query = query.filter(model.severity.in_(args['severities']))
+ # Adjust query based on report type selection.
+ if 'severities' in args and args['types']:
+ query = query.filter(model.type.in_(args['types']))
query = query.order_by(model.dt_to.desc()).order_by(model.label.desc())
@@ -204,6 +211,74 @@ class ShowView(hawat.base.HawatItemShowView):
)
return hawat.acl.PERMISSION_ADMIN.can() or permission_mm.can()
+ @classmethod
+ def get_action_menu(cls, item):
+ """
+ Get action menu for particular item.
+ """
+ action_menu = hawat.menu.HawatMenu()
+
+ action_menu.add_entry(
+ 'endpoint',
+ 'delete',
+ endpoint = 'reports.delete',
+ item = item,
+ title = lazy_gettext('Delete'),
+ icon = 'action-delete',
+ legend = lazy_gettext('Delete report "%(item)s"', item = item.label)
+ )
+ action_menu.add_entry(
+ 'submenu',
+ 'more',
+ icon = 'section-more',
+ legend = lazy_gettext('More actions')
+ )
+ action_menu.add_entry(
+ 'endpoint',
+ 'more.downloadjson',
+ item = item,
+ endpoint = 'reports.data',
+ link = flask.url_for('reports.data', fileid = item.label, filetype = 'json'),
+ title = lazy_gettext('Download data in JSON format'),
+ icon = 'action-save'
+ )
+ action_menu.add_entry(
+ 'endpoint',
+ 'more.downloadcsv',
+ item = item,
+ endpoint = 'reports.data',
+ link = flask.url_for('reports.data', fileid = item.label, filetype = 'csv'),
+ title = lazy_gettext('Download data in CSV format'),
+ icon = 'action-save'
+ )
+ action_menu.add_entry(
+ 'endpoint',
+ 'more.downloadjsonzip',
+ item = item,
+ endpoint = 'reports.data',
+ link = flask.url_for('reports.data', fileid = item.label, filetype = 'jsonzip'),
+ title = lazy_gettext('Download compressed data in JSON format'),
+ icon = 'action-save'
+ )
+ action_menu.add_entry(
+ 'endpoint',
+ 'more.downloadcsvzip',
+ item = item,
+ endpoint = 'reports.data',
+ link = flask.url_for('reports.data', fileid = item.label, filetype = 'csvzip'),
+ title = lazy_gettext('Download compressed data in CSV format'),
+ icon = 'action-save'
+ )
+
+ return action_menu
+
+ def do_before_render(self, item, context):
+ """
+ *Hook method*. Will be called before rendering the template.
+ """
+ context.update(
+ statistics = mentat.stats.idea.calculate_secondary_stats(item.statistics)
+ )
class UnauthShowView(ShowView):
"""
@@ -220,6 +295,13 @@ class UnauthShowView(ShowView):
"""
return 'unauth'
+ @classmethod
+ def get_view_template(cls):
+ """
+ *Interface implementation* of :py:func:`hawat.base.HawatRenderableView.get_view_template`.
+ """
+ return '{}/show.html'.format(cls.module_name)
+
def dispatch_request(self, item_id):
"""
Mandatory interface required by the :py:func:`flask.views.View.dispatch_request`.
@@ -369,6 +451,69 @@ class DashboardView(hawat.base.HawatSearchView):
)
+class DeleteView(hawat.base.HawatItemDeleteView):
+ """
+ View for deleting existing user accounts.
+ """
+ methods = ['GET','POST']
+
+ authentication = True
+
+ authorization = [hawat.acl.PERMISSION_ADMIN]
+
+ @classmethod
+ def get_menu_icon(cls):
+ """
+ *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_menu_icon`.
+ """
+ return 'module-{}'.format(BLUEPRINT_NAME)
+
+ @classmethod
+ def get_menu_title(cls):
+ """
+ *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_menu_title`.
+ """
+ return lazy_gettext('Delete event report')
+
+ def get_url_next(self):
+ """
+ *Interface implementation* of :py:func:`hawat.base.HawatItemActionView.get_url_next`.
+ """
+ return flask.url_for('{}.{}'.format(self.module_name, 'search'))
+
+ #---------------------------------------------------------------------------
+
+ @property
+ def dbmodel(self):
+ """
+ *Interface implementation* of :py:func:`hawat.base.HawatDbmodelView.dbmodel`.
+ """
+ return EventReportModel
+
+ #---------------------------------------------------------------------------
+
+ @staticmethod
+ def get_message_success(**kwargs):
+ """
+ *Hook method*. Implementation of :py:func:`hawat.base.HawatItemActionView.get_message_success` interface.
+ """
+ return gettext('Event report <strong>%(item_id)s</strong> was successfully and permanently deleted.', item_id = str(kwargs['item']))
+
+ @staticmethod
+ def get_message_failure(**kwargs):
+ """
+ *Hook method*. Implementation of :py:func:`hawat.base.HawatItemActionView.get_message_failure` interface.
+ """
+ return gettext('Unable to delete event report <strong>%(item_id)s</strong>.', item_id = str(kwargs['item']))
+
+ @staticmethod
+ def get_message_cancel(**kwargs):
+ """
+ *Hook method*. Implementation of :py:func:`hawat.base.HawatItemActionView.get_message_cancel` interface.
+ """
+ return gettext('Canceled deleting event report <strong>%(item_id)s</strong>.', item_id = str(kwargs['item']))
+
+
#-------------------------------------------------------------------------------
@@ -429,5 +574,6 @@ def get_blueprint():
hbp.register_view_class(UnauthShowView, '/<item_id>/unauth')
hbp.register_view_class(DataView, '/data/<fileid>/<filetype>')
hbp.register_view_class(DashboardView, '/dashboard')
+ hbp.register_view_class(DeleteView, '/<int:item_id>/delete')
return hbp
diff --git a/lib/hawat/blueprints/reports/forms.py b/lib/hawat/blueprints/reports/forms.py
index 6a6a11f94fd15d1a0cbd7f65f8e57aa8eb511ba7..2831fe5c4d7d559f0de6a105700de6def7e13dc4 100644
--- a/lib/hawat/blueprints/reports/forms.py
+++ b/lib/hawat/blueprints/reports/forms.py
@@ -28,6 +28,7 @@ from flask_babel import lazy_gettext
import hawat.const
import hawat.forms
+import mentat.const
from mentat.datatype.sqldb import UserModel, GroupModel
@@ -46,13 +47,34 @@ def get_available_groups():
order_by(GroupModel.name).\
all()
+def get_severity_choices():
+ """
+ Return select choices for report severities.
+ """
+ return list(
+ zip(
+ mentat.const.REPORT_SEVERITIES,
+ [lazy_gettext(x) for x in mentat.const.REPORT_SEVERITIES]
+ )
+ )
+
+def get_type_choices():
+ """
+ Return select choices for report severities.
+ """
+ return list(
+ zip(
+ mentat.const.REPORT_TYPES,
+ [lazy_gettext(x) for x in mentat.const.REPORT_TYPES]
+ )
+ )
class EventReportSearchForm(flask_wtf.FlaskForm):
"""
Class representing event report search form.
"""
- search = wtforms.StringField(
- lazy_gettext('Search reports:'),
+ label = wtforms.StringField(
+ lazy_gettext('Label:'),
validators = [
wtforms.validators.Optional()
]
@@ -62,6 +84,22 @@ class EventReportSearchForm(flask_wtf.FlaskForm):
query_factory = get_available_groups,
allow_blank = True
)
+ severities = wtforms.SelectMultipleField(
+ lazy_gettext('Severities:'),
+ validators = [
+ wtforms.validators.Optional(),
+ ],
+ choices = get_severity_choices(),
+ filters = [lambda x: x or []]
+ )
+ types = wtforms.SelectMultipleField(
+ lazy_gettext('Types:'),
+ validators = [
+ wtforms.validators.Optional(),
+ ],
+ choices = get_type_choices(),
+ filters = [lambda x: x or []]
+ )
dt_from = hawat.forms.DateTimeLocalField(
lazy_gettext('From:'),
validators = [
@@ -100,6 +138,7 @@ class EventReportSearchForm(flask_wtf.FlaskForm):
default = 1
)
+
class SimpleReportingDashboardForm(flask_wtf.FlaskForm):
"""
Class representing simple event reporting dashboard search form.
diff --git a/lib/hawat/blueprints/reports/templates/reports/dashboard.html b/lib/hawat/blueprints/reports/templates/reports/dashboard.html
index bcc7bcce567db0c2c7ee347b24fdb3b58d203359..f84f2afbda018f108ef1fc3647b1174e8d420b6b 100644
--- a/lib/hawat/blueprints/reports/templates/reports/dashboard.html
+++ b/lib/hawat/blueprints/reports/templates/reports/dashboard.html
@@ -89,6 +89,7 @@
{%- if permission_can('developer') %}
<hr>
+ {{ macros_site.render_raw_var('form_data', form_data) }}
{{ macros_site.render_raw_var('items', items) }}
{{ macros_site.render_raw_var('statistics', statistics) }}
{%- endif %}
@@ -100,6 +101,13 @@
{%- endcall %}
{%- endif %}
+
+ {%- if permission_can('developer') %}
+ <hr>
+ {{ macros_site.render_raw_var('request_args', request_args) }}
+ {{ macros_site.render_raw_var('query_params', query_params) }}
+ {%- endif %}
+
{%- endif %}
{%- endblock content %}
diff --git a/lib/hawat/blueprints/reports/templates/reports/search.html b/lib/hawat/blueprints/reports/templates/reports/search.html
index 89c0fd8441edd9e932957101b9b4215012c52f22..fe7c3afbea995084ee088e733223e01b4c3339e2 100644
--- a/lib/hawat/blueprints/reports/templates/reports/search.html
+++ b/lib/hawat/blueprints/reports/templates/reports/search.html
@@ -14,12 +14,14 @@
<hr>
<form method="GET" class="form" action="{{ url_for('reports.search') }}">
<div class="row">
- <div class="col-sm-12">
- {{ search_form.search.label(class_='sr-only') }}
- <div class="input-group">
- <div data-toggle="tooltip" class="input-group-addon" title="{{ gettext('Search report database for:') }}">{{ get_fa_icon('action-search') }}</div>
- {{ search_form.search(class_='form-control', placeholder=gettext('report label'), size='50') }}
- </div>
+ <div class="col-sm-4">
+ {{ macros_site.render_form_item_default(search_form.label) }}
+ </div>
+ <div class="col-sm-4">
+ {{ macros_site.render_form_item_datetime(search_form.dt_from, 'datetimepicker-hm-from') }}
+ </div>
+ <div class="col-sm-4">
+ {{ macros_site.render_form_item_datetime(search_form.dt_to, 'datetimepicker-hm-to') }}
</div>
</div>
<hr>
@@ -28,10 +30,10 @@
{{ macros_site.render_form_item_select(search_form.groups) }}
</div>
<div class="col-sm-4">
- {{ macros_site.render_form_item_datetime(search_form.dt_from, 'datetimepicker-hm-from') }}
+ {{ macros_site.render_form_item_select(search_form.severities) }}
</div>
<div class="col-sm-4">
- {{ macros_site.render_form_item_datetime(search_form.dt_to, 'datetimepicker-hm-to') }}
+ {{ macros_site.render_form_item_select(search_form.types) }}
</div>
</div>
<hr>
@@ -54,22 +56,30 @@
{%- for item in items %}
<a href="{{ url_for('reports.show', item_id = item.id ) }}" class="list-group-item">
<h4 class="list-group-item-heading">
+ {{ item.label }}
+ <hr>
{{ macros_site.render_report_label_type(item) }}
{{ macros_site.render_report_label_severity(item) }}
- {{ item.label }}
+ {{ macros_site.render_report_label_weight(item) }}
+ {%- if item.evcount_rlp %}
+ <span class="label label-danger" data-toggle="tooltip" title="{{ gettext('Report contains relapsed events') }}" >{{ get_fa_icon('relapses') }}</span>
+ {%- endif %}
{%- if item.flag_mailed %}
- <span title="{{ item.mail_res }}, {{ babel_format_datetime(item.mail_dt) }}" data-toggle="tooltip">{{ get_fa_icon('mail') }}</span>
+ <span class="label label-default" title="{{ item.mail_res }}, {{ babel_format_datetime(item.mail_dt) }}" data-toggle="tooltip">{{ get_fa_icon('mail') }}</span>
{%- endif %}
{%- if item.flag_testdata %}
- <span title="{{ gettext('Report was generated from test data') }}" data-toggle="tooltip">{{ get_fa_icon('debug') }}</span>
+ <span class="label label-default" title="{{ gettext('Report was generated from test data') }}" data-toggle="tooltip">{{ get_fa_icon('debug') }}</span>
{%- endif %}
</h4>
+ <hr>
<p class="list-group-item-text">
- {{ gettext('Abuse group:') }} {{ item.group.name }}
+ <strong>{{ gettext('Abuse group:') }}</strong> {{ item.group.name }}
|
- {{ gettext('Report window:') }} {{ babel_format_datetime(item.dt_from) }} - {{ babel_format_datetime(item.dt_to) }} ({{ babel_format_timedelta(item.delta) }})
+ <strong>{{ gettext('Report window:') }}</strong> {{ babel_format_datetime(item.dt_from) }} - {{ babel_format_datetime(item.dt_to) }} ({{ babel_format_timedelta(item.delta) }})
|
- {{ gettext('Created:') }} {{ babel_format_datetime(item.createtime) }} ({{ gettext('before') }} {{ babel_format_timedelta(current_datetime_utc - item.createtime) }})
+ <strong>{{ gettext('Created:') }}</strong> {{ babel_format_datetime(item.createtime) }} ({{ gettext('before') }} {{ babel_format_timedelta(current_datetime_utc - item.createtime) }})
+ |
+ <strong>{{ gettext('Event counts:') }}</strong> {{ item.evcount_rep }} {{ gettext('reported') }}{% if item.type == 'summary' %} ({{ item.evcount_all }} {{ gettext('matched') }}, {{ item.evcount_flt_blk }} {{ gettext('filtered out') }}, {{ item.evcount_thr_blk }} {{ gettext('thresholded') }}, {{ item.evcount_rlp }} {{ gettext('relapsed') }}){% endif %}
</p>
</a>
{%- endfor %}
@@ -81,20 +91,26 @@
{{ macros_site.render_pager('reports.search', query_params, pager_index_low, pager_index_high, pager_index_limit) }}
{%- if permission_can('developer') %}
-
<hr>
-
-{{ macros_site.render_raw_item_view(items) }}
-
+ {{ macros_site.render_raw_var('form_data', form_data) }}
+ {{ macros_site.render_raw_var('items', items) }}
+ {{ macros_site.render_raw_var('statistics', statistics) }}
{%- endif %}
{%- else %}
- {%- call macros_site.render_alert('warning', False) %}
+ {%- call macros_site.render_alert('warning', False) %}
{{ gettext('No data matches your search criteria.') }}
- {%- endcall %}
+ {%- endcall %}
{%- endif %}
+
+ {%- if permission_can('developer') %}
+ <hr>
+ {{ macros_site.render_raw_var('request_args', request_args) }}
+ {{ macros_site.render_raw_var('query_params', query_params) }}
+ {%- endif %}
+
{%- endif %}
{%- endblock content %}
diff --git a/lib/hawat/blueprints/reports/templates/reports/show.html b/lib/hawat/blueprints/reports/templates/reports/show.html
index 3c6bcb6a05122978c6667d79feb345c6f76525f1..963740692d0a9ecc2271027528512e8aede96e0d 100644
--- a/lib/hawat/blueprints/reports/templates/reports/show.html
+++ b/lib/hawat/blueprints/reports/templates/reports/show.html
@@ -18,8 +18,15 @@
{{ item.label }}
{{ macros_site.render_report_label_type(item) }}
{{ macros_site.render_report_label_severity(item) }}
+ {{ macros_site.render_report_label_weight(item) }}
+ {%- if item.evcount_rlp %}
+ <span class="label label-danger" data-toggle="tooltip" title="{{ gettext('Report contains relapsed events') }}" >{{ get_fa_icon('relapses') }}</span>
+ {%- endif %}
{%- if item.flag_mailed %}
- <span class="label label-default" title="{{ item.mail_res }}, {{ babel_format_datetime(item.mail_dt) }}" data-toggle="tooltip">{{ get_fa_icon('mail') }}</span>
+ <span class="label label-default" data-toggle="tooltip" title="{{ gettext('Report was mailed with following status:') }} {{ item.mail_res }}, {{ babel_format_datetime(item.mail_dt) }} ({{ gettext('before') }} {{ babel_format_timedelta(current_datetime_utc - item.mail_dt) }})">{{ get_fa_icon('mail') }}</span>
+ {%- endif %}
+ {%- if item.flag_testdata %}
+ <span class="label label-default" data-toggle="tooltip" title="{{ gettext('Report was generated from test data') }}" >{{ get_fa_icon('debug') }}</span>
{%- endif %}
</h3>
{%- if item.flag_testdata %}
@@ -27,44 +34,41 @@
{%- endif %}
<p>
<strong>{{ gettext('Target abuse group:')}}</strong>
- {{ item.group.name }}
+ <a href="{{ url_for('groups.show', item_id = item.group.id ) }}" data-toggle="tooltip" title="{{ gettext('View details of group "%(item)s"', item = item.group.name) }}">{{ item.group.name }}</a>
</p>
{%- if unauth %}
+ {%- if item.parent %}
+ <p>
+ <strong>{{ gettext('Parent summary report:')}}</strong>
+ <a href="{{ url_for('reports.unauth', item_id = item.parent.handle ) }}" data-toggle="tooltip" title="{{ gettext('View parent summary report "%(item)s"', item = item.parent.label) }}">{{ item.parent.handle }}</a>
+ </p>
+ {%- endif %}
<p>
<strong>{{ gettext('Protected access link:')}}</strong>
- <a href="{{ url_for('reports.show', item_id = item.id ) }}" title="{{ gettext('Protected access to report "%(item)s"', item = item.label) }}">{{ item.id }}</a>
+ <a href="{{ url_for('reports.show', item_id = item.id ) }}" data-toggle="tooltip" title="{{ gettext('Protected access to report "%(item)s"', item = item.label) }}">{{ gettext('link') }}</a>
</p>
{%- else %}
+ {%- if item.parent %}
+ <p>
+ <strong>{{ gettext('Parent summary report:')}}</strong>
+ <a href="{{ url_for('reports.show', item_id = item.parent.id ) }}" data-toggle="tooltip" title="{{ gettext('View parent summary report "%(item)s"', item = item.parent.label) }}">{{ item.parent.label }}</a>
+ </p>
+ {%- endif %}
<p>
<strong>{{ gettext('Unprotected access link:')}}</strong>
- <a href="{{ url_for('reports.unauth', item_id = item.handle ) }}" title="{{ gettext('Unprotected access to report "%(item)s"', item = item.label) }}">{{ item.handle }}</a>
+ <a href="{{ url_for('reports.unauth', item_id = item.handle ) }}" data-toggle="tooltip" title="{{ gettext('Unprotected access to report "%(item)s"', item = item.label) }}">{{ item.handle }}</a>
</p>
<div class="pull-right">
- <div class="btn-toolbar" role="toolbar" aria-label="{{ gettext('Action toolbar') }}">
- <div class="btn-group" role="group" aria-label="{{ gettext('Action buttons') }}">
- <a data-toggle="tooltip" role="button" class="btn btn-default btn-sm" href="{{ url_for('reports.show', item_id = item.id ) }}" title="{{ gettext('Remail report "%(item)s"', item = item.label) }}">{{ get_fa_icon('mail') }} {{ gettext('Remail') }}</a>
- <a data-toggle="tooltip" role="button" class="btn btn-default btn-sm" href="{{ url_for('reports.show', item_id = item.id ) }}" title="{{ gettext('Delete report "%(item)s"', item = item.label) }}">{{ get_fa_icon('trash') }} {{ gettext('Delete') }}</a>
- <div class="btn-group" role="group">
- <a href="#" class="btn btn-default btn-sm dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- {{ get_fa_icon('action-more') }} {{ gettext('More')}} <span class="caret"></span>
- </a>
- <ul class="dropdown-menu dropdown-menu-right">
- <li><a data-toggle="tooltip" href="{{ url_for('reports.data', fileid = item.label, filetype = 'json' ) }}" title="{{ gettext('Download data in JSON format for report "%(item)s"', item = item.label) }}">{{ get_fa_icon('save') }} {{ gettext('Download report data in JSON') }}</a></li>
- <li><a data-toggle="tooltip" href="{{ url_for('reports.data', fileid = item.label, filetype = 'csv' ) }}" title="{{ gettext('Download data in CSV format for report "%(item)s"', item = item.label) }}">{{ get_fa_icon('save') }} {{ gettext('Download report data in CSV') }}</a></li>
- <li role="separator" class="divider"></li>
- <li><a data-toggle="tooltip" href="{{ url_for('reports.data', fileid = item.label, filetype = 'jsonzip' ) }}" title="{{ gettext('Download compressed data in JSON format for report "%(item)s"', item = item.label) }}">{{ get_fa_icon('save') }} {{ gettext('Download compressed report data in JSON') }}</a></li>
- <li><a data-toggle="tooltip" href="{{ url_for('reports.data', fileid = item.label, filetype = 'csvzip' ) }}" title="{{ gettext('Download compressed data in CSV format for report "%(item)s"', item = item.label) }}">{{ get_fa_icon('save') }} {{ gettext('Download compressed report data in CSV') }}</a></li>
- </ul>
- </div>
- </div>
- </div>
+ {{ macros_site.render_menu_actions(action_menu) }}
</div>
{%- endif %}
<p>
<small>
- {{ gettext('Report created:') }} {{ babel_format_datetime(item.createtime) }} ({{ gettext('before') }} {{ babel_format_timedelta(current_datetime_utc - item.createtime) }})
+ <strong>{{ gettext('Report created:') }}</strong> {{ babel_format_datetime(item.createtime) }} ({{ gettext('before') }} {{ babel_format_timedelta(current_datetime_utc - item.createtime) }})
+ |
+ <strong>{{ gettext('Report window:') }}</strong> {{ babel_format_datetime(item.dt_from) }} - {{ babel_format_datetime(item.dt_to) }} ({{ babel_format_timedelta(item.delta) }})
|
- {{ gettext('Report window:') }} {{ babel_format_datetime(item.dt_from) }} - {{ babel_format_datetime(item.dt_to) }} ({{ babel_format_timedelta(item.delta) }})
+ <strong>{{ gettext('Report mailed:') }}</strong> {% if item.flag_mailed %}{{ babel_format_datetime(item.mail_dt) }} ({{ gettext('before') }} {{ babel_format_timedelta(current_datetime_utc - item.mail_dt) }}){% else %}{{ gettext('never')}}{% endif %}
</small>
</p>
</div>
@@ -101,6 +105,7 @@
<div class="tab-content">
<div role="tabpanel" class="tab-pane fade in active" id="tab-report-message">
+ <br>
<samp>
{{ item.message | replace("\n", "<br/>\n") | replace(' ', ' ') | replace("\t", ' ') | safe }}
</samp>
@@ -109,35 +114,79 @@
<div role="tabpanel" class="tab-pane fade" id="tab-report-metadata">
<table class="table table-condensed table-striped">
<tr>
- <th>{{ gettext('Type:') }}</th>
- <td>{{ gettext(item.type) }}</td>
+ <th>
+ {{ gettext('Type:') }}
+ </th>
+ <td>
+ {{ macros_site.render_report_label_type(item, True) }}
+ </td>
</tr>
<tr>
- <th>{{ gettext('Severity:') }}</th>
- <td>{{ gettext(item.severity) }}</td>
+ <th>
+ {{ gettext('Severity:') }}
+ </th>
+ <td>
+ {{ macros_site.render_report_label_severity(item, True) }}
+ </td>
</tr>
<tr>
- <th>{{ gettext('Report window:') }}</th>
- <td>{{ babel_format_datetime(item.dt_from) }} - {{ babel_format_datetime(item.dt_to) }} ({{ babel_format_timedelta(item.delta) }})</td>
+ <th>
+ {{ gettext('Report window:') }}
+ </th>
+ <td>
+ {{ babel_format_datetime(item.dt_from) }} - {{ babel_format_datetime(item.dt_to) }} ({{ babel_format_timedelta(item.delta) }})
+ </td>
</tr>
<tr>
- <th>{{ gettext('Report delay:') }}</th>
- <td>{{ babel_format_timedelta(item.createtime - item.dt_to) }}</td>
+ <th>
+ {{ gettext('Report delay:') }}
+ </th>
+ <td>
+ {{ babel_format_timedelta(item.createtime - item.dt_to) }}
+ </td>
</tr>
<tr>
- <th>{{ gettext('Event counts:') }}</th>
- <td>{{ item.statistics['cnt_alerts'] }}{% if item.evcount_all %} ({{ item.evcount_all }} {{ gettext('matched') }}, {{ item.evcount_flt_blk }} {{ gettext('filtered out') }}, {{ item.evcount_thr_blk }} {{ gettext('thresholded') }}, {{ item.evcount_rlp }} {{ gettext('relapsed') }}){% endif %}</td>
+ <th>
+ {{ gettext('Event count:') }}
+ </th>
+ <td>
+ <strong>{{ item.evcount_rep }}</strong> {{ gettext('reported') }}{% if item.type == 'summary' %} ({{ item.evcount_all }} {{ gettext('matched') }}, {{ item.evcount_flt_blk }} {{ gettext('filtered out') }}, {{ item.evcount_thr_blk }} {{ gettext('thresholded') }}, {{ item.evcount_rlp }} {{ gettext('relapsed') }}){% endif %}
+ </td>
</tr>
<tr>
- <th>{{ gettext('IP address counts:') }}</th>
- <td>{{ item.statistics['cnt_ips'] }}</td>
+ <th>
+ {{ gettext('IP address count:') }}
+ </th>
+ <td>
+ {{ statistics['cnt_ips'] }}
+ </td>
</tr>
- {%- if item.flag_mailed %}
<tr>
- <th>{{ gettext('Target mail:') }}</th>
- <td>{{ item.mail_res }}, {{ babel_format_datetime(item.mail_dt) }}</td>
+ <th>
+ {{ gettext('Target mail:') }}
+ </th>
+ <td>
+ {%- if item.flag_mailed %}
+ {{ item.mail_res }}, {{ babel_format_datetime(item.mail_dt) }} ({{ gettext('before') }} {{ babel_format_timedelta(current_datetime_utc - item.mail_dt) }})
+ {%- else %}
+ {{ gettext('never')}}
+ {%- endif %}
+ </td>
+ </tr>
+ <tr>
+ <th>
+ {{ gettext('Filtering:') }}
+ </th>
+ <td>
+ {%- if item.filtering %}
+ {%- for item in item.filtering | dictsort %}
+ {{ item[0] }}: {{ item[1] }}<br>
+ {%- endfor %}
+ {%- else %}
+ {{ gettext('no filters matched')}}
+ {%- endif %}
+ </td>
</tr>
- {%- endif %}
</table>
</div>
@@ -157,7 +206,7 @@
<div role="tabpanel" class="tab-pane fade{% if loop.first %} in active{% endif %}" id="tab-report-stats-{{ chsection }}">
{{ macros_site.render_chart_pie(
'report_stats',
- item.statistics,
+ statistics,
chsection,
'Number of events per ' + chsection,
)
@@ -179,20 +228,11 @@
</div>
{%- if permission_can('developer') %}
-
<hr>
-
-<pre>
-{{ item | pprint }}
-</pre>
-
-<pre>
-{{ item.statistics | pprint }}
-</pre>
-
-
-
- {%- endif %}
+ {{ macros_site.render_raw_var('item', item) }}
+ {{ macros_site.render_raw_var('statistics', statistics) }}
+ {{ macros_site.render_raw_var('filtering', item.filtering) }}
+ {%- endif %}
{% endblock content %}
diff --git a/lib/hawat/const.py b/lib/hawat/const.py
index 728285513511b66f2e797b5d4a08cc91581a2570..322d85cc5bd0517890aa018b4b6d5e6c02d661b5 100644
--- a/lib/hawat/const.py
+++ b/lib/hawat/const.py
@@ -146,6 +146,8 @@ FA_ICONS = {
'action-enable': '<i class="fas fa-fw fa-unlock"></i>',
'action-disable': '<i class="fas fa-fw fa-lock"></i>',
'action-delete': '<i class="fas fa-fw fa-trash"></i>',
+ 'action-save': '<i class="fas fa-fw fa-save"></i>',
+ 'action-mail': '<i class="fas fa-fw fa-envelope"></i>',
'action-add-user': '<i class="fas fa-fw fa-user-plus"></i>',
'alert-success': '<i class="fas fa-fw fa-check-circle"></i>',
@@ -161,6 +163,8 @@ FA_ICONS = {
'r-s-high': '<i class="fas fa-fw fa-thermometer-three-quarters"></i>',
'r-s-critical': '<i class="fas fa-fw fa-thermometer-full"></i>',
+ 'relapses': '<i class="fas fa-fw fa-thumbs-down"></i>',
+ 'caret-down': '<i class="fas fa-fw fa-caret-square-down"></i>',
'calendar': '<i class="fas fa-fw fa-calendar-alt"></i>',
'stopwatch': '<i class="fas fa-fw fa-stopwatch"></i>',
'clock': '<i class="fas fa-fw fa-clock"></i>',
@@ -180,6 +184,7 @@ FA_ICONS = {
'enable': '<i class="fas fa-fw fa-toggle-off"></i>',
'disable': '<i class="fas fa-fw fa-toggle-on"></i>',
'save': '<i class="fas fa-fw fa-save"></i>',
+ 'weight': '<i class="fas fa-fw fa-weight"></i>',
'view': '<i class="fas fa-fw fa-eye"></i>',
'list': '<i class="fas fa-fw fa-list-ul"></i>',
'mail': '<i class="fas fa-fw fa-envelope"></i>',
@@ -192,13 +197,13 @@ FA_ICONS = {
'import': '<i class="fas fa-fw fa-cloud-upload"></i>',
'export': '<i class="fas fa-fw fa-cloud-download"></i>',
'validate': '<i class="fas fa-fw fa-check-circle"></i>',
- 'min': '<i class="fas fa-fw fa-chevron-down" title="Minimal value" data-toggle="tooltip"></i>',
- 'max': '<i class="fas fa-fw fa-chevron-up" title="Maximal value" data-toggle="tooltip"></i>',
- 'sum': '<i class="fas fa-fw fa-plus" title="Sum of all values" data-toggle="tooltip"></i>',
- 'cnt': '<i class="fas fa-fw fa-asterisk" title="Count of all values" data-toggle="tooltip"></i>',
- 'avg': '<i class="fas fa-fw fa-dot-circle" title="Average value" data-toggle="tooltip"></i>',
- 'med': '<i class="fas fa-fw fa-bullseye" title="Median value" data-toggle="tooltip"></i>',
- 'na': '<i class="fas fa-fw fa-times" title="This value has no meaning" data-toggle="tooltip"></i>',
+ 'min': '<i class="fas fa-fw fa-angle-double-down"></i>',
+ 'max': '<i class="fas fa-fw fa-angle-double-up"></i>',
+ 'sum': '<i class="fas fa-fw fa-plus"></i>',
+ 'cnt': '<i class="fas fa-fw fa-hashtag"></i>',
+ 'avg': '<i class="fas fa-fw fa-dot-circle"></i>',
+ 'med': '<i class="fas fa-fw fa-bullseye"></i>',
+ 'na': '<i class="fas fa-fw fa-times"></i>',
'stats': '<i class="fas fa-fw fa-bar-chart"></i>',
'actions': '<i class="fas fa-fw fa-wrench"></i>',
'cog': '<i class="fas fa-fw fa-cog"></i>',
diff --git a/lib/mentat/datatype/sqldb.py b/lib/mentat/datatype/sqldb.py
index 78e81a2d446646a61cc3aa43d034cd9fe251cf52..34cfad23e58cafca8a54f78f0e89c77d5fe49b81 100644
--- a/lib/mentat/datatype/sqldb.py
+++ b/lib/mentat/datatype/sqldb.py
@@ -458,6 +458,9 @@ class EventReportModel(MODEL):
group_id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.ForeignKey('groups.id', onupdate = "CASCADE", ondelete = "CASCADE"), nullable = False)
group = sqlalchemy.orm.relationship('GroupModel', back_populates = 'reports')
+ parent_id = sqlalchemy.Column(sqlalchemy.Integer, sqlalchemy.ForeignKey('reports_events.id'))
+ children = sqlalchemy.orm.relationship('EventReportModel', backref = sqlalchemy.orm.backref('parent', remote_side='EventReportModel.id'))
+
label = sqlalchemy.Column(sqlalchemy.String, nullable = False, unique = True, index = True)
handle = sqlalchemy.Column(sqlalchemy.String, nullable = False, unique = True, index = True)
severity = sqlalchemy.Column(sqlalchemy.Enum(*REPORT_SEVERITIES, name='report_severities'), nullable = False)
@@ -471,11 +474,21 @@ class EventReportModel(MODEL):
flag_testdata = sqlalchemy.Column(sqlalchemy.Boolean, default = False, nullable = False)
flag_mailed = sqlalchemy.Column(sqlalchemy.Boolean, default = False, nullable = False)
+ # Number of events actually in report (evcount_thr + evcount_rlp).
+ evcount_rep = sqlalchemy.Column(sqlalchemy.Integer, nullable = False)
+ # Initial number of events for reporting (evcount_new + evcount_rlp).
evcount_all = sqlalchemy.Column(sqlalchemy.Integer, nullable = False)
+ # Number of matching events fetched from database.
+ evcount_new = sqlalchemy.Column(sqlalchemy.Integer)
+ # Number of events remaining after filtering.
evcount_flt = sqlalchemy.Column(sqlalchemy.Integer)
+ # Number of events blocked by filters (evcount_new - evcount_flt).
evcount_flt_blk = sqlalchemy.Column(sqlalchemy.Integer)
+ # Number of events remaining after thresholding.
evcount_thr = sqlalchemy.Column(sqlalchemy.Integer)
+ # Number of events blocked by thresholds (evcount_flt - evcount_thr).
evcount_thr_blk = sqlalchemy.Column(sqlalchemy.Integer)
+ # Number of relapsed events.
evcount_rlp = sqlalchemy.Column(sqlalchemy.Integer)
mail_to = sqlalchemy.Column(sqlalchemy.dialects.postgresql.ARRAY(sqlalchemy.String, dimensions = 1))
@@ -488,6 +501,9 @@ class EventReportModel(MODEL):
def __repr__(self):
return "<EventReport(label='%s')>" % (self.label)
+ def __str__(self):
+ return '{}'.format(self.label)
+
def calculate_delta(self):
"""
Calculate delta between internal time interval boundaries.
diff --git a/lib/mentat/reports/event.py b/lib/mentat/reports/event.py
index 075c7b3cc9a9271ec19ddad7f7c575fdcaa7a2d8..fdb7a2f3c2b02be3cac0fb2e6327acad7aa24a8c 100644
--- a/lib/mentat/reports/event.py
+++ b/lib/mentat/reports/event.py
@@ -198,7 +198,8 @@ class EventReporter(BaseReporter):
while True:
# A: Fetch events from database.
events_fetched = self.fetch_severity_events(abuse_group, severity, time_l, time_h, testdata)
- result['evcount_all'] = len(events_fetched)
+ result['evcount_new'] = len(events_fetched)
+ result['evcount_all'] = result['evcount_new']
if not events_fetched:
break
@@ -228,6 +229,7 @@ class EventReporter(BaseReporter):
# A: Detect possible event relapses.
events_rel = self.relapse_events(abuse_group, severity, time_h)
result['evcount_rlp'] = len(events_rel)
+ result['evcount_all'] = result['evcount_all'] + result['evcount_rlp']
if not events_rel:
break
@@ -240,14 +242,16 @@ class EventReporter(BaseReporter):
# Check, that there is anything to report (regular and/or relapsed events).
if 'regular' not in events and 'relapsed' not in events:
+ result['evcount_rep'] = 0
result['result'] = 'skipped-no-events'
return result
+ result['evcount_rep'] = len(events.get('regular', [])) + len(events.get('relapsed', []))
# Generate summary report.
- self.report_summary(result, events, abuse_group, settings, severity, time_l, time_h, template_vars)
+ report_summary = self.report_summary(result, events, abuse_group, settings, severity, time_l, time_h, template_vars, testdata)
# Generate extra reports.
- self.report_extra(result, events, abuse_group, settings, severity, time_l, time_h, template_vars)
+ self.report_extra(report_summary, result, events, abuse_group, settings, severity, time_l, time_h, template_vars, testdata)
# Update thresholding cache.
self.update_thresholding_cache(events, settings, severity, time_h)
@@ -255,7 +259,7 @@ class EventReporter(BaseReporter):
result['result'] = 'reported'
return result
- def report_summary(self, result, events, abuse_group, settings, severity, time_l, time_h, template_vars = None):
+ def report_summary(self, result, events, abuse_group, settings, severity, time_l, time_h, template_vars = None, testdata = False):
"""
Generate summary report from given events for given abuse group, severity and period.
@@ -267,6 +271,7 @@ class EventReporter(BaseReporter):
:param datetime.datetime time_l: Lower reporting time threshold.
:param datetime.datetime time_h: Upper reporting time threshold.
:param dict template_vars: Dictionary containing additional template variables.
+ :param bool testdata: Switch to use test data for reporting.
"""
# Instantinate the report object.
report = EventReportModel(
@@ -276,12 +281,16 @@ class EventReporter(BaseReporter):
dt_from = time_l,
dt_to = time_h,
- evcount_all = result['evcount_all'],
- evcount_flt = result['evcount_flt'],
- evcount_flt_blk = result['evcount_flt_blk'],
- evcount_thr = result['evcount_thr'],
- evcount_thr_blk = result['evcount_thr_blk'],
- evcount_rlp = result['evcount_rlp'],
+ evcount_rep = result.get('evcount_rep', 0),
+ evcount_all = result.get('evcount_all', 0),
+ evcount_new = result.get('evcount_new', 0),
+ evcount_flt = result.get('evcount_flt', 0),
+ evcount_flt_blk = result.get('evcount_flt_blk', 0),
+ evcount_thr = result.get('evcount_thr', 0),
+ evcount_thr_blk = result.get('evcount_thr_blk', 0),
+ evcount_rlp = result.get('evcount_rlp', 0),
+
+ flag_testdata = testdata,
filtering = result['filtering']
)
@@ -345,10 +354,13 @@ class EventReporter(BaseReporter):
result['summary_id'] = report.label
- def report_extra(self, result, events, abuse_group, settings, severity, time_l, time_h, template_vars = None):
+ return report
+
+ def report_extra(self, parent_rep, result, events, abuse_group, settings, severity, time_l, time_h, template_vars = None, testdata = False):
"""
Generate extra reports from given events for given abuse group, severity and period.
+ :param mentat.datatype.internal.EventReportModel parent_rep: Parent summary report.
:param dict result: Reporting result structure with various usefull metadata.
:param dict events: Dictionary structure with IDEA events to be reported.
:param mentat.datatype.internal.GroupModel abuse_group: Abuse group.
@@ -357,6 +369,7 @@ class EventReporter(BaseReporter):
:param datetime.datetime time_l: Lower reporting time threshold.
:param datetime.datetime time_h: Upper reporting time threshold.
:param dict template_vars: Dictionary containing additional template variables.
+ :param bool testdata: Switch to use test data for reporting.
"""
if settings.mode not in (mentat.const.REPORTING_MODE_EXTRA, mentat.const.REPORTING_MODE_BOTH):
return
@@ -369,23 +382,28 @@ class EventReporter(BaseReporter):
for src in sorted(sources):
+ events_regular_aggr = events.get('regular_aggr', {}).get(src, [])
+ events_relapsed_aggr = events.get('relapsed_aggr', {}).get(src, [])
+ events_all = events_regular_aggr + events_relapsed_aggr
+
# Instantinate the report object.
report = EventReportModel(
group = abuse_group,
+ parent = parent_rep,
severity = severity,
type = mentat.const.REPORTING_MODE_EXTRA,
dt_from = time_l,
dt_to = time_h,
- evcount_all = result['evcount_all']
+ evcount_rep = len(events_all),
+ evcount_all = result['evcount_rep'],
+
+ flag_testdata = testdata
)
report.generate_label()
report.generate_handle()
report.calculate_delta()
- events_regular_aggr = events.get('regular_aggr', {}).get(src, [])
- events_relapsed_aggr = events.get('relapsed_aggr', {}).get(src, [])
- events_all = events_regular_aggr + events_relapsed_aggr
report.statistics = mentat.stats.idea.truncate_evaluations(
mentat.stats.idea.evaluate_events(events_all)
)
diff --git a/lib/mentat/stats/idea.py b/lib/mentat/stats/idea.py
index a38580553f14420da63a5693dae0667f1a44736e..dbfb629418ca98fecaad9ce28a7c7855984dbd4a 100644
--- a/lib/mentat/stats/idea.py
+++ b/lib/mentat/stats/idea.py
@@ -125,7 +125,7 @@ def truncate_stats(stats, top_threshold = 20):
:return: Updated structure containing statistics.
:rtype: dict
"""
- if stats[ST_SKEY_CNT_ALERTS] > 0:
+ if stats.get(ST_SKEY_CNT_ALERTS, 0) > 0:
if ST_SKEY_LIST_IDS in stats:
del stats[ST_SKEY_LIST_IDS]
@@ -135,6 +135,23 @@ def truncate_stats(stats, top_threshold = 20):
return stats
+def calculate_secondary_stats(stats):
+ """
+ Calculate secondary statistics (cnt, min, max, sum, avg).
+
+ :param dict stats: Structure containing single statistic category.
+ :return: Updated structure containing statistics.
+ :rtype: dict
+ """
+ for key in LIST_CALCSTAT_KEYS:
+ if key in stats:
+ stats['cnt_{}'.format(key)] = len(stats[key])
+ stats['sum_{}'.format(key)] = sum(stats[key].values())
+ stats['min_{}'.format(key)] = min(stats[key].values())
+ stats['max_{}'.format(key)] = max(stats[key].values())
+ stats['avg_{}'.format(key)] = stats['sum_{}'.format(key)]/stats['cnt_{}'.format(key)]
+ stats['list_{}'.format(key)] = list(sorted(stats[key].keys()))
+ return stats
def truncate_evaluations(stats, top_threshold = 20):
"""
@@ -229,9 +246,7 @@ def evaluate_events(events, stats = None):
stats = _counter_inc(stats, ST_SKEY_DETECTORSWS, key)
# Calculate secondary statistics (cnt, min, max, sum, avg).
- for key in LIST_CALCSTAT_KEYS:
- if key in stats:
- stats = _calculate_substats(stats, key)
+ stats = calculate_secondary_stats(stats)
return stats
@@ -287,9 +302,8 @@ def aggregate_stats(stats, interval, dt_from, result = None):
for subkey, subval in stats[key].items():
result = _counter_inc(result, key, subkey, subval)
- for key in LIST_CALCSTAT_KEYS:
- if key in result:
- result = _calculate_substats(result, key)
+ # Calculate secondary statistics (cnt, min, max, sum, avg).
+ stats = calculate_secondary_stats(stats)
return result
@@ -388,23 +402,6 @@ def _counter_inc(stats, stat, key, increment = 1):
stats[stat][str(key)] = stats[stat].get(str(key), 0) + increment
return stats
-def _calculate_substats(stats, key):
- """
- Calculate substatistics for given key within given statistics.
-
- :param dict stats: Structure containing event statistics.
- :param str key: Designated key within structure.
- :return: Enriched event statistics.
- :rtype: dict
- """
- stats['cnt_{}'.format(key)] = len(stats[key])
- stats['sum_{}'.format(key)] = sum(stats[key].values())
- stats['min_{}'.format(key)] = min(stats[key].values())
- stats['max_{}'.format(key)] = max(stats[key].values())
- stats['avg_{}'.format(key)] = stats['sum_{}'.format(key)]/stats['cnt_{}'.format(key)]
- stats['list_{}'.format(key)] = list(sorted(stats[key].keys()))
- return stats
-
def _make_toplist(stats, dict_key, top_threshold):
"""
Produce only toplist of given statistical keys.
diff --git a/scripts/sqldb-migrate-data.py b/scripts/sqldb-migrate-data.py
index b59eeab03e170403b6584308329d826f693230d8..a70245117357ea1ea6a4ce5d2730b1298f236494 100644
--- a/scripts/sqldb-migrate-data.py
+++ b/scripts/sqldb-migrate-data.py
@@ -225,6 +225,8 @@ print("* found total of {:,d} 'user' objects in MongoDB".format(TOTAL_COUNT))
ITEMCOUNTER = 0
for rawitem in MONGO_ITEMS.find().sort('_id', 1):
try:
+ if 'orggroups' not in rawitem or not rawitem['orggroups']:
+ rawitem['orggroups'] = []
mongoitem = User(rawitem)
if ARGS.verbose:
print(mongoitem)
@@ -247,8 +249,8 @@ for rawitem in MONGO_ITEMS.find().sort('_id', 1):
#print("{}: {}".format(sqlitem, err))
SQLSTORAGE.session.rollback()
- except:
- print("[ FAIL ] Unable to convert user record '{}'".format(rawitem['_id']))
+ except Exception as err:
+ print("[ FAIL ] Unable to convert user record '{}': '{}'".format(rawitem['_id'], str(err)))
print("* converted total of {:,d} 'user' objects, {:,d} failure(s)".format(ITEMCOUNTER, TOTAL_COUNT - ITEMCOUNTER))
print("[ DONE ] Conversion: 'users'")
@@ -415,12 +417,14 @@ if not ARGS.skip_reports:
sqlrep.dt_to = mongorep['ts_to']
sqlrep.delta = delta.total_seconds()
+ sqlrep.evcount_rep = mongorep.get('cnt_alerts', mongorep.get('cnt_all', None))
sqlrep.evcount_all = mongorep.get('cnt_all', None)
- sqlrep.evcount_flt = mongorep.get('cnt_flt', None)
- sqlrep.evcount_flt_blk = mongorep.get('cnt_flt_blk', None)
- sqlrep.evcount_thr = mongorep.get('cnt_thr', None)
- sqlrep.evcount_thr_blk = mongorep.get('cnt_thr_blk', None)
- sqlrep.evcount_rlp = mongorep.get('cnt_rlp', None)
+ sqlrep.evcount_new = mongorep.get('cnt_all', None)
+ sqlrep.evcount_flt = mongorep.get('cnt_flt', 0)
+ sqlrep.evcount_flt_blk = mongorep.get('cnt_flt_blk', 0)
+ sqlrep.evcount_thr = mongorep.get('cnt_thr', 0)
+ sqlrep.evcount_thr_blk = mongorep.get('cnt_thr_blk', 0)
+ sqlrep.evcount_rlp = mongorep.get('cnt_rlp', 0)
sqlrep.mail_to = mongorep.get('mail_to', None)
if sqlrep.mail_to: