diff --git a/.pylintrc-lib b/.pylintrc-lib
index e8533358353c10fc33a34488feec4a7531490756..be3ced13197a221eff8f9de1f20c4b1da54b9503 100644
--- a/.pylintrc-lib
+++ b/.pylintrc-lib
@@ -291,7 +291,7 @@ ignore-on-opaque-inference=yes
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
+ignored-classes=optparse.Values,thread._local,_thread._local,jinja2.environment.Environment
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
diff --git a/Makefile b/Makefile
index eac4f3767ad2104b2c309cae62b9c929ba4a92fa..57bc99a062a4b0ed8bc297511fafb7d34b1cab8b 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,8 @@ DIR_BIN = bin
DIR_DOC = doc
DIR_LIB = lib
+DIR_TEMPLATES_INFORMANT = conf/templates/informant
+
DIR_LIB_HAWAT = lib/hawat
BIN_FILES := $(wildcard bin/mentat-*.py)
@@ -156,6 +158,21 @@ pybabel-compile: FORCE
@echo "\n${GREEN}*** Compiling translations for Hawat user interface ***${NC}\n"
@cd $(DIR_LIB_HAWAT) && pybabel-python3 compile -d translations
+mpybabel-extract: FORCE
+ @echo "\n${GREEN}*** Extracting babel translation for Mentat reports ***${NC}\n"
+ cd $(DIR_TEMPLATES_INFORMANT) && pybabel-python3 extract -F babel.cfg -o messages.pot -k lazy_gettext .
+
+mpybabel-update: FORCE
+ @echo "\n${GREEN}*** Updating translations for Mentat reports ***${NC}\n"
+ @cd $(DIR_TEMPLATES_INFORMANT) && pybabel-python3 update -i messages.pot -d translations -l cs
+
+mpybabel-pull: mpybabel-extract mpybabel-update
+
+mpybabel-compile: FORCE
+ @echo "\n${GREEN}*** Compiling translations for Mentat reports ***${NC}\n"
+ @cd $(DIR_TEMPLATES_INFORMANT) && pybabel-python3 compile -d translations
+
+
#-------------------------------------------------------------------------------
diff --git a/bin/mentat-informant.py b/bin/mentat-informant.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf57e1cc57fc78f3039e4b0f584cf4567bbbcd0f
--- /dev/null
+++ b/bin/mentat-informant.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license.
+#-------------------------------------------------------------------------------
+
+
+"""
+This Mentat module is a script providing periodical informational reports about
+overall system performance and statistics.
+
+To view built-in help please execute the application with ``--help`` command line
+option::
+
+ mentat-informant.py --help
+
+To view local documentation please use ``pydoc3``::
+
+ pydoc3 mentat.module.informant
+
+This script is implemented using the :py:mod:`pyzenkit.zenscript` framework and
+so it provides all of its core features. See the documentation for more in-depth
+details.
+
+
+License
+^^^^^^^
+
+Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+Use of this source is governed by the MIT license.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+#
+# Custom libraries.
+#
+from mentat.module.informant import MentatInformantScript
+
+#
+# Execute the script.
+#
+if __name__ == "__main__":
+
+ MentatInformantScript().run()
diff --git a/conf/core/reporting.json.conf b/conf/core/reporting.json.conf
index a12b75663ad6a695ee5be6266ed5bcb0860f110b..9a6625c08a9e5985484fba1ec8dde614aac396a0 100644
--- a/conf/core/reporting.json.conf
+++ b/conf/core/reporting.json.conf
@@ -4,5 +4,12 @@
#
"__core__reporting": {
"reports_dir": "/var/mentat/reports/reporter-ng"
+ },
+ #
+ # Definitions of core configurations for overview event reporting.
+ #
+ "__core__informant": {
+ "reports_dir": "/var/mentat/reports/informant",
+ "templates_dir": "/etc/mentat/templates/informant"
}
}
diff --git a/conf/mentat-informant.py.conf b/conf/mentat-informant.py.conf
new file mode 100644
index 0000000000000000000000000000000000000000..d7ea24ea7d6739f60ba8c674155418d9a90c121c
--- /dev/null
+++ b/conf/mentat-informant.py.conf
@@ -0,0 +1,185 @@
+#-------------------------------------------------------------------------------
+#
+# CONFIGURATION FILE FOR MENTAT-INFORMANT.PY MODULE
+#
+#-------------------------------------------------------------------------------
+
+{
+ #---------------------------------------------------------------------------
+ # Custom script configurations
+ #---------------------------------------------------------------------------
+
+ # Locale for generating reports.
+ # default: "en"
+ # type: string
+ #
+ #"locale": "en",
+
+ # Identifier of a template for generating reports.
+ # default: "default"
+ # type: string
+ #
+ #"template_id": "default",
+
+ #---------------------------------------------------------------------------
+ # Common mailer plugin configurations
+ #---------------------------------------------------------------------------
+
+ # Email subject.
+ # default: null
+ # type: string
+ #
+ #"mail_subject": "root",
+
+ # Source email address.
+ # default: null
+ # type: string
+ #
+ "mail_from": "root",
+
+ # Target email address(es).
+ # default: null
+ # type: string or list of strings
+ #
+ "mail_to": "root",
+
+ # Reply to email address(es).
+ # default: null
+ # type: string or list of strings
+ #
+ "mail_reply_to": "root",
+
+ # Target copy email address(es).
+ # default: null
+ # type: string or list of strings
+ #
+ #"mail_cc": "root",
+
+ # Target blind copy email address(es).
+ # default: null
+ # type: string or list of strings
+ #
+ #"mail_bcc": "root",
+
+ #---------------------------------------------------------------------------
+ # Common script configurations
+ #---------------------------------------------------------------------------
+
+ # Operational mode: regular script execution (flag).
+ # default: false
+ # type: boolean
+ #
+ #"regular": false,
+
+ # Operational mode: manual script execution from shell (flag).
+ # default: false
+ # type: boolean
+ #
+ #"shell": false,
+
+ # Name of the script command to be executed.
+ # default: "generate"
+ # type: string
+ #
+ #"command": "generate",
+
+ # Execution interval. This value should correspond with related cron script.
+ # default: daily
+ # type: string
+ #
+ "interval": "daily",
+
+ # Round-up time interval threshols to interval size (flag).
+ # default: false
+ # type: boolean
+ #
+ #"adjust_thresholds": false,
+
+ # Upper time interval threshold.
+ # default: time.time()
+ # type: float
+ #
+ #"time_high": null,
+
+ #---------------------------------------------------------------------------
+ # Common application configurations
+ #---------------------------------------------------------------------------
+
+ # Run in quiet mode (flag).
+ # default: false
+ # type: boolean
+ #
+ #"quiet": false,
+
+ # Application output verbosity.
+ # default: 0
+ # type: int
+ #
+ #"verbosity": 0,
+
+ # Name of the log file.
+ # default: "/var/mentat/log/mentat-dbmngr.py.log"
+ # type: string
+ #
+ #"log_file": "/var/mentat/log/mentat-dbmngr.py.log",
+
+ # Logging level ['debug', 'info', 'warning', 'error', 'critical'].
+ # default: "info"
+ # type: string
+ #
+ #"log_level": "info",
+
+ # Name of the runlog directory.
+ # default: "/var/mentat/run/mentat-dbmngr.py"
+ # type: string
+ #
+ #"runlog_dir": "/var/mentat/run/mentat-dbmngr.py",
+
+ # Dump runlog to stdout when done processing (flag).
+ # default: false
+ # type: boolean
+ #
+ #"runlog_dump": false,
+
+ # Write runlog to logging service when done processing (flag)
+ # default: false
+ # type: boolean
+ #
+ #"runlog_log": false,
+
+ # Name of the persistent state file.
+ # default: "/var/mentat/run/mentat-dbmngr.py.pstate"
+ # type: string
+ #
+ #"pstate_file": "/var/mentat/run/mentat-dbmngr.py.pstate",
+
+ # Dump persistent state to stdout when done processing (flag).
+ # default: false
+ # type: boolean
+ #
+ #"pstate_dump": false,
+
+ # Write persistent state to logging service when done processing (flag).
+ # default: false
+ # type: boolean
+ #
+ #"pstate_log": false,
+
+ # Name of the quick action to be performed.
+ # default: null
+ # type: string
+ #
+ #"action": null,
+
+ # Name/uid of the user.
+ # default: null
+ # type: string or integer
+ #
+ "user": "mentat",
+
+ # Name/gid of the group.
+ # default: null
+ # type: string or integer
+ #
+ "group": "mentat"
+}
diff --git a/conf/templates/informant/babel.cfg b/conf/templates/informant/babel.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..caf6dbe695a8e735184f79674b3d362b6fe2fbf6
--- /dev/null
+++ b/conf/templates/informant/babel.cfg
@@ -0,0 +1,2 @@
+[jinja2: **.j2]
+extensions=jinja2.ext.autoescape,jinja2.ext.with_
diff --git a/conf/templates/informant/default.html.j2 b/conf/templates/informant/default.html.j2
new file mode 100644
index 0000000000000000000000000000000000000000..14536248aec2544a16be146a9bc3c43b3c2c38e2
--- /dev/null
+++ b/conf/templates/informant/default.html.j2
@@ -0,0 +1,107 @@
+{#-
+
+ Template for generating informational summary reports about Mentat system performance.
+
+ Author: Jan Mach <jan.mach@cesnet.cz>
+
+#}
+{%- macro stats_table(chstats, chsection) %}
+ {%- if chsection in chstats %}
+ {%- set chdata = chstats[chsection] %}
+ {%- set chkeys = chstats['list_' + chsection] %}
+ {%- set chsum = chstats['sum_' + chsection] %}
+ {%- set tdstyle = 'border-width: 1px; padding-left: 3px; padding-right: 3px;' %}
+<table border="1" cellspacing="0" style="border-width: 1px; border-collapse: collapse;">
+ {%- for chitemkey in chkeys %}
+ <tr>
+ <td style="{{ tdstyle }} text-align: right;">
+ {{ '{:>2d}'.format(loop.index) }}
+ </td>
+ <td style="{{ tdstyle }}">
+ {{ '{:60s}'.format(format_section_key(chsection, chitemkey)) }}
+ </td>
+ <td style="{{ tdstyle }} text-align: right;">
+ {{ '{:>17s}'.format(format_decimal(chdata.get(chitemkey, 0))) }}
+ </td>
+ <td style="{{ tdstyle }} text-align: right;">
+ {{ '{:>6.2f} %'.format(chdata.get(chitemkey, 0)/chsum*100) }}
+ </td>
+ </tr>
+ {%- endfor %}
+ <tr>
+ <td colspan="2" style="{{ tdstyle }} ">
+ </td>
+ <td style="{{ tdstyle }} text-align: right;">
+ {{ '{:>17s}'.format(format_decimal(chsum)) }}
+ </td>
+ <td style="{{ tdstyle }} text-align: right;">
+ 100.00 %
+ </td>
+ </tr>
+</table>
+ {%- else %}
+<h4>{{ gettext('No data') }}</h4>
+ {%- endif %}
+{%- endmacro %}
+<h1>{{ title | upper }}</h1>
+
+<hr>
+
+<p>
+ <strong>{{ gettext('Total number of events:') }}</strong> {{ format_decimal(stats_events['count']) }}<br>
+ <strong>{{ gettext('Summary for time period:') }}</strong> {{ format_datetime(dt_l) }} - {{ format_datetime(dt_h) }} ({{ format_timedelta(dt_h - dt_l ) }})<br>
+</p>
+
+<h2>{{ gettext('Contents') }}</h2>
+<ol type="A">
+ <li>
+ <a href="#section-A">{{ gettext('Overall event statistics') }}</a>
+ <ol type="1">
+{%- for chsection in ('ips', 'categories', 'detectors', 'analyzers', 'abuses', 'asns', 'countries') %}
+ <li>
+ <a href="#section-A-{{ loop.index }}">{{ gettext('Number of events per') }} <strong>{{ format_section_name(chsection) }}</strong></a>
+ </li>
+{%- endfor %}
+ </ol>
+ </li>
+</ol>
+
+<h2>
+ <a name="section-A">A. {{ gettext('Overall event statistics') }}</a>
+</h2>
+
+{%- for chsection in ('ips', 'categories', 'detectors', 'analyzers', 'abuses', 'asns', 'countries', 'bogus') %}
+<h3>
+ <a name="section-A-{{ loop.index }}">{{ 'A.{:d}'.format(loop.index) }} {{ gettext('Number of events per') }} <strong>{{ format_section_name(chsection) }}</strong></a>
+</h3>
+{{
+ stats_table(
+ stats_events['stats_overall'],
+ chsection
+ )
+}}
+{% endfor %}
+
+<hr>
+
+<p>
+ {{ gettext('Have a nice day') }}<br>
+
+ <br>
+
+ {{ gettext('Mentat System') }} (<a href="{{ gettext('https://mentat.cesnet.cz/en/index') }}">{{ gettext('https://mentat.cesnet.cz/en/index') }}</a>)<br>
+ {{ gettext('CESNET-CERTS Computer Security Team') }} <<a href="mailto:certs@cesnet.cz">certs@cesnet.cz</a>> (<a href="{{ gettext('https://csirt.cesnet.cz/en/index') }}">{{ gettext('https://csirt.cesnet.cz/en/index') }}</a>)<br>
+ {{ gettext('CESNET, z.s.p.o.') }} (<a href="{{ gettext('http://www.ces.net/') }}">{{ gettext('http://www.ces.net/') }}</a>)<br>
+</p>
+
+<br>
+
+<p>
+ ---
+</p>
+
+<br>
+
+<p>
+ <strong>{{ gettext('Generated at:') }}</strong> {{ format_datetime(dt_c) }}
+</p>
diff --git a/conf/templates/informant/default.txt.j2 b/conf/templates/informant/default.txt.j2
new file mode 100644
index 0000000000000000000000000000000000000000..dba756275ffa568d100923ca5f76ec2f32175ba0
--- /dev/null
+++ b/conf/templates/informant/default.txt.j2
@@ -0,0 +1,61 @@
+{#-
+
+ Template for generating informational summary reports about Mentat system performance.
+
+ Author: Jan Mach <jan.mach@cesnet.cz>
+
+#}
+{%- macro stats_table(chstats, chsection) %}
+ {%- if chsection in chstats %}
+ {%- set chdata = chstats[chsection] %}
+ {%- set chkeys = chstats['list_' + chsection] %}
+ {%- set chsum = chstats['sum_' + chsection] %}
+{{ '{}'.format('-' * 90) }}
+ {%- for chitemkey in chkeys %}
+{{ '{:>2d}'.format(loop.index) }} {{ '{:60s}'.format(format_section_key(chsection, chitemkey)) }} {{ '{:>17s}'.format(format_decimal(chdata.get(chitemkey, 0))) }} {{ '{:>6.2f} %'.format(chdata.get(chitemkey, 0)/chsum*100) }}
+ {%- endfor %}
+{{ '{}'.format('-' * 90) }}
+{{ '{}'.format(' ' * 63) }} {{ '{:>17s}'.format(format_decimal(chsum)) }} 100.00 %
+ {%- else %}
+{{ gettext(' - No data') }}
+ {%- endif %}
+{%- endmacro %}
+{{ '{}'.format('*' * 90) }}
+
+{{ title | upper | center(90) }}
+
+{{ '{}'.format('*' * 90) }}
+
+{{ gettext('Total number of events:') }} {{ format_decimal(stats_events['count']) }}
+{{ gettext('Summary for time period:') }} {{ format_datetime(dt_l) }} - {{ format_datetime(dt_h) }} ({{ format_timedelta(dt_h - dt_l ) }})
+
+{{ gettext('CONTENTS') }}
+{{ '{}'.format('=' * 90) }}
+A. {{ gettext('Overall event statistics') }}
+{%- for chsection in ('ips', 'categories', 'detectors', 'analyzers', 'abuses', 'asns', 'countries') %}
+{{ 'A.{:d}'.format(loop.index) }} {{ gettext('Number of events per') }} '{{ format_section_name(chsection) }}':
+{%- endfor %}
+
+
+A. {{ gettext('Overall event statistics') }}
+{{ '{}'.format('=' * 90) }}
+
+{%- for chsection in ('ips', 'categories', 'detectors', 'analyzers', 'abuses', 'asns', 'countries', 'bogus') %}
+{{ '[A.{:d}]'.format(loop.index) }} {{ gettext('Number of events per') }} '{{ format_section_name(chsection) }}'
+{{
+ stats_table(
+ stats_events['stats_overall'],
+ chsection
+ )
+}}
+{% endfor %}
+
+{{ gettext('Have a nice day') }}
+
+{{ gettext('Mentat System') }} ({{ gettext('https://mentat.cesnet.cz/en/index') }})
+{{ gettext('CESNET-CERTS Computer Security Team') }} <certs@cesnet.cz> ({{ gettext('https://csirt.cesnet.cz/en/index') }})
+{{ gettext('CESNET, z.s.p.o.') }} ({{ gettext('http://www.ces.net/') }})
+
+---
+
+{{ gettext('Generated at:') }} {{ format_datetime(dt_c) }}
diff --git a/conf/templates/informant/messages.pot b/conf/templates/informant/messages.pot
new file mode 100644
index 0000000000000000000000000000000000000000..9d2e1194b355b7bc432e12895c5b5851ffd4815d
--- /dev/null
+++ b/conf/templates/informant/messages.pot
@@ -0,0 +1,83 @@
+# Translations template for PROJECT.
+# Copyright (C) 2018 ORGANIZATION
+# This file is distributed under the same license as the PROJECT project.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2018.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: PROJECT VERSION\n"
+"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
+"POT-Creation-Date: 2018-02-26 15:43+0100\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.5.1\n"
+
+#: default.html.j2:43
+msgid "No data"
+msgstr ""
+
+#: default.html.j2:51 default.txt.j2:29
+msgid "Total number of events:"
+msgstr ""
+
+#: default.html.j2:52 default.txt.j2:30
+msgid "Summary for time period:"
+msgstr ""
+
+#: default.html.j2:55
+msgid "Contents"
+msgstr ""
+
+#: default.html.j2:58 default.html.j2:70 default.txt.j2:34 default.txt.j2:40
+msgid "Overall event statistics"
+msgstr ""
+
+#: default.html.j2:62 default.html.j2:75 default.txt.j2:36 default.txt.j2:44
+msgid "Number of events per"
+msgstr ""
+
+#: default.html.j2:88 default.txt.j2:53
+msgid "Have a nice day"
+msgstr ""
+
+#: default.html.j2:92 default.txt.j2:55
+msgid "Mentat System"
+msgstr ""
+
+#: default.html.j2:92 default.txt.j2:55
+msgid "https://mentat.cesnet.cz/en/index"
+msgstr ""
+
+#: default.html.j2:93 default.txt.j2:56
+msgid "CESNET-CERTS Computer Security Team"
+msgstr ""
+
+#: default.html.j2:93 default.txt.j2:56
+msgid "https://csirt.cesnet.cz/en/index"
+msgstr ""
+
+#: default.html.j2:94 default.txt.j2:57
+msgid "CESNET, z.s.p.o."
+msgstr ""
+
+#: default.html.j2:94 default.txt.j2:57
+msgid "http://www.ces.net/"
+msgstr ""
+
+#: default.html.j2:106 default.txt.j2:61
+msgid "Generated at:"
+msgstr ""
+
+#: default.txt.j2:20
+msgid " - No data"
+msgstr ""
+
+#: default.txt.j2:32
+msgid "CONTENTS"
+msgstr ""
+
diff --git a/conf/templates/informant/translations/cs/LC_MESSAGES/messages.po b/conf/templates/informant/translations/cs/LC_MESSAGES/messages.po
new file mode 100644
index 0000000000000000000000000000000000000000..26b0a8a223f4417a965f8dfbf2e3d96c32172cef
--- /dev/null
+++ b/conf/templates/informant/translations/cs/LC_MESSAGES/messages.po
@@ -0,0 +1,109 @@
+
+msgid ""
+msgstr ""
+"Project-Id-Version: PROJECT VERSION\n"
+"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
+"POT-Creation-Date: 2018-02-26 15:43+0100\n"
+"PO-Revision-Date: 2017-09-22 15:33+0200\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language: cs\n"
+"Language-Team: cs <LL@li.org>\n"
+"Plural-Forms: nplurals=3; plural=((n==1) ? 0 : (n>=2 && n<=4) ? 1 : 2)\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=utf-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.5.1\n"
+
+#: default.html.j2:43
+msgid "No data"
+msgstr "Žádná data"
+
+#: default.html.j2:51 default.txt.j2:29
+msgid "Total number of events:"
+msgstr "CelkovĂ˝ poÄŤet událostĂ:"
+
+#: default.html.j2:52 default.txt.j2:30
+msgid "Summary for time period:"
+msgstr "Souhrn za ÄŤasovĂ© obdobĂ:"
+
+#: default.html.j2:55
+msgid "Contents"
+msgstr "Obsah"
+
+#: default.html.j2:58 default.html.j2:70 default.txt.j2:34 default.txt.j2:40
+msgid "Overall event statistics"
+msgstr "CelkovĂ© statistiky událostĂ"
+
+#: default.html.j2:62 default.html.j2:75 default.txt.j2:36 default.txt.j2:44
+msgid "Number of events per"
+msgstr "Počet událostà dle"
+
+#: default.html.j2:88 default.txt.j2:53
+msgid "Have a nice day"
+msgstr "Hezký den přeje"
+
+#: default.html.j2:92 default.txt.j2:55
+msgid "Mentat System"
+msgstr "Systém Mentat"
+
+#: default.html.j2:92 default.txt.j2:55
+msgid "https://mentat.cesnet.cz/en/index"
+msgstr "https://mentat.cesnet.cz/cs/index"
+
+#: default.html.j2:93 default.txt.j2:56
+msgid "CESNET-CERTS Computer Security Team"
+msgstr "Bezpečnostnà tým CESNET-CERTS"
+
+#: default.html.j2:93 default.txt.j2:56
+msgid "https://csirt.cesnet.cz/en/index"
+msgstr "https://csirt.cesnet.cz/cs/index"
+
+#: default.html.j2:94 default.txt.j2:57
+msgid "CESNET, z.s.p.o."
+msgstr "CESNET, z.s.p.o."
+
+#: default.html.j2:94 default.txt.j2:57
+msgid "http://www.ces.net/"
+msgstr "http://www.cesnet.cz/"
+
+#: default.html.j2:106 default.txt.j2:61
+msgid "Generated at:"
+msgstr "Vygenerováno v:"
+
+#: default.txt.j2:20
+msgid " - No data"
+msgstr "- Žádná data"
+
+#: default.txt.j2:32
+msgid "CONTENTS"
+msgstr "OBSAH"
+
+msgid "Periodical statistical summary report"
+msgstr "PravidelnĂ˝ statistickĂ˝ souhrnnĂ˝ report"
+
+msgid "__REST__"
+msgstr "__OSTATNĂŤ__"
+
+msgid "__unknown__"
+msgstr "__neznámé__"
+
+msgid "source IP"
+msgstr "zdrojové IP"
+
+msgid "category"
+msgstr "kategorie"
+
+msgid "detector"
+msgstr "detektoru"
+
+msgid "analyzer"
+msgstr "analyzátoru"
+
+msgid "abuse group"
+msgstr "abuse skupiny"
+
+msgid "autonomous system"
+msgstr "autonomnĂho systĂ©mu"
+
+msgid "source country"
+msgstr "zemÄ› pĹŻvodu"
diff --git a/deploy/mentat/ctrl/postinst b/deploy/mentat/ctrl/postinst
index 48af2bc0925c929a4cb73d8051eed1fcb7170eaf..232502fbc133461ade81f08cb1d8a2c8149b809c 100755
--- a/deploy/mentat/ctrl/postinst
+++ b/deploy/mentat/ctrl/postinst
@@ -23,6 +23,7 @@ mentat_dir_list=(
/var/mentat/log
/var/mentat/reports
/var/mentat/reports/reporter-ng
+ /var/mentat/reports/informant
/var/mentat/reports/statistician
/var/mentat/rrds
/var/mentat/run
diff --git a/doc/sphinx/_doclib/bin.rst b/doc/sphinx/_doclib/bin.rst
index 7b2c7cbe7e812dc6c5b2b8b1d1ab92d625b9b19d..4d500eebdf4af215a5d3858dcd5256366f494e6f 100644
--- a/doc/sphinx/_doclib/bin.rst
+++ b/doc/sphinx/_doclib/bin.rst
@@ -37,6 +37,7 @@ terminal by user.
bin_mentat-ideagen
bin_mentat-netmngr
bin_mentat-statistician
+ bin_mentat-informant
User interfaces
diff --git a/doc/sphinx/_doclib/bin_mentat-informant.rst b/doc/sphinx/_doclib/bin_mentat-informant.rst
new file mode 100644
index 0000000000000000000000000000000000000000..e9ff57019c18e9c0e8b7b25cf5fcadc846635347
--- /dev/null
+++ b/doc/sphinx/_doclib/bin_mentat-informant.rst
@@ -0,0 +1,9 @@
+.. _section-bin-mentat-informant:
+
+mentat-informant.py
+================================================================================
+
+.. automodule:: mentat.module.informant
+
+
+.. include:: _inc_bin.help.fetcher.rst
diff --git a/lib/mentat/emails/__init__.py b/lib/mentat/emails/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/mentat/emails/base.py b/lib/mentat/emails/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..72a1b4cefa12414cf57c1b0ba7a242f59f9c5c44
--- /dev/null
+++ b/lib/mentat/emails/base.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This module contains base class for various types of email messages and reports,
+that are generated and sent by Mentat system.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+REPORT_TYPE_HEADER = 'X-Mentat-Report-Type'
+"""Custom email header: Report type"""
+REPORT_ID_HEADER = 'X-Mentat-Report-Id'
+"""Custom email header: Report identifier"""
+
+
+class BaseEmail():
+ """
+ Base class for various types of email messages and reports.
+ """
+ report_type = None
+
+ def __init__(self, headers, **kwargs):
+ self.ident = None
+ self.email = self._get_container()
+ self._set_headers(headers)
+ self._set_content(headers, **kwargs)
+
+ def _get_container(self):
+ """
+ This method must return valid :py:mod:`email.mime` container to hold email
+ contents (for example :py:class:`email.mime.text.MIMEText` for simple emails
+ or :py:class:`email.mime.multipart.MIMEMultipart` for more complex ones).
+ Returned object will be populated with email contents.
+ """
+ raise NotImplementedError()
+
+ def _set_headers(self, headers):
+ """
+ Set appropriate email headers within the email container acquired by
+ :py:func:`mentat.emails.base.BaseEmail._get_container`. The ``headers``
+ parameter may contain following header configurations:
+
+ * subject (``str``)
+ * from (``str``)
+ * to (``str`` or ``list of str``)
+ * cc (``str`` or ``list of str``)
+ * bcc (``str`` or ``list of str``)
+ * reply_to (``str`` or ``list of str``)
+ * report_type (``str``)
+ * report_id (``str``)
+
+ :param dict headers: Dictionary containing header configurations.
+ """
+ if 'subject' in headers:
+ self.email['Subject'] = str(headers['subject'])
+ if 'from' in headers:
+ self.email['From'] = str(headers['from'])
+ for item in (('to', 'To'), ('cc', 'Cc'), ('bcc', 'Bcc'), ('reply_to', 'Reply-To')):
+ if item[0] in headers:
+ if isinstance(headers[item[0]], list):
+ self.email[item[1]] = ','.join(headers[item[0]])
+ else:
+ self.email[item[1]] = str(headers[item[0]])
+ if self.report_type:
+ self.email[REPORT_TYPE_HEADER] = self.report_type
+ if 'report_type' in headers:
+ self.email[REPORT_TYPE_HEADER] = str(headers['report_type'])
+ if 'report_id' in headers:
+ self.email[REPORT_ID_HEADER] = str(headers['report_id'])
+ self.ident = str(headers['report_id'])
+
+ def _set_content(self, headers, **kwargs):
+ """
+ This method should actualy construct the email object.
+ """
+ raise NotImplementedError()
+
+ def as_string(self):
+ """
+ Return email as string ready to be passed to sendmail library.
+ """
+ return self.email.as_string()
diff --git a/lib/mentat/emails/informant.py b/lib/mentat/emails/informant.py
new file mode 100644
index 0000000000000000000000000000000000000000..13a39ad6a8ebe3a4810ce38ff848b8fe943eee50
--- /dev/null
+++ b/lib/mentat/emails/informant.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This module contains implementation of email reports send by the *mentat-informant.py*
+component. It is based on :py:class:`mentat.emails.base.BaseEmail`.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import json
+import datetime
+
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+#from email.mime.application import MIMEApplication
+
+#
+# Custom libraries.
+#
+from mentat.emails.base import BaseEmail
+
+
+REPORT_TYPE_INFORMANT = 'overview-overall'
+"""Module contant for informant report types."""
+
+
+def json_default(val):
+ """
+ Helper function for JSON serialization of non basic data types.
+ """
+ if isinstance(val, datetime.datetime):
+ return val.isoformat()
+ return str(val)
+
+class ReportEmail(BaseEmail):
+ """
+ Implementation of email reports send by the *mentat-informant.py* Mentat component.
+ """
+ report_type = REPORT_TYPE_INFORMANT
+
+ def _get_container(self):
+ return MIMEMultipart('mixed')
+
+ def _set_content(self, headers, text_plain, text_html, data):
+ # Record the MIME types of both parts - text/plain and text/html.
+ msg_text = MIMEMultipart('alternative')
+ msg_text_part1 = MIMEText(text_plain, 'plain')
+ msg_text_part2 = MIMEText(text_html, 'html')
+
+ # Attach parts into message container. According to RFC 2046, the last
+ # part of a multipart message, in this case the HTML variant, is best
+ # and therefore preferred.
+ msg_text.attach(msg_text_part1)
+ msg_text.attach(msg_text_part2)
+
+ # Attach the text content to the message container.
+ self.email.attach(msg_text)
+
+ # Construct attachment file name.
+ filename = 'report-overview-overall.json.txt'
+ if self.ident:
+ filename = 'report-overview-overall-{}.json.txt'.format(self.ident)
+
+ # Attach data, ehm...attachment.
+ #msg_attach = MIMEApplication(str(args['idea']), 'json')
+ #msg_attach = MIMEApplication(json.dumps(args['idea'], default=args['idea'].json_default, sort_keys=True, indent=4), 'json')
+ msg_attach = MIMEText(
+ json.dumps(
+ data,
+ default = json_default,
+ sort_keys = True,
+ indent = 4
+ ),
+ 'plain'
+ )
+ msg_attach.add_header(
+ 'Content-Disposition',
+ 'attachment',
+ filename = filename
+ )
+ self.email.attach(msg_attach)
diff --git a/lib/mentat/emails/test_informant.py b/lib/mentat/emails/test_informant.py
new file mode 100644
index 0000000000000000000000000000000000000000..4762cd865aecc307cc13f1374006a497d4e040e9
--- /dev/null
+++ b/lib/mentat/emails/test_informant.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+Unit test module for testing the :py:mod:`mentat.emails.informant` module.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import unittest
+from unittest.mock import Mock, MagicMock, call
+
+import datetime
+
+import mentat.emails.informant
+
+class TestReportEmail(unittest.TestCase):
+ """
+ Unit test class for testing the :py:class:`mentat.emails.informant.ReportEmail` class.
+ """
+
+ def test_basic(self):
+ """
+ Perform the basic operativity tests.
+ """
+ msg = mentat.emails.informant.ReportEmail(
+ headers = {
+ 'subject': 'Test email',
+ 'from': 'root',
+ 'to': 'user',
+ 'cc': ['admin', 'manager'],
+ 'bcc': 'spy'
+ },
+ text_plain = "TEXT PLAIN",
+ text_html = "<h1>TEXT HTML</h1>",
+ data = {'a': 1, 'b': [datetime.datetime(2017,1,1,12,0,0)]}
+ )
+ print(msg.as_string())
+
+
+#-------------------------------------------------------------------------------
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/lib/mentat/module/informant.py b/lib/mentat/module/informant.py
new file mode 100644
index 0000000000000000000000000000000000000000..d892b681e80f29d83892da02b71cd29d89e15130
--- /dev/null
+++ b/lib/mentat/module/informant.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This Mentat module is a script providing periodical informational reports about
+overall performance of Mentat system.
+
+This script is implemented using the :py:mod:`pyzenkit.zenscript` framework and
+so it provides all of its core features. See the documentation for more in-depth
+details.
+
+It is further based on :py:mod:`mentat.script.fetcher` module, which provides
+database fetching and message post-processing capabilities.
+
+
+Usage examples
+--------------
+
+.. code-block:: shell
+
+ # Display help message and exit.
+ mentat-informant.py --help
+
+ # Run in debug mode (enable output of debugging information to terminal).
+ mentat-informant.py --debug
+
+ # Run with increased logging level.
+ mentat-informant.py --log-level debug
+
+
+Available script commands
+-------------------------
+
+``report`` (*default*)
+ Generate report containing overall Mentat system performance statistics
+ within configured time interval thresholds.
+
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+#
+# Custom libraries
+#
+import mentat.script.fetcher
+import mentat.plugin.app.mailer
+import mentat.const
+import mentat.reports.overview
+
+
+class MentatInformantScript(mentat.script.fetcher.FetcherScript):
+ """
+ Implementation of Mentat module (script) providing periodical statistical
+ overview for message processing performance analysis.
+ """
+
+ #
+ # Class constants.
+ #
+
+ # List of configuration keys.
+ CORECFG_INFORMANT = '__core__informant'
+ CONFIG_REPORTS_DIR = 'reports_dir'
+ CONFIG_TEMPLATES_DIR = 'templates_dir'
+ CONFIG_TEMPLATE_ID = 'template_id'
+ CONFIG_LOCALE = 'locale'
+
+ def __init__(self):
+ """
+ Initialize statistician script object. This method overrides the base
+ implementation in :py:func:`mentat.script.fetcher.FetcherScript.__init__`
+ and it aims to even more simplify the script object creation by providing
+ configuration values for parent contructor.
+ """
+ # Declare private attributes.
+ self.sqlservice = None
+ self.mailerservice = None
+ self.reporter = None
+
+ super().__init__(
+
+ description = 'mentat-informant.py - Mentat system overall performance statistics',
+
+ #
+ # Configure required script paths.
+ #
+ path_bin = '/usr/local/bin',
+ path_cfg = '/etc/mentat',
+ path_log = '/var/mentat/log',
+ path_run = '/var/mentat/run',
+ path_tmp = '/tmp',
+
+ #
+ # Override default configurations.
+ #
+ default_config_dir = '/etc/mentat/core',
+
+ #
+ # Load additional application-level plugins.
+ #
+ plugins = [
+ mentat.plugin.app.mailer.MailerPlugin()
+ ]
+ )
+
+ def _sub_stage_init(self, **kwargs):
+ """
+ **SUBCLASS HOOK**: Perform additional custom initialization actions.
+
+ This method is called from the main constructor in :py:func:`pyzenkit.baseapp.BaseApp.__init__`
+ as a part of the **__init__** stage of application`s life cycle.
+
+ :param kwargs: Various additional parameters passed down from constructor.
+ """
+ # Override default 'interval' value.
+ self.config[self.CONFIG_INTERVAL] = 'daily'
+
+ # Override default 'adjust_thresholds' value.
+ self.config[self.CONFIG_ADJUST_THRESHOLDS] = True
+
+ def _init_argparser(self, **kwargs):
+ """
+ Initialize script command line argument parser. This method overrides the
+ base implementation in :py:func:`pyzenkit.zenscript.ZenScript._init_argparser`
+ and it must return valid :py:class:`argparse.ArgumentParser` object. It
+ appends additional command line options custom for this script object.
+
+ This method is called from the main constructor in :py:func:`pyzenkit.baseapp.BaseApp.__init__`
+ as a part of the **__init__** stage of application`s life cycle.
+
+ :param kwargs: Various additional parameters passed down from object constructor.
+ :return: Valid argument parser object.
+ :rtype: argparse.ArgumentParser
+ """
+ argparser = super()._init_argparser(**kwargs)
+
+ #
+ # Create and populate options group for custom script arguments.
+ #
+ arggroup_script = argparser.add_argument_group('custom script arguments')
+
+ arggroup_script.add_argument('--locale', type = str, default = None, help = 'locale for generating reports')
+ arggroup_script.add_argument('--template-id', type = str, default = None, help = 'identifier of a template for generating reports')
+
+ return argparser
+
+ def _init_config(self, cfgs, **kwargs):
+ """
+ Initialize default script configurations. This method overrides the base
+ implementation in :py:func:`pyzenkit.zenscript.ZenScript._init_config`
+ and it appends additional configurations via ``cfgs`` parameter.
+
+ This method is called from the main constructor in :py:func:`pyzenkit.baseapp.BaseApp.__init__`
+ as a part of the **__init__** stage of application`s life cycle.
+
+ :param list cfgs: Additional set of configurations.
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Default configuration structure.
+ :rtype: dict
+ """
+ cfgs = (
+ (self.CONFIG_LOCALE, 'en'),
+ (self.CONFIG_TEMPLATE_ID, 'default'),
+ ) + cfgs
+ return super()._init_config(cfgs, **kwargs)
+
+ def _sub_stage_setup(self):
+ """
+ **SUBCLASS HOOK**: Perform additional custom setup actions.
+
+ This method is called from the main setup method :py:func:`pyzenkit.baseapp.BaseApp._stage_setup`
+ as a part of the **setup** stage of application`s life cycle.
+ """
+ self.reporter = mentat.reports.overview.OverviewReporter(
+ self.config[self.CORECFG_INFORMANT][self.CONFIG_REPORTS_DIR],
+ self.config[self.CORECFG_INFORMANT][self.CONFIG_TEMPLATES_DIR],
+ self.c(self.CONFIG_LOCALE)
+ )
+
+
+ #---------------------------------------------------------------------------
+
+
+ def get_default_command(self):
+ """
+ Return the name of the default script command. This command will be executed
+ in case it is not explicitly selected either by command line option, or
+ by configuration file directive.
+
+ :return: Name of the default command.
+ :rtype: str
+ """
+ return 'report'
+
+ def cbk_command_report(self):
+ """
+ Implementation of the **report** command (*default*).
+
+ Calculate statistics for messages stored into database within configured
+ time interval thresholds.
+ """
+ (time_l, time_h) = self.calculate_interval_thresholds(
+ time_high = self.c(self.CONFIG_TIME_HIGH),
+ interval = self.c(self.CONFIG_INTERVAL),
+ adjust = self.c(self.CONFIG_REGULAR)
+ )
+ self.logger.info("Lower summary report calculation time interval threshold: %s (%s)", time_l.isoformat(), time_l.timestamp())
+ self.logger.info("Upper summary report calculation time interval threshold: %s (%s)", time_h.isoformat(), time_h.timestamp())
+ self.logger.info("Using template '%s' to generate informational summary report.", self.c(self.CONFIG_TEMPLATE_ID))
+
+ (result, email) = self.reporter.report(
+ time_h,
+ time_l,
+ self.sqlservice.session,
+ self.c(self.CONFIG_TEMPLATE_ID),
+ self.mailerservice
+ )
+
+ return result
diff --git a/lib/mentat/plugin/app/mailer.py b/lib/mentat/plugin/app/mailer.py
new file mode 100644
index 0000000000000000000000000000000000000000..1835343bb5457fe6cfdea337d5c5f05f7f61a747
--- /dev/null
+++ b/lib/mentat/plugin/app/mailer.py
@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This Mentat application plugin provides functions for sending emails. It
+is usable both in script and daemon modules.
+
+
+Example usage
+^^^^^^^^^^^^^
+
+Using the plugin like in following way::
+
+ mentat.plugin.app.eventstorage.MailerPlugin()
+
+That will yield following results:
+
+* The application object will have a ``mailerservice`` attribute containing reference to
+ mailer service represented by :py:class:`mentat.plugin.app.MailerPlugin`.
+
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import weakref
+from subprocess import Popen, PIPE
+
+#
+# Custom libraries.
+#
+import pyzenkit.baseapp
+
+
+class MailerPlugin(pyzenkit.baseapp.ZenAppPlugin):
+ """
+ Implementation of Mentat application plugin providing functions for sending
+ emails.
+ """
+
+ #
+ # Class constants.
+ #
+
+ # List of configuration keys.
+ CONFIG_MAIL_SUBJECT = 'mail_subject'
+ CONFIG_MAIL_FROM = 'mail_from'
+ CONFIG_MAIL_TO = 'mail_to'
+ CONFIG_MAIL_CC = 'mail_cc'
+ CONFIG_MAIL_BCC = 'mail_bcc'
+ CONFIG_MAIL_REPLY_TO = 'mail_reply_to'
+
+ def __init__(self, settings = None):
+ """
+ Initialize internal plugin configuration.
+ """
+ self.get_application = None
+
+ if settings is None:
+ settings = {}
+ self.settings = settings
+
+
+ #---------------------------------------------------------------------------
+
+
+ def init_argparser(self, app, argparser, **kwargs):
+ """
+ Callback to be called during argparser initialization phase.
+ """
+ #
+ # Create and populate options group for custom script arguments.
+ #
+ arggroup_plugin = argparser.add_argument_group('mailer plugin arguments')
+
+ arggroup_plugin.add_argument('--mail-subject', type = str, default = None, help = 'email subject')
+ arggroup_plugin.add_argument('--mail-from', type = str, default = None, help = 'source email address')
+ arggroup_plugin.add_argument('--mail-to', type = str, default = None, help = 'target email address (repeatable)', action='append')
+ arggroup_plugin.add_argument('--mail-cc', type = str, default = None, help = 'target copy email address (repeatable)', action='append')
+ arggroup_plugin.add_argument('--mail-bcc', type = str, default = None, help = 'target blind copy email address (repeatable)', action='append')
+ arggroup_plugin.add_argument('--mail-reply-to', type = str, default = None, help = 'reply to email address (repeatable)', action='append')
+
+ return argparser
+
+ def init_config(self, app, config, **kwargs):
+ """
+ Callback to be called during default configuration initialization phase.
+ """
+ config.update({
+ self.CONFIG_MAIL_SUBJECT: None,
+ self.CONFIG_MAIL_FROM: None,
+ self.CONFIG_MAIL_TO: None,
+ self.CONFIG_MAIL_CC: None,
+ self.CONFIG_MAIL_BCC: None,
+ self.CONFIG_MAIL_REPLY_TO: None
+ })
+ return config
+
+ def configure(self, app):
+ """
+ Configure application. This method will be called from :py:func:`pyzenkit.baseapp.BaseApp._configure_plugins`
+ and it further updates current application configurations.
+
+ This method is part of the **setup** stage of application`s life cycle.
+
+ :param app: Reference to the parent application.
+ """
+ pass
+
+ def setup(self, app):
+ """
+ Configure application. This method will be called from :py:func:`pyzenkit.baseapp.BaseApp._stage_setup_plugins`
+ and it further updates current application configurations.
+
+ This method is part of the **setup** stage of application`s life cycle.
+
+ :param app: Reference to the parent application.
+ """
+ self.get_application = weakref.ref(app)
+ app.mailerservice = self
+ app.logger.debug("[STATUS] Set up mailer service plugin.")
+
+ #---------------------------------------------------------------------------
+
+ @staticmethod
+ def mail_sendmail(email):
+ """
+ Send given email directly through local sendmail binary. This method is
+ usefull for fire and forget scenarios.
+
+ :param mentat.emails.base.BaseEmail email: Email object.
+ """
+ with Popen(["/usr/sbin/sendmail", "-t", "-oi"], stdin=PIPE) as proc:
+ proc.communicate(bytes(email.as_string(), 'UTF-8'))
+
+ #---------------------------------------------------------------------------
+
+ def email_send(self, email_class, email_headers, email_params):
+ """
+ Create email according to given class, headers and parameters and send
+ it via :py:func:`mail_sendmail` method.
+
+ :param class email_class: Email class to be instantinated.
+ :param dict email_headers: Eplicitly specified email headers.
+ :param dict email_params: Additional email class constructor parameters.
+ :return: Constructed email object.
+ :rtype: mentat.emails.base.BaseEmail
+ """
+ for item in (
+ (self.CONFIG_MAIL_SUBJECT, 'subject'),
+ (self.CONFIG_MAIL_FROM, 'from'),
+ (self.CONFIG_MAIL_TO, 'to'),
+ (self.CONFIG_MAIL_CC, 'cc'),
+ (self.CONFIG_MAIL_BCC, 'bcc'),
+ (self.CONFIG_MAIL_REPLY_TO, 'reply_to')):
+ if self.get_application().c(item[0]):
+ email_headers[item[1]] = self.get_application().c(item[0])
+
+ self.get_application().logger.info("Sending email: '{}'".format(str(email_headers)))
+ msg = email_class(email_headers, **email_params)
+ self.mail_sendmail(msg)
+
+ return msg
diff --git a/lib/mentat/reports/__init__.py b/lib/mentat/reports/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/lib/mentat/reports/base.py b/lib/mentat/reports/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf097553f8ce3b7b16ca23ac022a72f86705af49
--- /dev/null
+++ b/lib/mentat/reports/base.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This module contains base implementation of generic reporting class. It provides
+common methods and utilities usefull for all kinds of reporters like:
+
+* Jinja2 template rendering
+* report localization
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import os
+import gettext
+import jinja2
+from babel.numbers import format_decimal
+from babel.dates import format_datetime, format_timedelta
+
+
+class BaseReporter:
+ """
+ Implementation of base reporting class providing following features and
+ services:
+
+ * ``self.renderer`` - Jinja2 template rendering
+ * ``self.translator`` - report localization
+ """
+
+ def __init__(self, reports_dir, templates_dir, locale):
+ """
+ :param str reports_dir: Name of the directory containing generated report files.
+ :param str templates_dir: Name of the directory containing report template files.
+ :param str locale: Locale for report rendering.
+ """
+ self.reports_dir = reports_dir
+ self.templates_dir = templates_dir
+ self.locale = locale
+
+ self.translator = self._setup_translator(
+ os.path.join(self.templates_dir, 'translations'),
+ self.locale
+ )
+ self.renderer = self._setup_renderer(
+ templates_dir,
+ self.translator,
+ self.locale
+ )
+
+ def _setup_translator(self, translations_dir, locale):
+ """
+ Setup translator by loading message catalog for given locale.
+
+ :param str translations_dir: Directory containing message catalogs for various locales.
+ :param str locale: Locale name.
+ """
+ try:
+ return gettext.translation('messages', translations_dir, [locale])
+ except OSError:
+ return gettext.NullTranslations()
+
+ def _setup_renderer(self, templates_dir, translator, locale):
+ """
+ Setup template renderer and registertranslator and localization tools.
+
+ :param str templates_dir: Directory containing Jinja2 templates.
+ :param gettext.Translation translator: Translator for template translation.
+ :param str locale: Locale for template localization.
+ """
+ renderer = jinja2.Environment(
+ loader = jinja2.FileSystemLoader(templates_dir),
+ extensions = ['jinja2.ext.i18n']
+ )
+
+ renderer.install_gettext_translations(translator)
+
+ renderer.globals['format_decimal'] = lambda x: format_decimal(x, locale = locale)
+ renderer.globals['format_datetime'] = lambda x: format_datetime(x, locale = locale)
+ renderer.globals['format_timedelta'] = lambda x: format_timedelta(x, locale = locale)
+
+ return renderer
diff --git a/lib/mentat/reports/overview.py b/lib/mentat/reports/overview.py
new file mode 100644
index 0000000000000000000000000000000000000000..7bc0518724c93ed24a7a3daee5e987f3971a62bb
--- /dev/null
+++ b/lib/mentat/reports/overview.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+Library for generating statistical overview reports.
+
+The implementation is based on :py:class:`mentat.reports.base.BaseReporter`.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import os
+import json
+import datetime
+
+#
+# Custom libraries
+#
+import mentat.stats.idea
+from mentat.datatype.sqldb import EventStatisticsModel
+from mentat.emails.informant import ReportEmail
+from mentat.reports.base import BaseReporter
+
+
+REPORT_SECTION_NAMES = {
+ 'ips': 'source IP',
+ 'categories': 'category',
+ 'detectors': 'detector',
+ 'abuses': 'abuse group',
+ 'analyzers': 'analyzer',
+ 'asns': 'autonomous system',
+ 'countries': 'source country'
+}
+"""Dictionary for translating section identifiers to section names."""
+
+def format_section_name(section, translator):
+ """
+ Helper function for section name translation and/or formating.
+ """
+ return translator.gettext(REPORT_SECTION_NAMES.get(section, section))
+
+def format_section_key(section, key, translator):
+ """
+ Helper function for section key translation and/or formating.
+ """
+ if key in ('__REST__', '__unknown__'):
+ return translator.gettext(key)
+ if section == 'asns':
+ return 'AS {}'.format(key)
+ return key
+
+def json_default(val):
+ """
+ Helper function for JSON serialization of non basic data types.
+ """
+ if isinstance(val, datetime.datetime):
+ return val.isoformat()
+ return str(val)
+
+
+class OverviewReporter(BaseReporter):
+ """
+ Implementation of reporting class providing Mentat system overview reports.
+ """
+
+ def _setup_renderer(self, templates_dir, translator, locale):
+ """
+ Overloaded implementation of base :py:func:`mentat.reports.base.BaseReporter._setup_renderer`
+ method. This implementation adds additional global functions into Jinja2
+ templating environment, that are necessary for report rendering.
+ """
+ renderer = super()._setup_renderer(templates_dir, translator, locale)
+
+ renderer.globals['format_section_key'] = lambda x,y: format_section_key(x,y, translator = translator)
+ renderer.globals['format_section_name'] = lambda x: format_section_name(x, translator = translator)
+
+ return renderer
+
+
+ #---------------------------------------------------------------------------
+
+
+ def report(self, time_h, time_l, sqlalchemy_session, template_id, mailer):
+ """
+ Perform reporting for given time interval.
+
+ :param datetime.datetime time_h: Upper threshold of reporting time interval.
+ :param datetime.datetime time_l: Lower threshold of reporting time interval.
+ :param sqlalchemy_session: SQLAlchemy session to be used to retrieve the raw data.
+ :param str template_id: Identifier of the template to use to render the report.
+ :param mailer: Mailer to use to mail the report.
+ :return: Tuple containing generated report as dict and email text.
+ :rtype: tuple
+ """
+ result = {}
+
+ result['ts_from_s'] = time_l.isoformat()
+ result['ts_to_s'] = time_h.isoformat()
+ result['ts_from'] = int(time_l.timestamp())
+ result['ts_to'] = int(time_h.timestamp())
+ result['interval'] = '{}_{}'.format(result['ts_from_s'], result['ts_to_s'])
+
+ # Fetch data.
+ stats_query = sqlalchemy_session.query(EventStatisticsModel)
+ stats_query = stats_query.filter(EventStatisticsModel.dt_from >= time_l)
+ stats_query = stats_query.filter(EventStatisticsModel.dt_to <= time_h)
+ stats_query = stats_query.order_by(EventStatisticsModel.interval)
+ stats_raw = stats_query.all()
+ sqlalchemy_session.commit()
+
+ # Process data.
+ stats = mentat.stats.idea.truncate_evaluations(
+ mentat.stats.idea.aggregate_stat_groups(stats_raw)
+ )
+ result['stats_events'] = stats
+
+ # Render reports.
+ template_txt = self.renderer.get_template('{}.txt.j2'.format(template_id))
+ template_html = self.renderer.get_template('{}.html.j2'.format(template_id))
+ report_txt = template_txt.render(
+ title = self.translator.gettext("Periodical statistical summary report"),
+ dt_c = datetime.datetime.utcnow(),
+ dt_h = time_h,
+ dt_l = time_l,
+ stats_events = stats
+ )
+ report_html = template_html.render(
+ title = self.translator.gettext("Periodical statistical summary report"),
+ dt_c = datetime.datetime.utcnow(),
+ dt_h = time_h,
+ dt_l = time_l,
+ stats_events = stats
+ )
+
+ # Send emails.
+ report_msg_headers = {
+ 'subject': self.translator.gettext("Periodical statistical summary report"),
+ 'report_id': result['interval']
+ }
+ report_msg_params = {
+ 'text_plain': report_txt,
+ 'text_html': report_html,
+ 'data': stats
+ }
+ email = mailer.email_send(ReportEmail, report_msg_headers, report_msg_params)
+
+ # Save report to disk.
+ self._save_to_json_file(
+ stats,
+ 'report-overview-overall-{}.json'.format(result['interval'])
+ )
+
+ return result, email
+
+
+ #---------------------------------------------------------------------------
+
+
+ def _save_to_json_file(self, data, filename):
+ """
+ Helper method for saving given data into given JSON file.
+
+ :param dict data: Data to be serialized.
+ :param str filename: Name of the target JSON file.
+ """
+ data_json = json.dumps(
+ data,
+ default = json_default,
+ sort_keys = True,
+ indent = 4
+ )
+
+ filepath = os.path.join(self.reports_dir, filename)
+
+ imf = open(filepath, 'w')
+ imf.write(data_json)
+ imf.close()
+
+ return filepath