Merge branch 'devel' into release

conf/mentat-cleanup.py.conf

@@ -10,10 +10,10 @@
     #---------------------------------------------------------------------------
 
     # Path to database files (for disk usage measurements).
-    #   default: "/var/lib/postgresql/12/main"
+    #   default: "/var/lib/postgresql"
     #   type:    string
     #
-    "db_path": "/var/lib/postgresql/12/main",
+    "db_path": "/var/lib/postgresql",
 
     # Perform simulation, do not remove anything (flag).
     #   default: false

conf/mentat-inspector.py.conf

@@ -36,7 +36,7 @@
         },
         {
             "name": "Assign class - attempt-login-ssh",
-            "rule": "Category in ['Attempt.Login'] and (Target.Proto in ['ssh'] or Source.Proto in ['ssh'] or Target.Port in [22])",
+            "rule": "Category in ['Attempt.Login', 'Intrusion.UserCompromise'] and (Target.Proto in ['ssh'] or Source.Proto in ['ssh'] or Target.Port in [22])",
             "actions": [
                 {"action": "tag", "args": {"path": "_Mentat.EventClass", "value": "attempt-login-ssh", "overwrite": false} },
                 {"action": "tag", "args": {"path": "_Mentat.EventSeverity", "value": "medium", "overwrite": false} }
@@ -124,7 +124,7 @@
         },
         {
             "name": "Assign class - vulnerable-config-netbios",
-            "rule": "Category in ['Vulnerable.Config'] and (Source.Proto in ['netbios-ns'] or Source.Port in [137])",
+            "rule": "Category in ['Vulnerable.Config'] and (Source.Proto in ['netbios-ns', 'netbios-dgm', 'netbios-ssn'] or Source.Port in [137, 138, 139])",
             "actions": [
                 {"action": "tag", "args": {"path": "_Mentat.EventClass", "value": "vulnerable-config-netbios", "overwrite": false} },
                 {"action": "tag", "args": {"path": "_Mentat.EventSeverity", "value": "medium", "overwrite": false} }

doc/sphinx/_doclib/upgrading.rst

@@ -43,13 +43,15 @@ to latest version:
     $ aptitude update
     $ aptitude upgrade
 
+    # Do not forget to review changes in configuration files/structure and
+    # adapt accordingly.
+
     # Step 3: To be safe activate the Python virtual environment for Mentat system:
     $ . /var/mentat/venv/bin/activate
 
     # Step 4: Make sure your database schema is up to date. Please be aware, that
     # these operations may need a lot of time to complete depending on the size
     # of your database:
-    (venv) $ time mentat-dbmngr.py --command init
     (venv) $ time hawat-cli db upgrade
     (venv) $ time /etc/mentat/scripts/sqldb-migrate.sh upgrade head
     (venv) $ time /etc/mentat/scripts/sqldb-optimize.sh
@@ -98,6 +100,21 @@ to quickly turn maintenance mode ON/OFF. Upgrading steps are then much more simp
     $ /etc/mentat/scripts/maintenance-mode.sh off
 
 
+.. _section-upgrading-mentat-2-8:
+
+Upgrading to Mentat 2.8
+--------------------------------------------------------------------------------
+
+* Rulesets for two inspectors have been merged in to one instance. That
+  concerns mentat-inspector.py, mentat-inspector-b.py.conf and
+  mentat-controller.py.conf. If you use vanilla configuration from .deb,
+  package manager will replace the old configuration files with new ones
+  automatically. If you used different installation method or made some
+  modifications into these configuration files, you are encouraged to review
+  the changes and decide whether you want to stay with split configuration
+  or also proceed with the merge.
+
+
 .. _section-upgrading-postgresql-10:
 
 Upgrading PostgreSQL from 10.x to 11.x
@@ -209,6 +226,8 @@ from version ``11.x`` to ``12.x``.
     It can take a lot of time depending on the size of the current database,
     because the data files need to be converted to new format.
 
+    Upgrade to the latest version of Mentat prior to upgrading PostgreSQL.
+
 .. code-block:: shell
 
     # Launch tmux or screen.
@@ -302,15 +321,6 @@ from version ``11.x`` to ``12.x``.
     $ systemctl restart apache2
 
 
-After these steps it is necessary to update following configuration files:
-
-``/etc/mentat/mentat-cleanup.py.conf``
-    Change configuration ``db_path`` to point to correct filesystem location. In default
-    Debian installations it should look something like this:
-
-    ``"db_path": "/var/lib/postgresql/12/main",``
-
-
 .. _section-upgrading-postgresql-12:
 
 Upgrading PostgreSQL from 12.x to 13.x
@@ -325,6 +335,8 @@ from version ``12.x`` to ``13.x``.
     It can take a lot of time depending on the size of the current database,
     because the data files need to be converted to new format.
 
+    Upgrade to the latest version of Mentat prior to upgrading PostgreSQL.
+
 .. code-block:: shell
 
     # Launch tmux or screen.
@@ -353,8 +365,8 @@ from version ``12.x`` to ``13.x``.
     $ psql mentat_events
     DROP INDEX events_detecttime_idx;
     DROP INDEX events_combined_idx;
-    DROP INDEX events_cesnet_storagetime_idx;
-    DROP INDEX events_cesnet_eventseverity_idx;
+    DROP INDEX events_storagetime_idx;
+    DROP INDEX events_eventseverity_idx;
     ALTER TABLE events_json DROP CONSTRAINT events_json_id_fkey;
     ALTER TABLE events_json DROP CONSTRAINT events_json_pkey;
     ALTER TABLE events DROP CONSTRAINT events_pkey;
@@ -422,9 +434,9 @@ from version ``12.x`` to ``13.x``.
     ALTER TABLE events_json ADD PRIMARY KEY (id);
     ALTER TABLE events_json ADD FOREIGN KEY (id) REFERENCES events(id) ON DELETE CASCADE;
     CREATE INDEX IF NOT EXISTS events_detecttime_idx ON events USING BTREE (detecttime);
-    CREATE INDEX IF NOT EXISTS events_cesnet_storagetime_idx ON events USING BTREE (cesnet_storagetime);
-    CREATE INDEX IF NOT EXISTS events_cesnet_eventseverity_idx ON events USING BTREE (cesnet_eventseverity) WHERE cesnet_eventseverity IS NOT NULL;
-    CREATE INDEX IF NOT EXISTS events_combined_idx ON events USING GIN (category, node_name, protocol, source_port, target_port, source_type, target_type, node_type, cesnet_resolvedabuses, cesnet_inspectionerrors);
+    CREATE INDEX IF NOT EXISTS events_storagetime_idx ON events USING BTREE (storagetime);
+    CREATE INDEX IF NOT EXISTS events_eventseverity_idx ON events USING BTREE (eventseverity) WHERE eventseverity IS NOT NULL;
+    CREATE INDEX IF NOT EXISTS events_combined_idx ON events USING GIN (category, node_name, protocol, source_port, target_port, source_type, target_type, node_type, resolvedabuses, inspectionerrors);
     CREATE INDEX IF NOT EXISTS events_ip_aggr_idx ON events USING GIST (source_ip_aggr_ip4, target_ip_aggr_ip4, source_ip_aggr_ip6, target_ip_aggr_ip6);
     CHECKPOINT;
 
@@ -442,15 +454,6 @@ from version ``12.x`` to ``13.x``.
     $ systemctl restart apache2
 
 
-After these steps it is necessary to update following configuration files:
-
-``/etc/mentat/mentat-cleanup.py.conf``
-    Change configuration ``db_path`` to point to correct filesystem location. In default
-    Debian installations it should look something like this:
-
-    ``"db_path": "/var/lib/postgresql/13/main",``
-
-
 .. _section-upgrading-geoip:
 
 Upgrading to authenticated version of GeoIP service

lib/hawat/blueprints/settings_reporting/forms.py

@@ -98,20 +98,6 @@ class BaseSettingsReportingForm(vial.forms.BaseItemForm):
         ],
         filters = [lambda x: x or None]
     )
-    attachments = wtforms.SelectField(
-        lazy_gettext('Report attachments:'),
-        validators = [
-            wtforms.validators.Optional(),
-        ],
-        choices = [
-            ('',                                 lazy_gettext('<< system default >>')),
-            (mentat.const.REPORTING_ATTACH_JSON, lazy_gettext('json')),
-            (mentat.const.REPORTING_ATTACH_CSV,  lazy_gettext('csv')),
-            (mentat.const.REPORTING_ATTACH_ALL,  lazy_gettext('all')),
-            (mentat.const.REPORTING_ATTACH_NONE, lazy_gettext('none'))
-        ],
-        filters = [lambda x: x or None]
-    )
     emails = vial.forms.CommaListField(
         lazy_gettext('Target emails:'),
         validators = [
@@ -145,19 +131,6 @@ class BaseSettingsReportingForm(vial.forms.BaseItemForm):
         filters = [vial.forms.str_to_bool_with_none],
         coerce = vial.forms.str_to_bool_with_none
     )
-    compress = vial.forms.RadioFieldWithNone(
-        lazy_gettext('Attachment compression:'),
-        validators = [
-            wtforms.validators.Optional(),
-        ],
-        choices = [
-            (None,  lazy_gettext('System default')),
-            (True,  lazy_gettext('Enabled')),
-            (False, lazy_gettext('Disabled'))
-        ],
-        filters = [vial.forms.str_to_bool_with_none],
-        coerce = vial.forms.str_to_bool_with_none
-    )
     template = wtforms.StringField(
         lazy_gettext('Template:'),
         validators = [
@@ -181,18 +154,6 @@ class BaseSettingsReportingForm(vial.forms.BaseItemForm):
         choices = [('', lazy_gettext('<< system default >>'))] + list(zip(pytz.common_timezones, pytz.common_timezones)),
         filters = [lambda x: x or None]
     )
-    max_attachment_size = vial.forms.SelectFieldWithNone(
-        lazy_gettext('Attachment size limit:'),
-        validators = [
-            wtforms.validators.Optional(),
-        ],
-        coerce = vial.forms.str_to_int_with_none,
-        choices = [
-            (None, lazy_gettext('<< system default >>')),
-            (0, lazy_gettext('<< no limit >>'))
-        ] + list(reversed(sorted(mentat.const.REPORT_ATTACHMENT_SIZES.items(), key = lambda x: x[0]))),
-        default = mentat.const.DFLT_REPORTING_MAXATTACHSIZE
-    )
     timing = vial.forms.RadioFieldWithNone(
         lazy_gettext('Reporting timing:'),
         validators = [

lib/mentat/module/cleanup.py

@@ -51,7 +51,7 @@ Custom command line options
 ``--db-path dir-name``
     Path to database files (for disk usage measurements).
 
-    *Type:* ``string``, *default:* ``/var/lib/postgresql/10/main``
+    *Type:* ``string``, *default:* ``/var/lib/postgresql``
 
 ``--simulate``
     Perform simulation, do not remove anything (*flag*).
@@ -235,7 +235,7 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
         :rtype: dict
         """
         cfgs = (
-            (self.CONFIG_DB_PATH,  '/var/lib/postgresql/11/main'),
+            (self.CONFIG_DB_PATH,  '/var/lib/postgresql'),
             (self.CONFIG_SIMULATE, False),
             (self.CONFIG_EVENTS,   []),
             (self.CONFIG_TABLES,   []),

packaging/debian/rules

@@ -26,6 +26,8 @@ binary:
 	cp -r ../etc/init.d debian/mentat-ng/etc/
 	cp -r ../etc/systemd debian/mentat-ng/etc/
 
+	if [ -L 'debian/mentat-ng/etc/mentat/migrations-events/.env' ]; then rm -f 'debian/mentat-ng/etc/mentat/migrations-events/.env'; fi
+
 ifeq ($(BUILD_SUITE),production)
 	echo "mentat-ng" > debian/mentat-ng/etc/mentat/install.pip
 else ifeq ($(BUILD_SUITE),release)