Merge branch 'devel' into release

.gitignore

 !.gitignore
 .directory
+.vagrant
 .coverage
 *~
 *.aux
@@ -83,3 +84,9 @@ parsetab.py
 *.substvars
 /packaging/debian/mentat-ng/
 /packaging/debian/files
+
+# Ignore custom stuff.
+/conf/install.pip
+/chroot/
+/data/
+/var/

MANIFEST.in

 include LICENSE.txt
 include README.rst
 include lib/hawat/Makefile*
+recursive-include lib/mentat/templates_email *
 recursive-include lib/hawat/static *
 recursive-include lib/hawat/templates *
 recursive-include lib/hawat/migrations *

Makefile

@@ -35,7 +35,7 @@ DIR_EVENT_CLASSES = $(shell pwd)/conf/event_classes
 BIN_FILES := $(wildcard bin/mentat-*.py)
 LIB_FILES := $(shell find $(DIR_LIB) -name '*.py' | grep -v 'test_')
 
-VENV_PYTHON ?= python3.7
+VENV_PYTHON ?= python3
 VENV_PATH   ?= venv
 
 PYTHON      = python3
@@ -49,7 +49,7 @@ CURRENT_DIR = $(shell pwd)
 DEV_SERVER      = localhost
 DEV_PORT        = 5000
 DEV_MAIL_SERVER = localhost
-DEV_MAIL_PORT   = 8025
+DEV_MAIL_PORT   = 1025
 
 #
 # Include common makefile configurations.
@@ -118,11 +118,9 @@ help:
 	@echo "  * $(GREEN)deps-postgresql$(NC): configure required PostgreSQL user accounts and databases"
 	@echo "  * $(GREEN)deps-translations$(NC): compile all available translations"
 	@echo ""
-	@echo "  * $(GREEN)run-mentat-dev$(NC): run development version of Mentat system"
 	@echo "  * $(GREEN)run-webui-dev$(NC): run development web server with development configuration on $(DEV_SERVER):$(DEV_PORT)"
-	@echo "  * $(GREEN)run-mailserver-dev$(NC): run development mail server on $(DEV_MAIL_SERVER):$(DEV_MAIL_PORT)"
-	@echo ""
-	@echo "  * $(GREEN)ctrl-mentat-dev COMMAND=cmd$(NC): execute mentat-controller.py with given command $(FAINT)cmd$(NC)"
+	@echo "  * $(GREEN)run-mailserver-sendria$(NC): run development mail server Sendria on $(DEV_MAIL_SERVER):$(DEV_MAIL_PORT)"
+	@echo "  * $(GREEN)run-mailserver-python$(NC): run python-based development mail server on $(DEV_MAIL_SERVER):$(DEV_MAIL_PORT)"
 	@echo ""
 	@echo "  * $(GREEN)clean-pycs$(NC): clean up Python compiled files"
 	@echo "  * $(GREEN)clean-build-docs$(NC): clean up documentation build directories"
@@ -215,9 +213,13 @@ venv: FORCE
 #   - download IP geolocation databases
 #   - install the project in editable mode
 #
-develop: deps install-whl-dev build-webui
+develop: deps build-webui
+
+develop-vagrant: deps-vagrant build-webui
 
-deps: deps-prerequisites deps-lwchroot deps-python deps-python-dev deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
+deps: deps-prerequisites deps-lwchroot deps-python-dev deps-python deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
+
+deps-vagrant: deps-prerequisites deps-runenv deps-python-dev deps-python deps-editable deps-webui deps-geoip-vagrant deps-negistry-vagrant deps-postgresql translations-compile
 
 clean: clean-pycs clean-build-docs clean-build-python clean-build-debian
 
@@ -272,11 +274,15 @@ deps-prerequisites: FORCE
 	@echo ""
 
 #
-# Install project`s lightweight chroot.
+# Install project`s runtime environment.
 #
 deps-lwchroot: FORCE
 	@echo "\n$(GREEN)*** Creating local lightweight chroot subdirectory structure ***$(NC)\n"
-	@./scripts/lwchroot-init.sh
+	@./conf/scripts/lwchroot-init.sh
+
+deps-runenv: FORCE
+	@echo "\n$(GREEN)*** Creating local runtime environment subdirectory structure ***$(NC)\n"
+	@./conf/scripts/init-runenv.sh
 
 #
 # Install project`s Python dependencies using pip requirements file. The dependencies
@@ -357,12 +363,23 @@ deps-geoip: FORCE
 	@geoipupdate --verbose --database-directory $(shell realpath ./chroot/usr/share/GeoIP)
 	@echo ""
 
+deps-geoip-vagrant: FORCE
+	@echo "\n$(GREEN)*** Installing IP geolocation databases ***$(NC)\n"
+	@mkdir -p $(shell realpath ./data/geoip)
+	@geoipupdate --verbose --database-directory $(shell realpath ./data/geoip)
+	@echo ""
+
 #
 # Fetch and install Negistry whois database.
 #
 deps-negistry: FORCE
 	@echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n"
-	@./scripts/fetch-negistry.sh --stub --target=$(shell realpath ./chroot/var/mentat/whois-negistry.json)
+	@./conf/scripts/fetch-negistry.sh --stub --target=$(shell realpath ./chroot/var/mentat/whois-negistry.json)
+	@echo ""
+
+deps-negistry-vagrant: FORCE
+	@echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n"
+	@./conf/scripts/fetch-negistry.sh --stub --target=/var/mentat/whois-negistry.json
 	@echo ""
 
 #
@@ -370,27 +387,22 @@ deps-negistry: FORCE
 #
 deps-postgresql: FORCE
 	@echo "\n$(GREEN)*** Installing and configuring PostgreSQL database ***$(NC)\n"
-	@./scripts/sqldb-init.sh
+	@./conf/scripts/sqldb-init.sh
 	@echo ""
 
 
 #-------------------------------------------------------------------------------
 
 
-ctrl-mentat-dev:
-	#APP_ROOT_PATH=$(shell realpath ./chroot) mentat-controller.py --command $(COMMAND)
-	mentat-controller.py --command $(COMMAND)
-
-run-mentat-dev:
-	@echo "\n$(GREEN)*** Running development version of Mentat system ***$(NC)\n"
-	#APP_ROOT_PATH=$(shell realpath ./chroot) mentat-controller.py --command start
-	mentat-controller.py --command start
-
 run-webui-dev:
 	@echo "\n$(GREEN)*** Running development web server with development configuration on $(DEV_SERVER):$(DEV_PORT) ***$(NC)\n"
 	#APP_ROOT_PATH=$(shell realpath ./chroot) hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT)
 	HAWAT_CONFIG_FILE=$(shell realpath ./hawat.local.conf) hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT)
 
+run-webui-vagrant:
+	@echo "\n$(GREEN)*** Running development web server with development configuration on $(DEV_SERVER):$(DEV_PORT) ***$(NC)\n"
+	FLASK_ENV=development HAWAT_CONFIG_FILE=$(shell realpath ./conf/mentat-hawat-dev.py.conf) hawat-cli run --host "0.0.0.0" --port $(DEV_PORT)
+
 run-mailserver-dev:
 	@echo "\n$(GREEN)*** Running development mail server on $(DEV_MAIL_SERVER):$(DEV_MAIL_PORT) ***$(NC)\n"
 	$(PYTHON) -m smtpd -n -c DebuggingServer $(DEV_MAIL_SERVER):$(DEV_MAIL_PORT)
@@ -466,6 +478,49 @@ presentations: FORCE
 		fi; \
 	done
 
+#-------------------------------------------------------------------------------
+
+data-fetch-config: FORCE
+	@echo "\n$(GREEN)*** Fetching GeoIP configuration file ***$(NC)\n"
+	@mkdir -p ./data
+	@scp root@mentat-alt.cesnet.cz:/etc/GeoIP.conf ./data/
+	@echo ""
+
+data-fetch-dbsnapshot: FORCE
+	@echo "\n$(GREEN)*** Fetching latest production database dump ***$(NC)\n"
+	@rm -rf ./data/var
+	@mkdir -p ./data
+	@time scp root@mentat-alt.cesnet.cz:/var/mentat/devsnapshots/devsnapshot-latest.tar.gz ./data/
+	@ls -alh ./data/devsnapshot-latest.tar.gz
+	@cd ./data/ && tar -xzvf ./devsnapshot-latest.tar.gz
+	@rm -f ./data/devsnapshot-latest.tar.gz
+	@mv ./data/var/mentat/devsnapshots/devsnapshot ./data/devsnapshot
+	@rm -rf ./data/var
+	@echo ""
+
+data-import-dbsnapshot: FORCE
+	@echo "\n$(GREEN)*** Importing latest production database dump ***$(NC)\n"
+
+	@sudo -u postgres pg_restore --verbose --clean --if-exists --format=d --dbname=mentat_main ./data/devsnapshot/main
+	@sudo -u postgres psql mentat_main -c "COPY reports_events ($(cat /vagrant/data/devsnapshot/main/reports_events.cols)) FROM stdin;" < ./data/devsnapshot/chunks/reports_events.dat
+	@sudo -u postgres psql mentat_main -c "COPY statistics_events ($(cat /vagrant/data/devsnapshot/main/statistics_events.cols)) FROM stdin;" < ./data/devsnapshot/chunks/statistics_events.dat
+	@sudo -u postgres psql mentat_main -c "COPY changelogs_items ($(cat /vagrant/data/devsnapshot/main/changelogs_items.cols)) FROM stdin;" < ./data/devsnapshot/chunks/changelogs_items.dat
+
+	@sudo -u postgres psql mentat_events -c "COPY events ($(cat /vagrant/data/devsnapshot/events/events.cols)) FROM stdin;" < ./data/devsnapshot/events/events.dat
+	@sudo -u postgres psql mentat_events -c "COPY events_json ($(cat /vagrant/data/devsnapshot/events/events_json.cols)) FROM stdin;" < ./data/devsnapshot/events/events_json.dat
+
+	@echo ""
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as user_count, min(createtime) as from_date, max(createtime) as to_date from users;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as group_count, min(createtime) as from_date, max(createtime) as to_date from groups;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as filter_count, min(createtime) as from_date, max(createtime) as to_date from filters;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as network_count, min(createtime) as from_date, max(createtime) as to_date from networks;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as report_count, min(createtime) as from_date, max(createtime) as to_date from reports_events;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as statistics_count, min(createtime) as from_date, max(createtime) as to_date from statistics_events;"
+	@sudo -u postgres psql mentat_main -e -c "select count(*) as changelog_count, min(createtime) as from_date, max(createtime) as to_date from changelogs_items;"
+
+	@sudo -u postgres psql mentat_events -e -c "select count(*) as event_count, min(storagetime) as from_date, max(storagetime) as to_date from events;"
+	@sudo -u postgres psql mentat_events -e -c "select count(*) as event_json_count, min(storagetime) as from_date, max(storagetime) as to_date from events_json;"
+	@echo ""
 
 #-------------------------------------------------------------------------------
 

Vagrantfile

+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure('2') do |config|
+  config.vm.hostname = 'mentat-devel.local'
+  config.vm.box = 'debian/contrib-buster64'
+
+  config.vm.provider 'virtualbox' do |v|
+    v.memory = 2048
+    v.cpus = 2
+  end
+
+  # Host port 5000: Flask development server
+  config.vm.network 'forwarded_port', guest: 5000, host: 5000
+  # Host port 4443: Apache web server
+  config.vm.network 'forwarded_port', guest: 443, host: 4443
+  # Host port 1080: Sendria mailtrap service for development
+  config.vm.network 'forwarded_port', guest: 1080, host: 1080
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  config.vm.provision 'bootstrap_system', type: 'shell', inline: <<-SHELL
+    bash /vagrant/vagrantenv/provisioning/p01_system.sh
+  SHELL
+
+  config.vm.provision 'bootstrap_mentat', type: 'shell', inline: <<-SHELL
+    bash /vagrant/vagrantenv/provisioning/p02_mentat.sh
+  SHELL
+
+  config.vm.provision 'bootstrap_warden', type: 'shell', inline: <<-SHELL
+    bash /vagrant/vagrantenv/provisioning/p03_warden.sh
+  SHELL
+
+  # Automatically connect as 'mentat' user with ssh command.
+  # VAGRANT_COMMAND = ARGV[0]
+  # config.ssh.username = 'mentat' if VAGRANT_COMMAND == 'ssh'
+  config.ssh.extra_args = ['-t', 'cd /vagrant; bash --login']
+end

bin/mentat-hawat-dev.wsgi

@@ -5,6 +5,7 @@
 # Use of this source is governed by the MIT license, see LICENSE file.
 #-------------------------------------------------------------------------------
 
+
 from dotenv import load_dotenv
 
 import hawat
@@ -18,4 +19,5 @@ load_dotenv(dotenv_path = '/home/mentat/mentat-ng/.env')
 # capabilities or for purposes of testing. Please refer to the documentation
 # for more information.
 #
+
 application = hawat.create_app()

bin/mentat-hawat-vagrant.wsgi

+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+import hawat
+
+#
+# Use prepared factory function to create application instance. The factory
+# function takes number of arguments, that can be used to fine tune configuration
+# of the application. This is can be very usefull when extending applications`
+# capabilities or for purposes of testing. Please refer to the documentation
+# for more information.
+#
+application = hawat.create_app_full(
+    config_object = 'hawat.config.DevelopmentConfig',
+    config_file   = '/etc/mentat/mentat-hawat-dev.py.conf',
+    config_env    = 'FLASK_CONFIG_FILE'
+)

conf/apache/maintenance/index.html

@@ -3,7 +3,7 @@
     <head>
         <meta charset="utf-8">
             <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-        <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css" integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T" crossorigin="anonymous">
+        <link rel="stylesheet" href="bootstrap.min.css">
         <title>Mentat - Under maintenance</title>
     </head>
     <body>

conf/apache/site_mentat_vagrant.conf

+#-------------------------------------------------------------------------------
+# Example APACHE web server configuration file for HAWAT - development version
+#
+# This is an example configuration for authentication based on SSO service
+# Shibboleth. Please adjust the configuration file according to your needs.
+# In most cases customization of following directives should be enough to
+# get it working properly and quickly:
+#
+#   * ServerAdmin
+#   * ServerName
+#   * SSLCertificateFile
+#   * SSLCertificateKeyFile
+#   * SSLCertificateChainFile
+#
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+WSGIPythonHome /var/mentat/venv
+
+<IfModule mod_ssl.c>
+    <VirtualHost _default_:443>
+        ServerAdmin webmaster@mentat.organization.org
+        ServerName mentat.organization.org
+
+        <IfModule mod_headers.c>
+                # Enforce HTTPS protocol at all times.
+                Header always set Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+                # Deny access to content via HTML iframe.
+                Header always set X-Frame-Options DENY
+        </IfModule>
+
+        DocumentRoot /var/www
+
+        WSGIDaemonProcess hawat user=mentat group=mentat threads=5 python-home=/var/mentat/venv
+        WSGIScriptAlias /mentat /vagrant/bin/mentat-hawat-vagrant.wsgi
+
+        <Location />
+            Require all granted
+        </Location>
+
+        <Directory /vagrant/lib/hawat>
+            WSGIProcessGroup hawat
+            WSGIApplicationGroup %{GLOBAL}
+            Require all granted
+        </Directory>
+
+        <Location /mentat/auth_env/login>
+            AuthType Basic
+            AuthName "Mentat system login"
+            AuthUserFile /etc/mentat/apache/htpasswd.example
+            Require valid-user
+        </Location>
+        <Location /mentat/auth_env/register>
+            AuthType Basic
+            AuthName "Mentat system registration"
+            AuthUserFile /etc/mentat/apache/htpasswd.example
+            Require valid-user
+        </Location>
+
+        RewriteEngine on
+        # Rewrite the path when accessed without a trailing slash.
+        RewriteRule ^/mentat$ /mentat/ [R]
+        # Rewrite webserver root to mentat application.
+        RewriteRule ^/$ /mentat/ [R]
+
+
+        ErrorLog ${APACHE_LOG_DIR}/error.log
+
+        # Possible values: debug, info, notice, warn, error, crit, alert, emerg.
+        LogLevel warn
+
+        CustomLog ${APACHE_LOG_DIR}/ssl_access.log combined
+
+        SSLEngine on
+        #SSLOptions +FakeBasicAuth +ExportCertData +StrictRequire
+
+        # Using default self-signed server certificates
+        SSLCertificateFile  /etc/ssl/certs/ssl-cert-snakeoil.pem
+        SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key
+
+        # Using custom server certificates
+        #SSLCertificateFile      /etc/ssl/servercert/cert.pem
+        #SSLCertificateKeyFile   /etc/ssl/servercert/key.pem
+        #SSLCertificateChainFile /etc/ssl/servercert/chain.pem
+
+        # Enforce only strong SSL protocols. Generator as of August 2016. This
+        # tool is available at https://mozilla.github.io/server-side-tls/ssl-config-generator/
+        # Resource: https://httpd.apache.org/docs/trunk/ssl/ssl_howto.html#onlystrong
+        SSLProtocol         all -SSLv3 -TLSv1 -TLSv1.1
+        SSLCipherSuite      ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256
+        SSLHonorCipherOrder on
+        SSLCompression      off
+        SSLSessionTickets   off
+
+        <FilesMatch "\.(cgi|shtml|phtml|php)$">
+                SSLOptions +StdEnvVars
+        </FilesMatch>
+        <Directory /usr/lib/cgi-bin>
+                SSLOptions +StdEnvVars
+        </Directory>
+
+        BrowserMatch "MSIE [2-6]" \
+                nokeepalive ssl-unclean-shutdown \
+                downgrade-1.0 force-response-1.0
+        # MSIE 7 and newer should be able to use keepalive
+        BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown
+
+    </VirtualHost>
+</IfModule>

conf/core/common.json.conf

@@ -23,6 +23,9 @@
     "mail_return_path": "root",
     "mail_admin": "root",
     "mail_test_mode": true,
+    "mail_dev_mode": false,
+    "mail_dev_server": "localhost",
+    "mail_dev_port": 1025,
 
     #---------------------------------------------------------------------------
     # Common useful piper daemon configurations

conf/core/database.json.conf

@@ -8,7 +8,9 @@
         # SQL storage connection settings.
         #
         "sqlstorage": {
-            "url": "postgresql://mentat:mentat@localhost/mentat_main",
+            # Connect to the PostgreSQL running on outer host machine.
+            # "url": "postgresql://mentat:mentat@10.0.2.2:5432/mentat_main",
+            "url": "postgresql://mentat:mentat@localhost:5432/mentat_main",
             "echo": false
         },
 
@@ -19,6 +21,8 @@
             "dbname":   "mentat_events",
             "user":     "mentat",
             "password": "mentat",
+            # Connect to the PostgreSQL running on outer host machine.
+            # "host":     "10.0.2.2",
             "host":     "localhost",
             "port":     5432
         }

conf/core/reporting.json.conf

@@ -7,6 +7,7 @@
         "templates_dir": "/etc/mentat/templates/reporter",
         "event_classes_dir": "/etc/mentat/event_classes",
         "mail_admin": "root",
+        "fallback": ["abuse@cesnet.cz"],
 
         # Additional variables that can be used inside the report templates.
         #   default: {}

conf/cron/mentat-backup-py.cron

@@ -11,4 +11,4 @@
 #
 
 # Run every day at 5am.
-0 5 * * *   root /usr/local/bin/mentat-backup.py --regular
+0 5 * * *   root /var/mentat/venv/bin/mentat-backup.py --regular

conf/cron/mentat-cleanup-py.cron

@@ -11,4 +11,4 @@
 #
 
 # Run once in every 6 hours.
-5 */6 * * *   root /usr/local/bin/mentat-cleanup.py --regular
+5 */6 * * *   root /var/mentat/venv/bin/mentat-cleanup.py --regular

conf/cron/mentat-informant-py.cron

@@ -11,4 +11,4 @@
 #
 
 # Provide periodical informational report emails about overall performance of Mentat system.
-0 7 * * *   mentat /usr/local/bin/mentat-informant.py --regular
+0 7 * * *   mentat /var/mentat/venv/bin/mentat-informant.py --regular

conf/cron/mentat-netmngr-py.cron

@@ -11,4 +11,4 @@
 #
 
 # Run every day at 6am.
-0 6 * * *   mentat /usr/local/bin/mentat-netmngr.py --regular --command update --whois-file /var/mentat/whois-negistry.json
+0 6 * * *   mentat /var/mentat/venv/bin/mentat-netmngr.py --regular --command update --whois-file /var/mentat/whois-negistry.json --source negistry

conf/cron/mentat-precache-py.cron

@@ -11,4 +11,4 @@
 #
 
 # Run once in every 10 minutes.
-*/10 * * * *   mentat /usr/local/bin/mentat-precache.py --regular
+*/10 * * * *   mentat /var/mentat/venv/bin/mentat-precache.py --regular

conf/cron/mentat-reporter-py.cron

@@ -11,4 +11,4 @@
 #
 
 # Run once in every 10 minutes.
-*/10 * * * *  mentat /usr/local/bin/mentat-reporter.py --regular
+*/10 * * * *  mentat /var/mentat/venv/bin/mentat-reporter.py --regular

conf/cron/mentat-statistician-py.cron

@@ -11,4 +11,4 @@
 #
 
 # Run once in every 5 minutes.
-*/5 * * * *  mentat /usr/local/bin/mentat-statistician.py --regular
+*/5 * * * *  mentat /var/mentat/venv/bin/mentat-statistician.py --regular

conf/cron/mentat-watchdog-events-py.cron

@@ -11,4 +11,4 @@
 #
 
 # Run every 15 minutes.
-*/15 * * * *   root /usr/local/bin/mentat-dbmngr.py --command=watchdog-events --regular
+*/15 * * * *   root /var/mentat/venv/bin/mentat-dbmngr.py --command=watchdog-events --regular