Finetuning of the Vagrant provisioning.

(Redmine issue: #7041)

Makefile

@@ -168,39 +168,7 @@ help:
 # Install and configure Python virtual environment for local project development.
 #
 venv: FORCE
-	@echo "\n$(GREEN)*** Installing Python virtual environment for local development ***$(NC)\n"
-	@echo "Requested version: $(VENV_PYTHON)"
-	@echo "Path to binary:    `which $(VENV_PYTHON)`"
-	@echo "Path to venv:      $(VENV_PATH)"
-	@echo ""
-	@if [ -d $(VENV_PATH) ]; then\
-		echo "$(CYAN)Virtual environment already exists in '$(VENV_PATH)'.$(NC)";\
-	else\
-		$(VENV_PYTHON) -m venv $(VENV_PATH);\
-		echo "$(CYAN)Virtual environment successfully created in '$(VENV_PATH)'.$(NC)";\
-	fi
-	@echo ""
-	@echo "Upgrading 'pip' and 'wheel' within the virtual environment to latest versions"
-	@echo ""
-	@$(VENV_PATH)/bin/pip install --upgrade pip
-	@$(VENV_PATH)/bin/pip install --upgrade wheel
-	@echo ""
-	@echo "Venv path: `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`"
-	@echo "Python stuff versions:"
-	@echo ""
-	@$(VENV_PATH)/bin/python --version
-	@$(VENV_PATH)/bin/pip --version
-	@echo ""
-	@ls -al $(VENV_PATH)/bin | grep python
-	@ls -al $(VENV_PATH)/bin | grep pip
-
-	@echo "\n$(CYAN)Your development environment is ready in `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`.$(NC)\n"
-	@echo "Please activate it manually with following command:\n"
-	@echo "\t$(ORANGE). $(VENV_PATH)/bin/activate$(NC)\n"
-	@echo "Consider adding following alias to your ~/.bashrc file for easier environment activation:\n"
-	@echo "\t$(ORANGE)alias entervenv='. venv/bin/activate'$(NC)\n"
-	@echo "$(BOLD)!!! Please keep in mind, that all makefile targets except this one ('develop') leave it up to you to activate the correct virtual environment !!!$(NC)"
-	@echo ""
+	@./conf/scripts/init-venv.sh
 
 #
 # Install and configure project locally for development. This target will perform
@@ -217,7 +185,7 @@ venv: FORCE
 #
 develop: deps build-webui
 
-deps: deps-prerequisites deps-lwchroot deps-python-dev deps-python deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
+deps: deps-prerequisites deps-runenv deps-python-dev deps-python deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
 
 clean: clean-pycs clean-build-docs clean-build-python clean-build-debian
 
@@ -245,7 +213,7 @@ release-whl: FORCE
 #===============================================================================
 
 
-deps-build: deps-prerequisites deps-lwchroot deps-python deps-python-dev deps-editable deps-webui translations-compile
+deps-build: deps-prerequisites deps-runenv deps-python deps-python-dev deps-editable deps-webui translations-compile
 deps-build-whl: deps-build
 deps-build-deb: deps-build
 
@@ -272,11 +240,11 @@ deps-prerequisites: FORCE
 	@echo ""
 
 #
-# Install project`s lightweight chroot.
+# Install project`s runtime environment.
 #
-deps-lwchroot: FORCE
-	@echo "\n$(GREEN)*** Creating local lightweight chroot subdirectory structure ***$(NC)\n"
-	@./conf/scripts/lwchroot-init.sh
+deps-runenv: FORCE
+	@echo "\n$(GREEN)*** Creating local runtime environment subdirectory structure ***$(NC)\n"
+	@./conf/scripts/init-runenv.sh
 
 #
 # Install project`s Python dependencies using pip requirements file. The dependencies
@@ -353,8 +321,7 @@ deps-webui-upgrade: FORCE
 #
 deps-geoip: FORCE
 	@echo "\n$(GREEN)*** Installing IP geolocation databases ***$(NC)\n"
-	@mkdir -p ./chroot/usr/share/GeoIP
-	@geoipupdate --verbose --database-directory $(shell realpath ./chroot/usr/share/GeoIP)
+	@geoipupdate --verbose
 	@echo ""
 
 #
@@ -362,7 +329,7 @@ deps-geoip: FORCE
 #
 deps-negistry: FORCE
 	@echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n"
-	@./conf/scripts/fetch-negistry.sh --stub --target=$(shell realpath ./chroot/var/mentat/whois-negistry.json)
+	@./conf/scripts/fetch-negistry.sh --stub --target=/var/mentat/whois-negistry.json
 	@echo ""
 
 #

Vagrantfile

@@ -5,6 +5,11 @@ Vagrant.configure('2') do |config|
   config.vm.hostname = 'mentat-devel.local'
   config.vm.box = 'debian/contrib-buster64'
 
+  config.vm.provider 'virtualbox' do |v|
+    v.memory = 2048
+    v.cpus = 2
+  end
+
   # Host port 5000: Flask development server
   config.vm.network 'forwarded_port', guest: 5000, host: 5000
   # Host port 4443: Apache web server
@@ -41,7 +46,7 @@ Vagrant.configure('2') do |config|
   #
 
   config.vm.provision 'shell', inline: <<-SHELL
-    bash /vagrant/.vagrantenv/provision.sh
+    bash /vagrant/vagrantenv/provision.sh
   SHELL
 
   # Automatically connect as 'mentat' user with ssh command.

bin/mentat-hawat-dev.wsgi

@@ -5,12 +5,9 @@
 # Use of this source is governed by the MIT license, see LICENSE file.
 #-------------------------------------------------------------------------------
 
-from dotenv import load_dotenv
 
 import hawat
 
-load_dotenv(dotenv_path = '/home/mentat/mentat-ng/.env')
-
 #
 # Use prepared factory function to create application instance. The factory
 # function takes number of arguments, that can be used to fine tune configuration
@@ -18,4 +15,8 @@ load_dotenv(dotenv_path = '/home/mentat/mentat-ng/.env')
 # capabilities or for purposes of testing. Please refer to the documentation
 # for more information.
 #
-application = hawat.create_app()
+application = hawat.create_app_full(
+    config_object = 'hawat.config.DevelopmentConfig',
+    config_file   = '/etc/mentat/mentat-hawat-dev.py.conf',
+    config_env    = 'FLASK_CONFIG_FILE'
+)

hawat.local.conf → conf/mentat-hawat-dev.py.conf

-SECRET_KEY = 'local-secret-key'
+# Force enable the debug mode.
+DEBUG = True
+
+# Debug template file loading precedence.
+#EXPLAIN_TEMPLATE_LOADING = True
+
+# Enable profiler on debug toolbar. DEBUG must be 'True' for toolbar to be visible.
+#DEBUG_TB_PROFILER_ENABLED = True
+
+#FLASK_ENV=production
+#FLASK_ENV=development
+#FLASK_CONFIG=production
+#FLASK_CONFIG=development
+
+# Move the application from webserver root to different location.
+#APPLICATION_ROOT = '/mentat'
+
+# Setup local secret key to be used as cryptographic salt.
+SECRET_KEY = '!!!-CHANGE-ME-ASAP-!!!-local-secret-key'
+
+# Define list of Hawat administrator accounts.
+#EMAIL_ADMINS = ['boss@domain.org', 'admin@domain.org']
+
+# Define list of Hawat administrator accounts, that receive feedback messages for reports.
+#HAWAT_REPORT_FEEDBACK_MAILS = ['admin@domain.org']
+
+# Directories with translations used in Hawat.
+BABEL_DEFAULT_LOCALE          = 'en'
+BABEL_DEFAULT_TIMEZONE        = 'UTC'
+#BABEL_DEFAULT_LOCALE         = 'cs'
+#BABEL_DEFAULT_TIMEZONE       = 'Europe/Prague'
+
+# Limit for number of objects for which to automatically fetch additional data services.
+HAWAT_LIMIT_AODS = 20
+
+# Event database search query quota.
+HAWAT_SEARCH_QUERY_QUOTA = 7
+
 MAIL_SERVER = 'localhost'
 MAIL_PORT = 8025
 MAIL_DEFAULT_SENDER = 'mentat@cesnet.cz'
+
 HAWAT_LOG_DEFAULT_LEVEL = 'debug'
 HAWAT_LOG_FILE = '/var/tmp/mentat-hawat.py.log'
 HAWAT_LOG_FILE_LEVEL = 'debug'
+
 ENABLED_BLUEPRINTS = [
     'vial.blueprints.auth',
     'vial.blueprints.auth_api',

conf/scripts/init-runenv.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+# Check exit status of any command and fail on error
+set -e
+
+# Source global configurations
+. /etc/default/mentat
+
+# Make sure required directory structure exists and has proper permissions
+mentat_dir_list=(
+    /var/mentat
+    /var/mentat/backups
+    /var/mentat/cache
+    /var/mentat/charts
+    /var/mentat/log
+    /var/mentat/reports
+    /var/mentat/reports/reporter
+    /var/mentat/reports/informant
+    /var/mentat/reports/statistician
+    /var/mentat/rrds
+    /var/mentat/run
+    /var/mentat/spool
+    /var/mentat/spool/mentat-inspector.py
+    /var/mentat/spool/mentat-inspector.py/incoming
+    /var/mentat/spool/mentat-inspector.py/pending
+    /var/mentat/spool/mentat-inspector.py/tmp
+    /var/mentat/spool/mentat-inspector.py/errors
+    /var/mentat/spool/mentat-enricher.py
+    /var/mentat/spool/mentat-enricher.py/incoming
+    /var/mentat/spool/mentat-enricher.py/pending
+    /var/mentat/spool/mentat-enricher.py/tmp
+    /var/mentat/spool/mentat-enricher.py/errors
+    /var/mentat/spool/mentat-storage.py
+    /var/mentat/spool/mentat-storage.py/incoming
+    /var/mentat/spool/mentat-storage.py/pending
+    /var/mentat/spool/mentat-storage.py/tmp
+    /var/mentat/spool/mentat-storage.py/errors
+)
+for d in "${mentat_dir_list[@]}"
+do
+    mkdir -p "$d"
+    chown "$MENTAT_USER" "$d"
+    chgrp "$MENTAT_GROUP" "$d"
+    chmod 775 "$d"
+done

conf/scripts/init-venv.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+# Check exit status of any command and fail on error
+set -e
+
+# Source global configurations
+. /etc/default/mentat
+
+echo -e "\n*** Initializing Python virtual environment for local development ***\n"
+echo "Requested version: $PYTHON_VERSION"
+echo "Path to binary:    `which $PYTHON_VERSION`"
+echo "Path to venv:      $MENTAT_VENV"
+echo ""
+if [ -d $MENTAT_VENV ]; then
+    echo "Virtual environment already exists in '$MENTAT_VENV'.";
+else
+    $PYTHON_VERSION -m venv $MENTAT_VENV;
+    echo "Virtual environment successfully created in '$MENTAT_VENV'.";
+fi
+
+echo -e "\n*** Upgrading 'pip' and 'wheel' within the virtual environment to latest versions ***\n"
+$MENTAT_VENV/bin/pip install --upgrade pip
+$MENTAT_VENV/bin/pip install --upgrade wheel
+echo ""
+echo "Venv path: `. $MENTAT_VENV/bin/activate && python -c 'import sys; print(sys.prefix)'`"
+echo "Python stuff versions:"
+echo ""
+$MENTAT_VENV/bin/python --version
+$MENTAT_VENV/bin/pip --version
+echo ""
+
+echo -e "\nYour development environment is ready in `. $MENTAT_VENV/bin/activate && python -c 'import sys; print(sys.prefix)'`.\n"
+echo -e "Please activate it manually with following command:\n"
+echo -e "\t$MENTAT_VENV/bin/activate\n"
+echo -e "Consider adding following alias to your ~/.bashrc file for easier environment activation:\n"
+echo -e "\talias entervenv='. venv/bin/activate'"
+echo -e "\n!!! Please keep in mind, that all makefile targets except this one ('develop') leave it up to you to activate the correct virtual environment !!!\n"

conf/scripts/lwchroot-init.sh

-#!/bin/bash
-#-------------------------------------------------------------------------------
-# Utility script for creating local lightweight chroot substructure.
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-source ./etc/default/mentat
-source ./conf/scripts/lib.sh
-
-ensure_link $(realpath ./etc/default/mentat) /etc/default/mentat
-ensure_link $(realpath ./conf) /etc/mentat
-
-mkdir -p ./chroot/var
-
-ensure_link $(realpath ./chroot/var) /var/mentat
-
-if [ ! -f .env ]; then
-	echo "FLASK_ENV=development" >> .env
-	echo "FLASK_CONFIG=development" >> .env
-	echo "FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf)" >> .env
-fi
-
-if [ ! -L ./conf/migrations-events/.env ]; then
-	ln -s $(realpath ./.env) $(realpath ./conf/migrations-events)/.env;
-fi

etc/default/mentat

@@ -8,6 +8,7 @@
 
 PG_MAJOR=13
 NODE_MAJOR="12"
+PYTHON_VERSION="python3.7"
 
 # Master switch for Debian system scripts.
 MENTAT_IS_ENABLED=yes

packaging/debian/postinst

@@ -30,73 +30,7 @@ ${MENTAT_VENV}/bin/pip install -r /etc/mentat/requirements.pip
 # Install Mentat system itself.
 ${MENTAT_VENV}/bin/pip install -r /etc/mentat/install.pip --upgrade --no-deps
 
-mentat_bin_list=(
-    hawat-cli
-    mentat-backup.py
-    mentat-cleanup.py
-    mentat-controller.py
-    mentat-dbmngr.py
-    mentat-enricher.py
-    mentat-hawat.py
-    mentat-hawat.wsgi
-    mentat-ideagen.py
-    mentat-informant.py
-    mentat-inspector.py
-    mentat-netmngr.py
-    mentat-precache.py
-    mentat-reporter.py
-    mentat-sampler.py
-    mentat-statistician.py
-    mentat-storage.py
-)
-for b in "${mentat_bin_list[@]}"
-do
-    if [ -f "/usr/local/bin/${b}" ]; then
-        rm -f "/usr/local/bin/${b}"
-    fi
-    if [ ! -L "/usr/local/bin/${b}" ]; then
-        ln -s "${MENTAT_VENV}/bin/${b}" "/usr/local/bin/${b}"
-    fi
-done
-
-
-# Make sure required directory structure exists and has proper permissions
-mentat_dir_list=(
-    /var/mentat
-    /var/mentat/backups
-    /var/mentat/cache
-    /var/mentat/charts
-    /var/mentat/log
-    /var/mentat/reports
-    /var/mentat/reports/reporter
-    /var/mentat/reports/informant
-    /var/mentat/reports/statistician
-    /var/mentat/rrds
-    /var/mentat/run
-    /var/mentat/spool
-    /var/mentat/spool/mentat-inspector.py
-    /var/mentat/spool/mentat-inspector.py/incoming
-    /var/mentat/spool/mentat-inspector.py/pending
-    /var/mentat/spool/mentat-inspector.py/tmp
-    /var/mentat/spool/mentat-inspector.py/errors
-    /var/mentat/spool/mentat-enricher.py
-    /var/mentat/spool/mentat-enricher.py/incoming
-    /var/mentat/spool/mentat-enricher.py/pending
-    /var/mentat/spool/mentat-enricher.py/tmp
-    /var/mentat/spool/mentat-enricher.py/errors
-    /var/mentat/spool/mentat-storage.py
-    /var/mentat/spool/mentat-storage.py/incoming
-    /var/mentat/spool/mentat-storage.py/pending
-    /var/mentat/spool/mentat-storage.py/tmp
-    /var/mentat/spool/mentat-storage.py/errors
-)
-for d in "${mentat_dir_list[@]}"
-do
-    mkdir -p "$d"
-    chown "$MENTAT_USER" "$d"
-    chgrp "$MENTAT_GROUP" "$d"
-    chmod 775 "$d"
-done
+/etc/mentat/scripts/init-runenv.sh
 
 # Update the init scripts
 update-rc.d mentat defaults-disabled

.vagrantenv/.bashrc → vagrantenv/.bashrc

@@ -124,8 +124,8 @@ fi
 if [ ! -z "$SSH_CONNECTION" ]; then
     echo ""
     echo ""
-    if [ -x /vagrant/.vagrantenv/system-banner.sh ]; then
-        /vagrant/.vagrantenv/system-banner.sh
+    if [ -x /vagrant/vagrantenv/system-banner.sh ]; then
+        /vagrant/vagrantenv/system-banner.sh
     fi
     echo ""
 fi

.vagrantenv/provision.sh → vagrantenv/provision.sh

@@ -24,7 +24,7 @@ dpkg-reconfigure -f noninteractive tzdata && \
 echo "==========> Linking configuration files"
 ensure_link /vagrant/etc/default/mentat /etc/default/mentat
 ensure_link /vagrant/conf /etc/mentat
-ensure_link /vagrant/.vagrantenv/.bashrc /home/vagrant/.bashrc
+ensure_link /vagrant/vagrantenv/.bashrc /home/vagrant/.bashrc
 ensure_link /vagrant/data/GeoIP.conf /etc/GeoIP.conf
 
 echo "==========> Creating Mentat user for development"
@@ -132,7 +132,7 @@ echo "==========> Preparing Python development environment"
 cd /vagrant/ && make develop
 
 echo "==========> Configuring email services for development"
-ensure_link /vagrant/.vagrantenv/postfix.main.cf /etc/postfix/main.cf
+ensure_link /vagrant/vagrantenv/postfix.main.cf /etc/postfix/main.cf
 systemctl restart postfix.service
 
 ensure_link /vagrant/etc/systemd/system/sendria.service /etc/systemd/system/sendria.service
@@ -162,6 +162,7 @@ if [ $? -ne 0 ]; then
     echo "Creating default PostgreSQL user 'vagrant'"
     sudo -u postgres psql -c "CREATE USER vagrant WITH SUPERUSER CREATEDB ENCRYPTED PASSWORD 'vagrant';"
 fi
+#sudo -u mentat ${MENTAT_VENV}/bin/python /vagrant/bin/mentat-dbmngr.py --command user-add login=admin "fullname=Developer Admin" email=root "organization=CESNET, z.s.p.o." roles=user,admin
 
 echo "==========> Configuring Apache to serve development server"
 if [ ! -L /etc/apache2/sites-enabled/site_mentat_vagrant.conf ] ; then
@@ -180,5 +181,8 @@ if [ ! -L /etc/apache2/sites-enabled/site_mentat_vagrant.conf ] ; then
   systemctl restart apache2.service
 fi
 
+echo "==========> Importing production data into database"
+cd /vagrant/ && make data-import
+
 echo "==========> <DONE> provision.sh"
 date