Skip to content
Snippets Groups Projects
Commit d54d6881 authored by Jan Mach's avatar Jan Mach
Browse files

Finetuning of the Vagrant provisioning. 

(Redmine issue: #7041)
parent 863643f6
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 199 additions and 145 deletions
Makefile
+ 9
− 42
View file @ d54d6881
......@@ -168,39 +168,7 @@ help:
# Install and configure Python virtual environment for local project development.
#
venv: FORCE
@echo "\n$(GREEN)*** Installing Python virtual environment for local development ***$(NC)\n"
@echo "Requested version: $(VENV_PYTHON)"
@echo "Path to binary: `which $(VENV_PYTHON)`"
@echo "Path to venv: $(VENV_PATH)"
@echo ""
@if [ -d $(VENV_PATH) ]; then\
echo "$(CYAN)Virtual environment already exists in '$(VENV_PATH)'.$(NC)";\
else\
$(VENV_PYTHON) -m venv $(VENV_PATH);\
echo "$(CYAN)Virtual environment successfully created in '$(VENV_PATH)'.$(NC)";\
fi
@echo ""
@echo "Upgrading 'pip' and 'wheel' within the virtual environment to latest versions"
@echo ""
@$(VENV_PATH)/bin/pip install --upgrade pip
@$(VENV_PATH)/bin/pip install --upgrade wheel
@echo ""
@echo "Venv path: `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`"
@echo "Python stuff versions:"
@echo ""
@$(VENV_PATH)/bin/python --version
@$(VENV_PATH)/bin/pip --version
@echo ""
@ls -al $(VENV_PATH)/bin | grep python
@ls -al $(VENV_PATH)/bin | grep pip
@echo "\n$(CYAN)Your development environment is ready in `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`.$(NC)\n"
@echo "Please activate it manually with following command:\n"
@echo "\t$(ORANGE). $(VENV_PATH)/bin/activate$(NC)\n"
@echo "Consider adding following alias to your ~/.bashrc file for easier environment activation:\n"
@echo "\t$(ORANGE)alias entervenv='. venv/bin/activate'$(NC)\n"
@echo "$(BOLD)!!! Please keep in mind, that all makefile targets except this one ('develop') leave it up to you to activate the correct virtual environment !!!$(NC)"
@echo ""
@./conf/scripts/init-venv.sh
#
# Install and configure project locally for development. This target will perform
......@@ -217,7 +185,7 @@ venv: FORCE
#
develop: deps build-webui
deps: deps-prerequisites deps-lwchroot deps-python-dev deps-python deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
deps: deps-prerequisites deps-runenv deps-python-dev deps-python deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
clean: clean-pycs clean-build-docs clean-build-python clean-build-debian
......@@ -245,7 +213,7 @@ release-whl: FORCE
#===============================================================================
deps-build: deps-prerequisites deps-lwchroot deps-python deps-python-dev deps-editable deps-webui translations-compile
deps-build: deps-prerequisites deps-runenv deps-python deps-python-dev deps-editable deps-webui translations-compile
deps-build-whl: deps-build
deps-build-deb: deps-build
......@@ -272,11 +240,11 @@ deps-prerequisites: FORCE
@echo ""
#
# Install project`s lightweight chroot.
# Install project`s runtime environment.
#
deps-lwchroot: FORCE
@echo "\n$(GREEN)*** Creating local lightweight chroot subdirectory structure ***$(NC)\n"
@./conf/scripts/lwchroot-init.sh
deps-runenv: FORCE
@echo "\n$(GREEN)*** Creating local runtime environment subdirectory structure ***$(NC)\n"
@./conf/scripts/init-runenv.sh
#
# Install project`s Python dependencies using pip requirements file. The dependencies
......@@ -353,8 +321,7 @@ deps-webui-upgrade: FORCE
#
deps-geoip: FORCE
@echo "\n$(GREEN)*** Installing IP geolocation databases ***$(NC)\n"
@mkdir -p ./chroot/usr/share/GeoIP
@geoipupdate --verbose --database-directory $(shell realpath ./chroot/usr/share/GeoIP)
@geoipupdate --verbose
@echo ""
#
......@@ -362,7 +329,7 @@ deps-geoip: FORCE
#
deps-negistry: FORCE
@echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n"
@./conf/scripts/fetch-negistry.sh --stub --target=$(shell realpath ./chroot/var/mentat/whois-negistry.json)
@./conf/scripts/fetch-negistry.sh --stub --target=/var/mentat/whois-negistry.json
@echo ""
#
......
Vagrantfile
+ 6
− 1
View file @ d54d6881
......@@ -5,6 +5,11 @@ Vagrant.configure('2') do |config|
config.vm.hostname = 'mentat-devel.local'
config.vm.box = 'debian/contrib-buster64'
config.vm.provider 'virtualbox' do |v|
v.memory = 2048
v.cpus = 2
end
# Host port 5000: Flask development server
config.vm.network 'forwarded_port', guest: 5000, host: 5000
# Host port 4443: Apache web server
......@@ -41,7 +46,7 @@ Vagrant.configure('2') do |config|
#
config.vm.provision 'shell', inline: <<-SHELL
bash /vagrant/.vagrantenv/provision.sh
bash /vagrant/vagrantenv/provision.sh
SHELL
# Automatically connect as 'mentat' user with ssh command.
......
bin/mentat-hawat-dev.wsgi
+ 5
− 4
View file @ d54d6881
......@@ -5,12 +5,9 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
from dotenv import load_dotenv
import hawat
load_dotenv(dotenv_path = '/home/mentat/mentat-ng/.env')
#
# Use prepared factory function to create application instance. The factory
# function takes number of arguments, that can be used to fine tune configuration
......@@ -18,4 +15,8 @@ load_dotenv(dotenv_path = '/home/mentat/mentat-ng/.env')
# capabilities or for purposes of testing. Please refer to the documentation
# for more information.
#
application = hawat.create_app()
application = hawat.create_app_full(
config_object = 'hawat.config.DevelopmentConfig',
config_file = '/etc/mentat/mentat-hawat-dev.py.conf',
config_env = 'FLASK_CONFIG_FILE'
)
hawat.local.conf → conf/mentat-hawat-dev.py.conf
+ 75
− 0
View file @ d54d6881
SECRET_KEY = 'local-secret-key'
# Force enable the debug mode.
DEBUG = True
# Debug template file loading precedence.
#EXPLAIN_TEMPLATE_LOADING = True
# Enable profiler on debug toolbar. DEBUG must be 'True' for toolbar to be visible.
#DEBUG_TB_PROFILER_ENABLED = True
#FLASK_ENV=production
#FLASK_ENV=development
#FLASK_CONFIG=production
#FLASK_CONFIG=development
# Move the application from webserver root to different location.
#APPLICATION_ROOT = '/mentat'
# Setup local secret key to be used as cryptographic salt.
SECRET_KEY = '!!!-CHANGE-ME-ASAP-!!!-local-secret-key'
# Define list of Hawat administrator accounts.
#EMAIL_ADMINS = ['boss@domain.org', 'admin@domain.org']
# Define list of Hawat administrator accounts, that receive feedback messages for reports.
#HAWAT_REPORT_FEEDBACK_MAILS = ['admin@domain.org']
# Directories with translations used in Hawat.
BABEL_DEFAULT_LOCALE = 'en'
BABEL_DEFAULT_TIMEZONE = 'UTC'
#BABEL_DEFAULT_LOCALE = 'cs'
#BABEL_DEFAULT_TIMEZONE = 'Europe/Prague'
# Limit for number of objects for which to automatically fetch additional data services.
HAWAT_LIMIT_AODS = 20
# Event database search query quota.
HAWAT_SEARCH_QUERY_QUOTA = 7
MAIL_SERVER = 'localhost'
MAIL_PORT = 8025
MAIL_DEFAULT_SENDER = 'mentat@cesnet.cz'
HAWAT_LOG_DEFAULT_LEVEL = 'debug'
HAWAT_LOG_FILE = '/var/tmp/mentat-hawat.py.log'
HAWAT_LOG_FILE_LEVEL = 'debug'
ENABLED_BLUEPRINTS = [
'vial.blueprints.auth',
'vial.blueprints.auth_api',
......
conf/scripts/init-runenv.sh 0 → 100755
+ 51
− 0
View file @ d54d6881
#!/bin/bash
#-------------------------------------------------------------------------------
# This file is part of Mentat system (https://mentat.cesnet.cz/).
#
# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
# Check exit status of any command and fail on error
set -e
# Source global configurations
. /etc/default/mentat
# Make sure required directory structure exists and has proper permissions
mentat_dir_list=(
/var/mentat
/var/mentat/backups
/var/mentat/cache
/var/mentat/charts
/var/mentat/log
/var/mentat/reports
/var/mentat/reports/reporter
/var/mentat/reports/informant
/var/mentat/reports/statistician
/var/mentat/rrds
/var/mentat/run
/var/mentat/spool
/var/mentat/spool/mentat-inspector.py
/var/mentat/spool/mentat-inspector.py/incoming
/var/mentat/spool/mentat-inspector.py/pending
/var/mentat/spool/mentat-inspector.py/tmp
/var/mentat/spool/mentat-inspector.py/errors
/var/mentat/spool/mentat-enricher.py
/var/mentat/spool/mentat-enricher.py/incoming
/var/mentat/spool/mentat-enricher.py/pending
/var/mentat/spool/mentat-enricher.py/tmp
/var/mentat/spool/mentat-enricher.py/errors
/var/mentat/spool/mentat-storage.py
/var/mentat/spool/mentat-storage.py/incoming
/var/mentat/spool/mentat-storage.py/pending
/var/mentat/spool/mentat-storage.py/tmp
/var/mentat/spool/mentat-storage.py/errors
)
for d in "${mentat_dir_list[@]}"
do
mkdir -p "$d"
chown "$MENTAT_USER" "$d"
chgrp "$MENTAT_GROUP" "$d"
chmod 775 "$d"
done
conf/scripts/init-venv.sh 0 → 100755
+ 43
− 0
View file @ d54d6881
#!/bin/bash
#-------------------------------------------------------------------------------
# This file is part of Mentat system (https://mentat.cesnet.cz/).
#
# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
# Check exit status of any command and fail on error
set -e
# Source global configurations
. /etc/default/mentat
echo -e "\n*** Initializing Python virtual environment for local development ***\n"
echo "Requested version: $PYTHON_VERSION"
echo "Path to binary: `which $PYTHON_VERSION`"
echo "Path to venv: $MENTAT_VENV"
echo ""
if [ -d $MENTAT_VENV ]; then
echo "Virtual environment already exists in '$MENTAT_VENV'.";
else
$PYTHON_VERSION -m venv $MENTAT_VENV;
echo "Virtual environment successfully created in '$MENTAT_VENV'.";
fi
echo -e "\n*** Upgrading 'pip' and 'wheel' within the virtual environment to latest versions ***\n"
$MENTAT_VENV/bin/pip install --upgrade pip
$MENTAT_VENV/bin/pip install --upgrade wheel
echo ""
echo "Venv path: `. $MENTAT_VENV/bin/activate && python -c 'import sys; print(sys.prefix)'`"
echo "Python stuff versions:"
echo ""
$MENTAT_VENV/bin/python --version
$MENTAT_VENV/bin/pip --version
echo ""
echo -e "\nYour development environment is ready in `. $MENTAT_VENV/bin/activate && python -c 'import sys; print(sys.prefix)'`.\n"
echo -e "Please activate it manually with following command:\n"
echo -e "\t$MENTAT_VENV/bin/activate\n"
echo -e "Consider adding following alias to your ~/.bashrc file for easier environment activation:\n"
echo -e "\talias entervenv='. venv/bin/activate'"
echo -e "\n!!! Please keep in mind, that all makefile targets except this one ('develop') leave it up to you to activate the correct virtual environment !!!\n"
conf/scripts/lwchroot-init.sh deleted 100755 → 0
+ 0
− 27
View file @ 863643f6
#!/bin/bash
#-------------------------------------------------------------------------------
# Utility script for creating local lightweight chroot substructure.
#
# Copyright (C) since 2011 CESNET, z.s.p.o
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
source ./etc/default/mentat
source ./conf/scripts/lib.sh
ensure_link $(realpath ./etc/default/mentat) /etc/default/mentat
ensure_link $(realpath ./conf) /etc/mentat
mkdir -p ./chroot/var
ensure_link $(realpath ./chroot/var) /var/mentat
if [ ! -f .env ]; then
echo "FLASK_ENV=development" >> .env
echo "FLASK_CONFIG=development" >> .env
echo "FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf)" >> .env
fi
if [ ! -L ./conf/migrations-events/.env ]; then
ln -s $(realpath ./.env) $(realpath ./conf/migrations-events)/.env;
fi
etc/default/mentat
+ 1
− 0
View file @ d54d6881
......@@ -8,6 +8,7 @@
PG_MAJOR=13
NODE_MAJOR="12"
PYTHON_VERSION="python3.7"
# Master switch for Debian system scripts.
MENTAT_IS_ENABLED=yes
......
packaging/debian/postinst
+ 1
− 67
View file @ d54d6881
......@@ -30,73 +30,7 @@ ${MENTAT_VENV}/bin/pip install -r /etc/mentat/requirements.pip
# Install Mentat system itself.
${MENTAT_VENV}/bin/pip install -r /etc/mentat/install.pip --upgrade --no-deps
mentat_bin_list=(
hawat-cli
mentat-backup.py
mentat-cleanup.py
mentat-controller.py
mentat-dbmngr.py
mentat-enricher.py
mentat-hawat.py
mentat-hawat.wsgi
mentat-ideagen.py
mentat-informant.py
mentat-inspector.py
mentat-netmngr.py
mentat-precache.py
mentat-reporter.py
mentat-sampler.py
mentat-statistician.py
mentat-storage.py
)
for b in "${mentat_bin_list[@]}"
do
if [ -f "/usr/local/bin/${b}" ]; then
rm -f "/usr/local/bin/${b}"
fi
if [ ! -L "/usr/local/bin/${b}" ]; then
ln -s "${MENTAT_VENV}/bin/${b}" "/usr/local/bin/${b}"
fi
done
# Make sure required directory structure exists and has proper permissions
mentat_dir_list=(
/var/mentat
/var/mentat/backups
/var/mentat/cache
/var/mentat/charts
/var/mentat/log
/var/mentat/reports
/var/mentat/reports/reporter
/var/mentat/reports/informant
/var/mentat/reports/statistician
/var/mentat/rrds
/var/mentat/run
/var/mentat/spool
/var/mentat/spool/mentat-inspector.py
/var/mentat/spool/mentat-inspector.py/incoming
/var/mentat/spool/mentat-inspector.py/pending
/var/mentat/spool/mentat-inspector.py/tmp
/var/mentat/spool/mentat-inspector.py/errors
/var/mentat/spool/mentat-enricher.py
/var/mentat/spool/mentat-enricher.py/incoming
/var/mentat/spool/mentat-enricher.py/pending
/var/mentat/spool/mentat-enricher.py/tmp
/var/mentat/spool/mentat-enricher.py/errors
/var/mentat/spool/mentat-storage.py
/var/mentat/spool/mentat-storage.py/incoming
/var/mentat/spool/mentat-storage.py/pending
/var/mentat/spool/mentat-storage.py/tmp
/var/mentat/spool/mentat-storage.py/errors
)
for d in "${mentat_dir_list[@]}"
do
mkdir -p "$d"
chown "$MENTAT_USER" "$d"
chgrp "$MENTAT_GROUP" "$d"
chmod 775 "$d"
done
/etc/mentat/scripts/init-runenv.sh
# Update the init scripts
update-rc.d mentat defaults-disabled
......
.vagrantenv/.bashrc → vagrantenv/.bashrc
+ 2
− 2
View file @ d54d6881
......@@ -124,8 +124,8 @@ fi
if [ ! -z "$SSH_CONNECTION" ]; then
echo ""
echo ""
if [ -x /vagrant/.vagrantenv/system-banner.sh ]; then
/vagrant/.vagrantenv/system-banner.sh
if [ -x /vagrant/vagrantenv/system-banner.sh ]; then
/vagrant/vagrantenv/system-banner.sh
fi
echo ""
fi
.vagrantenv/apt.list → vagrantenv/apt.list
+ 0
− 0
View file @ d54d6881
File moved
.vagrantenv/postfix.main.cf → vagrantenv/postfix.main.cf
+ 0
− 0
View file @ d54d6881
File moved
.vagrantenv/provision.sh → vagrantenv/provision.sh
+ 6
− 2
View file @ d54d6881
......@@ -24,7 +24,7 @@ dpkg-reconfigure -f noninteractive tzdata && \
echo "==========> Linking configuration files"
ensure_link /vagrant/etc/default/mentat /etc/default/mentat
ensure_link /vagrant/conf /etc/mentat
ensure_link /vagrant/.vagrantenv/.bashrc /home/vagrant/.bashrc
ensure_link /vagrant/vagrantenv/.bashrc /home/vagrant/.bashrc
ensure_link /vagrant/data/GeoIP.conf /etc/GeoIP.conf
echo "==========> Creating Mentat user for development"
......@@ -132,7 +132,7 @@ echo "==========> Preparing Python development environment"
cd /vagrant/ && make develop
echo "==========> Configuring email services for development"
ensure_link /vagrant/.vagrantenv/postfix.main.cf /etc/postfix/main.cf
ensure_link /vagrant/vagrantenv/postfix.main.cf /etc/postfix/main.cf
systemctl restart postfix.service
ensure_link /vagrant/etc/systemd/system/sendria.service /etc/systemd/system/sendria.service
......@@ -162,6 +162,7 @@ if [ $? -ne 0 ]; then
echo "Creating default PostgreSQL user 'vagrant'"
sudo -u postgres psql -c "CREATE USER vagrant WITH SUPERUSER CREATEDB ENCRYPTED PASSWORD 'vagrant';"
fi
#sudo -u mentat ${MENTAT_VENV}/bin/python /vagrant/bin/mentat-dbmngr.py --command user-add login=admin "fullname=Developer Admin" email=root "organization=CESNET, z.s.p.o." roles=user,admin
echo "==========> Configuring Apache to serve development server"
if [ ! -L /etc/apache2/sites-enabled/site_mentat_vagrant.conf ] ; then
......@@ -180,5 +181,8 @@ if [ ! -L /etc/apache2/sites-enabled/site_mentat_vagrant.conf ] ; then
systemctl restart apache2.service
fi
echo "==========> Importing production data into database"
cd /vagrant/ && make data-import
echo "==========> <DONE> provision.sh"
date
.vagrantenv/system-banner.sh → vagrantenv/system-banner.sh
+ 0
− 0
View file @ d54d6881
File moved
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment