Skip to content
Snippets Groups Projects
Commit 0dd30f0a authored by Jan Mach's avatar Jan Mach
Browse files

Tweaks and improvements in Vagrant development environment bootstrap scripts. 

(Redmine issue: #7041)
parent 15305227
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
.vagrantenv/.bashrc
+ 1
− 2
View file @ 0dd30f0a
......@@ -107,8 +107,7 @@ alias ..='cd ..'
alias ...='cd ../..'
alias ....='cd ../../..'
alias cdm='cd /vagrant/'
alias entervenv='. venv/bin/activate'
alias mentatvenv='. /var/mentat/venv/bin/activate'
alias entervenv='. /var/mentat/venv/bin/activate'
# enable programmable completion features (you don't need to enable
# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
......
.vagrantenv/finalize.sh deleted 100644 → 0
+ 0
− 47
View file @ 15305227
#!/bin/bash
#-------------------------------------------------------------------------------
# Final provisioning of Vagrant environment suitable for Mentat development.
#
# Copyright (C) since 2011 CESNET, z.s.p.o
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
. /etc/default/mentat
. /etc/mentat/scripts/lib.sh
echo "==========> <BEGIN> finalize.sh"
echo "==========> Bootstraping database schema and configuration"
sudo -u mentat ${MENTAT_VENV}/python /vagrant/bin/mentat-dbmngr.py --command init
sudo -u mentat bash -c "source ${MENTAT_VENV}/activate ; hawat-cli db stamp head"
sudo -u mentat bash -c "source ${MENTAT_VENV}/activate ; /etc/mentat/scripts/sqldb-migrate.sh stamp head"
/etc/mentat/scripts/sqldb-optimize.sh
echo "==========> Pregenerating application data"
sudo -u mentat ${MENTAT_VENV}/python /vagrant/bin/mentat-precache.py --allow-empty
echo "==========> Tweaking database users"
sudo -u postgres psql -c "SELECT usename FROM pg_catalog.pg_user;" | grep root > /dev/null
if [ $? -ne 0 ]; then
echo "Creating default PostgreSQL user 'root'"
sudo -u postgres psql -c "CREATE USER root SUPERUSER WITH PASSWORD 'root';"
fi
echo "==========> Configuring Apache to serve development server"
if [ ! -L /etc/apache2/sites-enabled/site_mentat_vagrant.conf ] ; then
ensure_linked_file /vagrant/conf/apache/site_mentat_vagrant.conf /etc/apache2/sites-available/site_mentat_vagrant.conf
sed -i.bak s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=mentat/g /etc/apache2/envvars
sed -i.bak s/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=mentat/g /etc/apache2/envvars
rm -f /etc/apache2/envvars.bak
a2enmod rewrite
a2enmod ssl
a2enmod wsgi
a2dissite 000-default.conf
a2dissite default-ssl.conf
a2ensite site_mentat_vagrant.conf
systemctl restart apache2.service
fi
echo "==========> <DONE> finalize.sh"
.vagrantenv/provision.sh
+ 59
− 21
View file @ 0dd30f0a
......@@ -8,6 +8,9 @@
echo "==========> <BEGIN> provision.sh"
source /vagrant/etc/default/mentat
source /vagrant/conf/scripts/lib.sh
echo "==========> Configuring timezone and locales"
echo "Europe/Prague" > /etc/timezone
dpkg-reconfigure -f noninteractive tzdata && \
......@@ -17,15 +20,11 @@ dpkg-reconfigure -f noninteractive tzdata && \
dpkg-reconfigure --frontend=noninteractive locales && \
update-locale LANG=en_US.UTF-8
echo "==========> Linking default configurations"
if [ ! -L /etc/default/mentat ] ; then
ln -s /vagrant/etc/default/mentat /etc/default/mentat
fi
if [ ! -L /etc/mentat ] ; then
ln -s /vagrant/conf /etc/mentat
fi
. /etc/default/mentat
. /etc/mentat/scripts/lib.sh
echo "==========> Linking configuration files"
ensure_link /vagrant/etc/default/mentat /etc/default/mentat
ensure_link /vagrant/conf /etc/mentat
ensure_link /vagrant/.vagrantenv/.bashrc /home/vagrant/.bashrc
ensure_link /vagrant/.vagrantenv/.bashrc /home/mentat/.bashrc
echo "==========> Creating Mentat user for development"
if ! getent passwd ${MENTAT_USER} > /dev/null 2>&1; then
......@@ -35,23 +34,27 @@ if ! getent passwd ${MENTAT_USER} > /dev/null 2>&1; then
echo "%${MENTAT_USER} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/mentat
fi
echo "==========> Linking additional configuration files"
ensure_linked_file /vagrant/.vagrantenv/.bashrc /home/vagrant/.bashrc
ensure_linked_file /vagrant/.vagrantenv/.bashrc /home/mentat/.bashrc
echo "==========> Installing essential dependencies"
apt-get update -qq \
&& DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade \
&& DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
build-essential \
gnupg2 \
rsync \
tmux \
curl \
wget \
less \
git \
vim \
mc \
openssl \
ssl-cert \
locales \
fakeroot \
devscripts \
debhelper \
lintian \
&& apt-get clean \
&& rm -rf /tmp/* /var/tmp/*
......@@ -82,13 +85,14 @@ apt-get update -qq \
python3-setuptools \
python3-pip \
python3-venv \
libpython3-dev \
postgresql-13 \
postgresql-13-ip4r \
postgresql-client-13 \
postgresql-server-dev-13 \
libpq-dev \
nodejs \
yarn=$YARN_VERSION-1 \
yarn \
rrdtool \
librrd-dev \
libdb5.3 \
......@@ -118,19 +122,53 @@ if [ ! -f /etc/mentat/install.pip ] ; then
fi
echo "==========> Preparing Python development environment"
if ! [ -d "${MENTAT_VENV}" ]; then
python3 -m venv "${MENTAT_VENV}"
fi
${MENTAT_VENV}/bin/pip install pip --upgrade
${MENTAT_VENV}/bin/pip install -r /etc/mentat/requirements-dev.pip
cd /vagrant/ && make venv
source ${MENTAT_VENV}/bin/activate
echo "==========> Preparing Python development environment"
cd /vagrant/ && make develop
echo "==========> Configuring email services for development"
ensure_linked_file /vagrant/.vagrantenv/postfix.main.cf /etc/postfix/main.cf
ensure_link /vagrant/.vagrantenv/postfix.main.cf /etc/postfix/main.cf
systemctl restart postfix.service
ensure_linked_file /vagrant/etc/systemd/system/sendria.service /etc/systemd/system/sendria.service
ensure_link /vagrant/etc/systemd/system/sendria.service /etc/systemd/system/sendria.service
systemctl daemon-reload
systemctl start sendria.service
echo "==========> Bootstraping database schema and configuration"
sudo -u mentat ${MENTAT_VENV}/python /vagrant/bin/mentat-dbmngr.py --command init
sudo -u mentat bash -c "source ${MENTAT_VENV}/activate ; hawat-cli db stamp head"
sudo -u mentat bash -c "source ${MENTAT_VENV}/activate ; /etc/mentat/scripts/sqldb-migrate.sh stamp head"
/etc/mentat/scripts/sqldb-optimize.sh
echo "==========> Pregenerating application data"
sudo -u mentat ${MENTAT_VENV}/python /vagrant/bin/mentat-precache.py --allow-empty
echo "==========> Tweaking database users"
sudo -u postgres psql -c "SELECT usename FROM pg_catalog.pg_user;" | grep root > /dev/null
if [ $? -ne 0 ]; then
echo "Creating default PostgreSQL user 'root'"
sudo -u postgres psql -c "CREATE USER root SUPERUSER WITH PASSWORD 'root';"
fi
echo "==========> Configuring Apache to serve development server"
if [ ! -L /etc/apache2/sites-enabled/site_mentat_vagrant.conf ] ; then
ensure_linked_file /vagrant/conf/apache/site_mentat_vagrant.conf /etc/apache2/sites-available/site_mentat_vagrant.conf
sed -i.bak s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=mentat/g /etc/apache2/envvars
sed -i.bak s/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=mentat/g /etc/apache2/envvars
rm -f /etc/apache2/envvars.bak
a2enmod rewrite
a2enmod ssl
a2enmod wsgi
a2dissite 000-default.conf
a2dissite default-ssl.conf
a2ensite site_mentat_vagrant.conf
systemctl restart apache2.service
fi
echo "==========> <DONE> provision.sh"
Makefile
+ 2
− 24
View file @ 0dd30f0a
......@@ -215,13 +215,9 @@ venv: FORCE
# - download IP geolocation databases
# - install the project in editable mode
#
develop: deps install-whl-dev build-webui
develop: deps build-webui
develop-docker: deps-docker translations-compile build-webui
deps: deps-prerequisites deps-lwchroot deps-python deps-python-dev deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
deps-docker: deps-prerequisites deps-dirs-docker deps-python deps-python-dev deps-editable deps-webui deps-geoip-docker deps-negistry-docker
deps: deps-prerequisites deps-lwchroot deps-python-dev deps-python deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
clean: clean-pycs clean-build-docs clean-build-python clean-build-debian
......@@ -282,10 +278,6 @@ deps-lwchroot: FORCE
@echo "\n$(GREEN)*** Creating local lightweight chroot subdirectory structure ***$(NC)\n"
@./conf/scripts/lwchroot-init.sh
deps-dirs-docker: FORCE
@echo "\n$(GREEN)*** Creating local lightweight chroot subdirectory structure ***$(NC)\n"
@./conf/scripts/devdirs-init.sh
#
# Install project`s Python dependencies using pip requirements file. The dependencies
# are already listed in setup.py file and pip can install them automatically. It
......@@ -365,12 +357,6 @@ deps-geoip: FORCE
@geoipupdate --verbose --database-directory $(shell realpath ./chroot/usr/share/GeoIP)
@echo ""
deps-geoip-docker: FORCE
@echo "\n$(GREEN)*** Installing IP geolocation databases ***$(NC)\n"
@mkdir -p /usr/share/GeoIP
#@geoipupdate --verbose --database-directory /usr/share/GeoIP
@echo ""
#
# Fetch and install Negistry whois database.
#
......@@ -379,11 +365,6 @@ deps-negistry: FORCE
@./conf/scripts/fetch-negistry.sh --stub --target=$(shell realpath ./chroot/var/mentat/whois-negistry.json)
@echo ""
deps-negistry-docker: FORCE
@echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n"
@./conf/scripts/fetch-negistry.sh --stub --target=/var/mentat/whois-negistry.json
@echo ""
#
# Create and configure required PostgreSQL user accounts and databases.
#
......@@ -397,17 +378,14 @@ deps-postgresql: FORCE
ctrl-mentat-dev:
#APP_ROOT_PATH=$(shell realpath ./chroot) mentat-controller.py --command $(COMMAND)
mentat-controller.py --command $(COMMAND)
run-mentat-dev:
@echo "\n$(GREEN)*** Running development version of Mentat system ***$(NC)\n"
#APP_ROOT_PATH=$(shell realpath ./chroot) mentat-controller.py --command start
mentat-controller.py --command start
run-webui-dev:
@echo "\n$(GREEN)*** Running development web server with development configuration on $(DEV_SERVER):$(DEV_PORT) ***$(NC)\n"
#APP_ROOT_PATH=$(shell realpath ./chroot) hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT)
HAWAT_CONFIG_FILE=$(shell realpath ./hawat.local.conf) hawat-cli run --host "0.0.0.0" --port $(DEV_PORT)
run-mailserver-dev:
......
Vagrantfile
+ 0
− 2
View file @ 0dd30f0a
......@@ -42,8 +42,6 @@ Vagrant.configure('2') do |config|
config.vm.provision 'shell', inline: <<-SHELL
bash /vagrant/.vagrantenv/provision.sh
bash /vagrant/packaging/debian/postinst
bash /vagrant/.vagrantenv/finalize.sh
SHELL
# Automatically connect as 'mentat' user with ssh command.
......
conf/scripts/lib.sh
+ 6
− 4
View file @ 0dd30f0a
......@@ -5,9 +5,11 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
function ensure_linked_file {
if [ -f $2 ] ; then
rm -f $2
function ensure_link {
if [ ! -L $2 ]
if [ -f $2 ] ; then
rm -f $2
fi
ln -s $1 $2
fi
ln -s $1 $2
}
conf/scripts/lwchroot-init.sh
+ 10
− 19
View file @ 0dd30f0a
......@@ -6,31 +6,22 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
mkdir -p ./chroot
mkdir -p ./chroot/etc
mkdir -p ./chroot/etc/cron.d
source ./etc/default/mentat
source ./conf/scripts/lib.sh
ensure_link $(realpath ./etc/default/mentat) /etc/default/mentat
ensure_link $(realpath ./conf) /etc/mentat
mkdir -p ./chroot/var
mkdir -p ./chroot/var/mentat/backups
mkdir -p ./chroot/var/mentat/cache
mkdir -p ./chroot/var/mentat/charts
mkdir -p ./chroot/var/mentat/log
mkdir -p ./chroot/var/mentat/reports
mkdir -p ./chroot/var/mentat/rrds
mkdir -p ./chroot/var/mentat/run
mkdir -p ./chroot/var/mentat/spool
mkdir -p ./chroot/usr/share/GeoIP
if [ ! -L ./chroot/etc/mentat ]; then
ln -s $(realpath ./conf) $(realpath ./chroot/etc)/mentat;
fi
ensure_link $(realpath ./chroot/var) /var/mentat
if [ ! -f .env ]; then
echo "APP_ROOT_PATH=$(realpath ./chroot)" >> .env
echo "FLASK_ENV=development" >> .env
echo "FLASK_CONFIG=development" >> .env
echo "FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf)" >> .env
fi
if [ ! -L ./migrations-events/.env ]; then
ln -s $(realpath ./.env) $(realpath ./migrations-events)/.env;
fi
\ No newline at end of file
if [ ! -L ./conf/migrations-events/.env ]; then
ln -s $(realpath ./.env) $(realpath ./conf/migrations-events)/.env;
fi
etc/default/mentat
+ 0
− 1
View file @ 0dd30f0a
......@@ -8,7 +8,6 @@
PG_MAJOR=13
NODE_MAJOR="12"
YARN_VERSION="1.22.5"
# Master switch for Debian system scripts.
MENTAT_IS_ENABLED=yes
......
packaging/debian/control
+ 3
− 2
View file @ 0dd30f0a
......@@ -6,8 +6,9 @@ Build-Depends: debhelper (>= 9)
Package: mentat-ng
Architecture: all
Depends: sudo, adduser, python3, python3-venv, libpython3-dev, libpq-dev,
rrdtool, librrd-dev, libdb5.3, libdb5.3-dev, init-system-helpers (>= 1.50)
Depends: sudo, adduser, python3, python3-venv, libpython3-dev, python3-setuptools,
python3-pip, libpq-dev, rrdtool, librrd-dev, libdb5.3, libdb5.3-dev,
init-system-helpers (>= 1.50)
Homepage: https://mentat.cesnet.cz/
Description: Mentat - SIEM for network operators
Next generation of Mentat SIEM system for network operators
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment