Optimizations in Vagrant box provisioning scripts.

(Redmine issue: #7041)

Makefile

@@ -501,6 +501,11 @@ data-import-dbsnapshot: FORCE
 	@echo "\n$(GREEN)*** Importing latest production database dump ***$(NC)\n"
 	@sudo -u postgres pg_restore --verbose --format=d --dbname=mentat_main /vagrant/data/mentat_psqldb_latest/main
 	@echo ""
+	@sudo -u postgres psql mentat_main -t -e -c "select count(*) from users;"
+	@sudo -u postgres psql mentat_main -t -e -c "select count(*) from groups;"
+	@sudo -u postgres psql mentat_main -t -e -c "select count(*) from filters;"
+	@sudo -u postgres psql mentat_main -t -e -c "select count(*) from networks;"
+	@echo ""
 
 #-------------------------------------------------------------------------------
 

Vagrantfile

@@ -26,8 +26,16 @@ Vagrant.configure('2') do |config|
   # your network.
   # config.vm.network "public_network"
 
-  config.vm.provision 'bootstrap', type: 'shell', inline: <<-SHELL
-    bash /vagrant/vagrantenv/provision.sh
+  config.vm.provision 'bootstrap_system', type: 'shell', inline: <<-SHELL
+    bash /vagrant/vagrantenv/provisioning/p01_system.sh
+  SHELL
+
+  config.vm.provision 'bootstrap_mentat', type: 'shell', inline: <<-SHELL
+    bash /vagrant/vagrantenv/provisioning/p02_mentat.sh
+  SHELL
+
+  config.vm.provision 'bootstrap_warden', type: 'shell', inline: <<-SHELL
+    bash /vagrant/vagrantenv/provisioning/p03_warden.sh
   SHELL
 
   # Automatically connect as 'mentat' user with ssh command.

conf/scripts/lib.sh

@@ -16,3 +16,14 @@ function ensure_link {
 		ln -s $1 $2
 	fi
 }
+
+function print_title {
+	echo "================================================================================"
+	echo " $1"
+	echo "================================================================================"
+	date
+}
+
+function print_subtitle {
+	echo "==========> $1"
+}

vagrantenv/provisioning/p01_system.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Initial provisioning of Vagrant environment suitable for Mentat development.
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+source /vagrant/etc/default/mentat
+source /vagrant/conf/scripts/lib.sh
+
+print_title '<BEGIN> PROVISIONING BASE SYSTEM'
+
+print_subtitle 'Configuring timezone and locales'
+echo "Europe/Prague" > /etc/timezone
+dpkg-reconfigure -f noninteractive tzdata && \
+  sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+  sed -i -e 's/# cs_CZ.UTF-8 UTF-8/cs_CZ.UTF-8 UTF-8/' /etc/locale.gen && \
+  echo 'LANG="en_US.UTF-8"'>/etc/default/locale && \
+  dpkg-reconfigure --frontend=noninteractive locales && \
+  update-locale LANG=en_US.UTF-8
+
+print_subtitle 'Preparing system for development'
+chmod g+w /vagrant
+
+print_subtitle 'Linking configuration files'
+ensure_link /vagrant/vagrantenv/system/.bashrc /root/.bashrc
+ensure_link /vagrant/vagrantenv/system/.bashrc /home/vagrant/.bashrc
+ensure_link /vagrant/data/GeoIP.conf /etc/GeoIP.conf
+ensure_link /vagrant/data/geoip /usr/share/GeoIP
+
+print_subtitle 'Installing essential dependencies'
+apt-get update -qq \
+  && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade \
+  && DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
+    sudo \
+    bash-completion \
+    build-essential \
+    gnupg2 \
+    rsync \
+    tmux \
+    curl \
+    wget \
+    less \
+    git \
+    vim \
+    mc \
+    net-tools \
+    openssl \
+    ssl-cert \
+    locales \
+  && apt-get clean \
+  && rm -rf /tmp/* /var/tmp/*
+
+print_subtitle 'Installing third party package repository - PostgreSQL'
+if [ ! -f /etc/apt/sources.list.d/pgdg.list ] ; then
+  curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+  echo 'deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list
+fi
+
+print_subtitle 'Installing third party package repository - NodeJS'
+if [ ! -f /etc/apt/sources.list.d/nodesource.list ] ; then
+  curl -sL https://deb.nodesource.com/setup_$NODE_MAJOR.x | bash -
+fi
+
+print_subtitle 'Installing third party package repository - Yarn'
+if [ ! -f /etc/apt/sources.list.d/yarn.list ] ; then
+  curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
+  echo 'deb http://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list
+fi
+
+print_title '<DONE> PROVISIONING BASE SYSTEM'

vagrantenv/provision.sh → vagrantenv/provisioning/p02_mentat.sh

@@ -6,90 +6,14 @@
 # Use of this source is governed by the MIT license, see LICENSE file.
 #-------------------------------------------------------------------------------
 
-echo "==========> <BEGIN> provision.sh"
-date
-
 source /vagrant/etc/default/mentat
 source /vagrant/conf/scripts/lib.sh
 
-echo "==========> Configuring timezone and locales"
-echo "Europe/Prague" > /etc/timezone
-dpkg-reconfigure -f noninteractive tzdata && \
-  sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
-  sed -i -e 's/# cs_CZ.UTF-8 UTF-8/cs_CZ.UTF-8 UTF-8/' /etc/locale.gen && \
-  echo 'LANG="en_US.UTF-8"'>/etc/default/locale && \
-  dpkg-reconfigure --frontend=noninteractive locales && \
-  update-locale LANG=en_US.UTF-8
-
-echo "==========> Preparing system for development"
-chmod g+w /vagrant
-
-echo "==========> Creating Mentat user for development"
-if ! getent passwd ${MENTAT_USER} > /dev/null 2>&1; then
-    useradd -m -s /bin/bash -U ${MENTAT_USER} --groups sudo
-    cp -pr /home/vagrant/.ssh /home/${MENTAT_USER}/
-    chown -R ${MENTAT_USER}:${MENTAT_USER} /home/${MENTAT_USER}
-    echo "%${MENTAT_USER} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/mentat
-    usermod -a -G mentat vagrant
-    usermod -a -G vagrant mentat
-fi
-
-echo "==========> Linking configuration files"
-ensure_link /vagrant/etc/default/mentat /etc/default/mentat
-ensure_link /vagrant/conf /etc/mentat
-ensure_link /vagrant/vagrantenv/.bashrc /root/.bashrc
-ensure_link /vagrant/vagrantenv/.bashrc /home/vagrant/.bashrc
-ensure_link /vagrant/vagrantenv/.bashrc /home/mentat/.bashrc
-ensure_link /vagrant/data/GeoIP.conf /etc/GeoIP.conf
-ensure_link /vagrant/data/geoip /usr/share/GeoIP
-
-echo "==========> Installing essential dependencies"
-apt-get update -qq \
-  && DEBIAN_FRONTEND=noninteractive apt-get -yq dist-upgrade \
-  && DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
-    build-essential \
-    gnupg2 \
-    rsync \
-    tmux \
-    curl \
-    wget \
-    less \
-    git \
-    vim \
-    mc \
-    net-tools \
-    openssl \
-    ssl-cert \
-    locales \
-    fakeroot \
-    devscripts \
-    debhelper \
-    lintian \
-  && apt-get clean \
-  && rm -rf /tmp/* /var/tmp/*
-
-echo "==========> Installing third party package repository - PostgreSQL"
-if [ ! -f /etc/apt/sources.list.d/pgdg.list ] ; then
-  curl -sSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-  echo 'deb http://apt.postgresql.org/pub/repos/apt/ buster-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list
-fi
-
-echo "==========> Installing third party package repository - NodeJS"
-if [ ! -f /etc/apt/sources.list.d/nodesource.list ] ; then
-  curl -sL https://deb.nodesource.com/setup_$NODE_MAJOR.x | bash -
-fi
-
-echo "==========> Installing third party package repository - Yarn"
-if [ ! -f /etc/apt/sources.list.d/yarn.list ] ; then
-  curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
-  echo 'deb http://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list
-fi
+print_title '<BEGIN> PROVISIONING MENTAT SYSTEM'
 
-echo "==========> Installing application dependencies"
+print_subtitle 'Installing application dependencies'
 apt-get update -qq \
   && DEBIAN_FRONTEND=noninteractive apt-get install -yq --no-install-recommends \
-    sudo \
-    bash-completion \
     python3 \
     python3-dev \
     python3-setuptools \
@@ -112,37 +36,51 @@ apt-get update -qq \
     apache2 \
     libapache2-mod-wsgi-py3 \
     init-system-helpers \
+    fakeroot \
+    devscripts \
+    debhelper \
+    lintian \
   && apt-get clean \
   && rm -rf /tmp/* /var/tmp/*
 
-echo "==========> Installing GeoIPupdate tool"
+print_subtitle 'Installing Grunt task runner'
+if [ ! -f /usr/bin/grunt ] && [ ! -e /usr/bin/grunt ] ; then
+  npm install -g grunt-cli
+fi
+
+print_subtitle 'Installing GeoIPupdate tool'
 if [ ! -f /usr/bin/geoipupdate ] && [ ! -e /usr/bin/geoipupdate ] ; then
   wget -O /tmp/geoipupdate_4.6.0_linux_amd64.deb https://github.com/maxmind/geoipupdate/releases/download/v4.6.0/geoipupdate_4.6.0_linux_amd64.deb && \
     dpkg -i /tmp/geoipupdate_4.6.0_linux_amd64.deb && \
     rm -f /tmp/geoipupdate_4.6.0_linux_amd64.deb
 fi
 
-echo "==========> Installing Grunt task runner"
-if [ ! -f /usr/bin/grunt ] && [ ! -e /usr/bin/grunt ] ; then
-  npm install -g grunt-cli
+print_subtitle 'Creating Mentat user for development'
+if ! getent passwd ${MENTAT_USER} > /dev/null 2>&1; then
+    useradd -m -s /bin/bash -U ${MENTAT_USER} --groups sudo
+    cp -pr /home/vagrant/.ssh /home/${MENTAT_USER}/
+    chown -R ${MENTAT_USER}:${MENTAT_USER} /home/${MENTAT_USER}
+    echo "%${MENTAT_USER} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/mentat
+    usermod -a -G mentat vagrant
+    usermod -a -G vagrant mentat
 fi
 
-echo "==========> Installing application installation PIP file"
-if [ ! -f /etc/mentat/install.pip ] ; then
-  echo '--editable "/vagrant/"' > /etc/mentat/install.pip
-fi
+print_subtitle 'Linking configuration files'
+ensure_link /vagrant/vagrantenv/system/.bashrc /home/mentat/.bashrc
+ensure_link /vagrant/etc/default/mentat /etc/default/mentat
+ensure_link /vagrant/conf /etc/mentat
 
-echo "==========> Preparing Python virtual environment"
+print_subtitle 'Preparing Python virtual environment'
 /etc/mentat/scripts/init-venv.sh
 source ${MENTAT_VENV}/bin/activate
 
-echo "==========> Preparing application runtime environment"
+print_subtitle 'Preparing application runtime environment'
 /etc/mentat/scripts/init-runenv.sh
 
-echo "==========> Preparing development environment"
+print_subtitle 'Preparing development environment'
 cd /vagrant/ && make develop-vagrant
 
-echo "==========> Configuring email services for development"
+print_subtitle 'Configuring email services for development'
 ensure_link /vagrant/vagrantenv/postfix.main.cf /etc/postfix/main.cf
 systemctl restart postfix.service
 
@@ -155,7 +93,7 @@ systemctl daemon-reload
 systemctl start sendria.service
 systemctl enable sendria.service
 
-echo "==========> Tweaking database users"
+print_subtitle 'Tweaking database users'
 for dbuname in root mentat vagrant watchdog
 do
   sudo -u postgres psql -c "SELECT usename FROM pg_catalog.pg_user;" | grep $dbuname > /dev/null
@@ -166,16 +104,16 @@ do
 done
 #sudo -u mentat ${MENTAT_VENV}/bin/python /vagrant/bin/mentat-dbmngr.py --command user-add login=admin "fullname=Developer Admin" email=root "organization=CESNET, z.s.p.o." roles=user,admin
 
-echo "==========> Bootstraping database schema and configuration"
+print_subtitle 'Bootstraping database schema and configuration'
 sudo -u mentat ${MENTAT_VENV}/bin/python /vagrant/bin/mentat-dbmngr.py --command init
 sudo -u mentat bash -c "source ${MENTAT_VENV}/bin/activate ; hawat-cli db stamp head"
 sudo -u mentat bash -c "source ${MENTAT_VENV}/bin/activate ; /etc/mentat/scripts/sqldb-migrate.sh stamp head"
 /etc/mentat/scripts/sqldb-optimize.sh
 
-echo "==========> Pregenerating application data"
+print_subtitle 'Pregenerating application data'
 sudo -u mentat ${MENTAT_VENV}/bin/python /vagrant/bin/mentat-precache.py --allow-empty
 
-echo "==========> Configuring Apache to serve development server"
+print_subtitle 'Configuring Apache to serve development server'
 if [ ! -L /etc/apache2/sites-enabled/site_mentat_vagrant.conf ] ; then
   ensure_link /vagrant/conf/apache/site_mentat_vagrant.conf /etc/apache2/sites-available/site_mentat_vagrant.conf
   sed -i.bak s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=mentat/g /etc/apache2/envvars
@@ -192,25 +130,4 @@ if [ ! -L /etc/apache2/sites-enabled/site_mentat_vagrant.conf ] ; then
   systemctl restart apache2.service
 fi
 
-echo "==========> Installing Warden client library"
-if [ ! -d /opt/warden3 ] ; then
-  git clone https://homeproj.cesnet.cz/git/warden.git/ /opt/warden3
-fi
-mkdir -p /etc/warden_client/warden_filer
-mkdir -p /run/warden_filer
-mkdir -p /var/lib/warden_filer
-chown -R mentat:mentat /run/warden_filer
-chown -R mentat:mentat /var/lib/warden_filer
-ensure_link /opt/warden3/warden_client/warden_client.py /usr/local/bin/warden_client.py
-ensure_link /opt/warden3/warden_filer/warden_filer.py /usr/local/bin/warden_filer.py
-ensure_link /opt/warden3/warden_ra/warden_apply.sh /usr/local/bin/warden_apply.sh
-chmod a+x /usr/local/bin/warden_client.py
-chmod a+x /usr/local/bin/bin/warden_filer.py
-chmod a+x /usr/local/bin/warden_apply.sh
-ensure_link /vagrant/vagrantenv/warden/warden_filer /etc/default/warden_filer
-ensure_link /vagrant/vagrantenv/warden/warden_filer.cfg /etc/warden_client/warden_filer.cfg
-ensure_link /vagrant/vagrantenv/warden/warden_filer_receiver.service /etc/systemd/system/warden_filer_receiver.service
-systemctl daemon-reload
-
-echo "==========> <DONE> provision.sh"
-date
+print_title '<DONE> PROVISIONING MENTAT SYSTEM'

vagrantenv/provisioning/p03_warden.sh

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Initial provisioning of Vagrant environment suitable for Mentat development.
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+source /vagrant/etc/default/mentat
+source /vagrant/conf/scripts/lib.sh
+
+print_title '<BEGIN> PROVISIONING WARDEN SYSTEM'
+
+print_subtitle 'Fetching Warden client library'
+if [ ! -d /opt/warden3 ] ; then
+  git clone https://homeproj.cesnet.cz/git/warden.git/ /opt/warden3
+fi
+
+print_subtitle 'Installing Warden client library'
+mkdir -p /etc/warden_client/warden_filer
+mkdir -p /run/warden_filer
+mkdir -p /var/lib/warden_filer
+chown -R mentat:mentat /run/warden_filer
+chown -R mentat:mentat /var/lib/warden_filer
+ensure_link /opt/warden3/warden_client/warden_client.py /usr/local/bin/warden_client.py
+ensure_link /opt/warden3/warden_filer/warden_filer.py /usr/local/bin/warden_filer.py
+ensure_link /opt/warden3/warden_ra/warden_apply.sh /usr/local/bin/warden_apply.sh
+chmod a+x /usr/local/bin/warden_client.py
+chmod a+x /usr/local/bin/warden_filer.py
+chmod a+x /usr/local/bin/warden_apply.sh
+ensure_link /vagrant/vagrantenv/warden/warden_filer /etc/default/warden_filer
+ensure_link /vagrant/vagrantenv/warden/warden_filer.cfg /etc/warden_client/warden_filer.cfg
+ensure_link /vagrant/vagrantenv/warden/warden_filer_receiver.service /etc/systemd/system/warden_filer_receiver.service
+systemctl daemon-reload
+
+print_title '<DONE> PROVISIONING WARDEN SYSTEM'

vagrantenv/.bashrc → vagrantenv/system/.bashrc

@@ -126,8 +126,8 @@ fi
 if [ ! -z "$SSH_CONNECTION" ]; then
     echo ""
     echo ""
-    if [ -x /vagrant/vagrantenv/system-banner.sh ]; then
-        /vagrant/vagrantenv/system-banner.sh
+    if [ -x /vagrant/vagrantenv/system/login-banner.sh ]; then
+        /vagrant/vagrantenv/system/login-banner.sh
     fi
     echo ""
 fi