Updated the installation manual with Git installation.

The installation manual was enhanced with the option of installing the project directly from Git repository. This option overlaps with the installation process for local development, so the development documentation page was also updated accordingly. (Redmine issue: #4216)

Makefile

@@ -362,14 +362,17 @@ deps-postgresql: FORCE
 
 
 ctrl-mentat-dev:
+	#APP_ROOT_PATH=$(shell realpath ./chroot) mentat-controller.py --command $(COMMAND)
 	mentat-controller.py --command $(COMMAND)
 
 run-mentat-dev:
 	@echo "\n$(GREEN)*** Running development version of Mentat system ***$(NC)\n"
+	#APP_ROOT_PATH=$(shell realpath ./chroot) mentat-controller.py --command start
 	mentat-controller.py --command start
 
 run-webui-dev:
 	@echo "\n$(GREEN)*** Running development web server with development configuration on $(DEV_SERVER):$(DEV_PORT) ***$(NC)\n"
+	#APP_ROOT_PATH=$(shell realpath ./chroot) hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT)
 	hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT)
 
 run-mailserver-dev:

doc/sphinx/_doclib/development.rst

@@ -22,14 +22,8 @@ Getting the code
 --------------------------------------------------------------------------------
 
 We are using the `Git <https://git-scm.com/>`__ SCM system to manage our codebase.
-Use following command to clone the repository without the push privileges::
-
-	$ git clone --recurse-submodules https://homeproj.cesnet.cz/git/mentat-ng.git/ mentat-ng
-	$ cd mentat-ng
-
-Please note the use of ``--recurse-submodules`` option. We are using submodules
-in our project and some options may be unavailable to you without them present
-(like generating documentation locally).
+Please follow instructions in section :ref:`section-installation-git` to install
+the project into your desired location.
 
 
 General guidelines

doc/sphinx/_doclib/installation.rst

@@ -4,7 +4,9 @@ Installation
 ================================================================================
 
 
-The Mentat system is available as a package for Debian based systems via our custom
+The Mentat system is available as native Python package from PyPI. However there
+is only so much you can do with setuptools. To make the installation more convenient
+for users we are also providing package for Debian based systems via our custom
 internal repository. We are trying hard to keep the installation process as simple
 as possible, however at the same time we try not to force too many decisions on the
 users of the system. For that reason the Debian package for Mentat system does not
@@ -23,6 +25,7 @@ to your setup and preferences.
 Currently you have following installation options:
 
 #. :ref:`section-installation-manual`
+#. :ref:`section-installation-git`
 #. :ref:`section-installation-ansible`
 
 .. note::
@@ -151,142 +154,145 @@ Installation guide - Manual installation
 The installation of base system is pretty straightforward, please follow these
 simple steps:
 
-
 .. code-block:: shell
 
     # Step 1: Install and configure PostgreSQL database. See official documentation or use our Ansible role.
 
     # Step 2: Download apt list file for mentat production package repository:
-    wget -O /etc/apt/sources.list.d/mentat-production.list https://alchemist.cesnet.cz/mentat/deb/mentat-production.list
+    $ wget -O /etc/apt/sources.list.d/mentat-production.list https://alchemist.cesnet.cz/mentat/deb/mentat-production.list
 
     # Step 3: Download and install GnuPG signing key for mentat production package repository:
-    wget -O - https://alchemist.cesnet.cz/mentat/deb/mentat-production.gpg | apt-key add -
+    $ wget -O - https://alchemist.cesnet.cz/mentat/deb/mentat-production.gpg | apt-key add -
 
     # Step 4: Refresh apt cache:
-    apt-get update
+    $ apt-get update
 
     # Step 5: List all packages available in newly added repository (optional):
-    grep "Package:" /var/lib/apt/lists/alchemist.cesnet.cz_*mentat*_Packages | uniq
+    $ grep "Package:" /var/lib/apt/lists/alchemist.cesnet.cz_*mentat*_Packages | uniq
 
     # Step 6: Install the Mentat system:
-    apt-get install mentat-ng
+    $ apt-get install mentat-ng
 
-    # Step 7: Install additional Python requirements:
-    pip3 install -r /etc/mentat/requirements.pip --upgrade
+After these steps there are several post-installation tasks to be performed:
 
-    # Step 8: Download IP geolocation databases:
-    /etc/mentat/scripts/fetch-geoipdb.sh
+.. code-block:: shell
 
-    # Step 9: Create necessary databases and database user accounts:
-    /etc/mentat/scripts/sqldb-init.sh
+    # Step 8: Initialize PostgreSQL database schema:
+    $ mentat-dbmngr.py --command init
 
-    # Step 10: Initialize PostgreSQL database schema:
-    mentat-dbmngr.py --command init
+    # Step 9: Stamp both metadata end event databases with latest migration revisions:
+    $ /etc/mentat/scripts/sqldb-migrate.py db stamp head --directory /etc/mentat/migrations
+    $ /etc/mentat/scripts/sqldb-migrate-e.sh stamp head
 
-    # Step 11: Stamp both metadata end event databases with latest migration revisions:
-    /etc/mentat/scripts/sqldb-migrate.py db stamp head --directory /etc/mentat/migrations
-    /etc/mentat/scripts/sqldb-migrate-e.sh stamp head
+    # Step 10: Precache various event search form select option dictionaries:
+    $ mentat-precache.py --allow-empty
 
-    # Step 12: Precache various event search form select option dictionaries:
-    mentat-precache.py --allow-empty
+The Mentat system is now successfully installed. You may try to start everything up:
 
+.. code-block:: shell
 
-If you wish to use the web interface Hawat, you wil have to install and configure
-a web server. You may use any server you like as long as it supports execution of
-WSGI Python3 scripts. Do not forget to setup some initial user accounts, either
-use the demonstration fixtures, or setup real user accounts, otherwise you will
-not be able to log in.
+    $ mentat-controller.py
+    $ mentat-controller.py --command start
+    $ hawat-cli run --host localhost --port 5000
 
-.. code-block:: shell
+The ``hawat-cli run`` command in the example above launches built-in development
+web server providing the *Hawat* web interface. It should be used only for demonstration
+or development purposes and not accessible from the outside world. You must use
+a production level webserver like Apache2 for serving the web interface, please
+refer to section :ref:`section-installation-apache` below for more details.
 
-    # Step 12: Install Apache web server:
-    apt-get install apache2 libapache2-mod-wsgi-py3
 
-    # Step 13: Copy example Apache site configuration file and adjust it according to your needs:
-    cp /etc/mentat/apache/site_mentat.conf.htpasswd.example /etc/apache2/sites-available/site_mentat.conf
+.. _section-installation-git:
 
-    # Step 14: Configure Apache2 to run as mentat user in '/etc/apache2/envvars' file:
-    sed -i.bak s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=mentat/g /etc/apache2/envvars
-    sed -i.bak s/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=mentat/g /etc/apache2/envvars
-    rm /etc/apache2/envvars.bak
+Installation guide - Installation from Git repository
+--------------------------------------------------------------------------------
 
-    # Step 15: Enable/disable appropriate Apache sites and modules:
-    a2enmod rewrite
-    a2enmod ssl
-    a2enmod wsgi
-    a2dissite 000-default.conf
-    a2dissite default-ssl.conf
-    a2ensite site_mentat.conf
+This type of installation is useful mainly either for developers, or for deploying
+the application for testing/debugging/demonstration purposes. Prior to installation
+you must provide following prerequisites on the target host system:
 
-    # !!!!! THIS STEP IS SUPER IMPORTANT !!!!!
-    # Step 16: Change default SECRET_KEY in /etc/mentat/mentat-hawat.py.conf:
-    # !!!!! THIS STEP IS SUPER IMPORTANT !!!!!
-    vim /etc/mentat/mentat-hawat.py.conf
+* Python3>=3.5.3
+* pip
+* PostgreSQL 11
+* `Yarn <https://yarnpkg.com/en/>`__
+* `Grunt <https://gruntjs.com/>`__
 
-    # Step 17: Restart Apache service:
-    systemctl restart apache2.service
+The installation of base system is pretty straightforward, please follow these
+simple steps:
 
-    # Step 18: OPTION A: Insert demonstration data into Mentat`s metadata database
-    # (this will insert some demo accounts, groups, networks and filters into db
-    # and can be used in conjuction with htpasswd file above):
-    mentat-dbmngr.py --command fixtures-add
+.. code-block:: shell
 
-    # Step 19: OPTION B: Create initial user account/s in Mentat`s metadata database
-    # (please adjust the attributes, do not simply copy and paste):
-    mentat-dbmngr.py --command user-add login=superman "fullname=Clark Kent" email=kent@dailyplanet.com "organization=Daily Planet, inc." roles=user,admin
+    # Step 1: Install and configure PostgreSQL database (please refer to the official documentation or use our Ansible role).
 
-.. warning::
+    # Step 2: Install Yarn package manager for Node.JS (please refer to the official documentation).
 
-    For demonstration purposes Mentat package ships with preconfigured ``htpasswd``
-    file containing following user accounts (login - password):
+    # Step 3: Install Grunt task manager (please refer to the official documentation).
 
-        * user - user
-        * manager - manager
-        * developer - developer
-        * admin - admin
+    # Step 4: Clone the git repository. Please note, that following command uses
+    # HTTPS and you will therefore have read-only access to the repository. In
+    # case you will be working and contributing code, you must either send patches
+    # via email, or be granted read-write access to our repository. In that case
+    # the repository URL will be something like [username]@homeproj.cesnet.cz:mentat-ng,
+    # please adjust following command accordingly.
+    $ git clone --recurse-submodules https://homeproj.cesnet.cz/git/mentat-ng.git/ mentat-ng
+    $ cd mentat-ng
 
-    This can be used in conjunction with database fixtures above (step 17, option A)
-    to immediately start exploring the Mentat system. Please make sure NOT to use
-    it in production environment!!!
+    # Step 5: Switch to development branch (optional, depending on the use case)
+    $ git checkout devel
 
-Following recipe is another example installation of Apache web server, this time
-with `Shibboleth SSO <https://www.shibboleth.net/index/basic/>`__ login service
-(this SSO login service is used for example in Czech
-`eduID identity federation <https://www.eduid.cz/en/index>`__):
+    # Step 6: Check for presence of all prerequisites:
+    $ make deps-prerequisites
+
+    # Step 7: Bootstrap Python virtual environment for development:
+    $ make venv
+
+    # Step 8: Activate virtual environment before any further work:
+    $ . venv/bin/activate
+
+    # Step 9: Perform the installation magic:
+    (venv) $ make develop
+
+After these steps there are several post-installation tasks to be performed:
 
 .. code-block:: shell
 
-    # Step 12: Install Apache web server:
-    apt-get install apache2 libapache2-mod-wsgi-py3 libapache2-mod-shib2
+    # Do not forget to activate the virtual environment:
+    $ . venv/bin/activate
 
-    # Step 13: Copy example Apache site configuration file and adjust it according to your needs:
-    cp /etc/mentat/apache/site_mentat.conf.shibboleth.example /etc/apache2/sites-available/site_mentat.conf
+    # Step 10: Initialize PostgreSQL database schema:
+    (venv) $ mentat-dbmngr.py --command init
 
-    # Step 14: Configure Apache2 to run as mentat user in '/etc/apache2/envvars' file:
-    sed -i.bak s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=mentat/g /etc/apache2/envvars
-    sed -i.bak s/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=mentat/g /etc/apache2/envvars
-    rm /etc/apache2/envvars.bak
+    # Step 11: Stamp both metadata end event databases with latest migration revisions:
+    (venv) $ ./scripts/sqldb-migrate.py db stamp head --directory ./migrations-events
+    (venv) $ ./scripts/sqldb-migrate-e.sh stamp head
 
-    # Step 15: Enable/disable appropriate Apache sites and modules:
-    a2enmod rewrite
-    a2enmod ssl
-    a2enmod shib2
-    a2enmod wsgi
-    a2dissite 000-default.conf
-    a2dissite default-ssl.conf
-    a2ensite site_mentat.conf
+    # Step 12: Precache various event search form select option dictionaries:
+    (venv) $ mentat-precache.py --allow-empty
 
-    # !!!!! THIS STEP IS SUPER IMPORTANT !!!!!
-    # Step 16: Change default SECRET_KEY in /etc/mentat/mentat-hawat.py.conf:
-    # !!!!! THIS STEP IS SUPER IMPORTANT !!!!!
-    vim /etc/mentat/mentat-hawat.py.conf
+The Mentat system is now successfully installed. You may try to start everything up:
 
-    # Step 17: Restart Apache service:
-    systemctl restart apache2.service
+.. code-block:: shell
 
-    # Step 18: Create initial user account/s in Mentat`s metadata database
-    # (please adjust the attributes, do not simply copy and paste):
-    mentat-dbmngr.py --command user-add login=superman "fullname=Clark Kent" email=kent@dailyplanet.com "organization=Daily Planet, inc." roles=user,admin
+    # Do not forget to activate the virtual environment:
+    $ . venv/bin/activate
+
+    (venv) $ mentat-controller.py
+    (venv) $ mentat-controller.py --command start
+    (venv) $ hawat-cli run --host localhost --port 5000
+
+The ``hawat-cli run`` command in the example above launches built-in development
+web server providing the *Hawat* web interface. It should be used only for demonstration
+or development purposes and not accessible from the outside world. You must use
+a production level webserver like Apache2 for serving the web interface, please
+refer to section :ref:`section-installation-apache` below for more details.
+
+The whole application uses the ``./chroot`` subdirectory of current working
+directory as a kind of lightweight chroot jail and places all runtime files
+inside it.
+
+Please also note the existence of ``.env`` configuration file, that was created for
+you during the installation. This file contains various local custom configurations,
+that are loaded using ``python-dotenv``.
 
 
 .. _section-installation-ansible:
@@ -379,6 +385,117 @@ And voilà! Your Mentat demo server is up and running!
     duplicate the information.
 
 
+.. _section-installation-apache:
+
+Installation guide - Web interface with Apache2
+--------------------------------------------------------------------------------
+
+If you wish to use the web interface Hawat, you wil have to install and configure
+a web server. You may use any server you like as long as it supports execution of
+WSGI Python3 scripts. Do not forget to setup some initial user accounts, either
+use the demonstration fixtures, or setup real user accounts. Otherwise you will
+not be able to log in.
+
+.. code-block:: shell
+
+    # Step 1: Install Apache web server:
+    apt-get install apache2 libapache2-mod-wsgi-py3
+
+    # Step 2: Copy example Apache site configuration file and adjust it according to your needs:
+    cp /etc/mentat/apache/site_mentat.conf.htpasswd.example /etc/apache2/sites-available/site_mentat.conf
+    # Or version for installations with git:
+    cp ./conf/apache/site_mentat.conf.htpasswd.example /etc/apache2/sites-available/site_mentat.conf
+
+    # Step 3: Configure Apache2 to run as mentat user in '/etc/apache2/envvars' file:
+    sed -i.bak s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=mentat/g /etc/apache2/envvars
+    sed -i.bak s/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=mentat/g /etc/apache2/envvars
+    rm /etc/apache2/envvars.bak
+
+    # Step 4: Enable/disable appropriate Apache sites and modules:
+    a2enmod rewrite
+    a2enmod ssl
+    a2enmod wsgi
+    a2dissite 000-default.conf
+    a2dissite default-ssl.conf
+    a2ensite site_mentat.conf
+
+    # !!!!! THIS STEP IS SUPER IMPORTANT !!!!!
+    # Step 5: Change default SECRET_KEY in /etc/mentat/mentat-hawat.py.conf:
+    # !!!!! THIS STEP IS SUPER IMPORTANT !!!!!
+    vim /etc/mentat/mentat-hawat.py.conf
+    # Or version for installations with git:
+    vim ./conf/mentat-hawat.py.conf
+
+    # Step 6: Restart Apache service:
+    systemctl restart apache2.service
+
+    # Step 7: OPTION A: Insert demonstration data into Mentat`s metadata database
+    # (this will insert some demo accounts, groups, networks and filters into db
+    # and can be used in conjuction with htpasswd file above):
+    mentat-dbmngr.py --command fixtures-add
+
+    # Step 8: OPTION B: Create initial user account/s in Mentat`s metadata database
+    # (please adjust the attributes, do not simply copy and paste):
+    mentat-dbmngr.py --command user-add login=superman "fullname=Clark Kent" email=kent@dailyplanet.com "organization=Daily Planet, inc." roles=user,admin
+
+.. warning::
+
+    For demonstration purposes Mentat package ships with preconfigured ``htpasswd``
+    file containing following user accounts (login - password):
+
+        * user - user
+        * manager - manager
+        * developer - developer
+        * admin - admin
+
+    This can be used in conjunction with database fixtures above (step 17, option A)
+    to immediately start exploring the Mentat system. Please make sure NOT to use
+    it in production environment!!!
+
+Following recipe is another example installation of Apache web server, this time
+with `Shibboleth SSO <https://www.shibboleth.net/index/basic/>`__ login service
+(this SSO login service is used for example in Czech
+`eduID identity federation <https://www.eduid.cz/en/index>`__):
+
+.. code-block:: shell
+
+    # Step 1: Install Apache web server:
+    apt-get install apache2 libapache2-mod-wsgi-py3 libapache2-mod-shib2
+
+    # Step 2: Copy example Apache site configuration file and adjust it according to your needs:
+    cp /etc/mentat/apache/site_mentat.conf.shibboleth.example /etc/apache2/sites-available/site_mentat.conf
+    # Or version for installations with git:
+    cp ./conf/apache/site_mentat.conf.shibboleth.example /etc/apache2/sites-available/site_mentat.conf
+
+    # Step 3: Configure Apache2 to run as mentat user in '/etc/apache2/envvars' file:
+    sed -i.bak s/APACHE_RUN_USER=www-data/APACHE_RUN_USER=mentat/g /etc/apache2/envvars
+    sed -i.bak s/APACHE_RUN_GROUP=www-data/APACHE_RUN_GROUP=mentat/g /etc/apache2/envvars
+    rm /etc/apache2/envvars.bak
+
+    # Step 4: Enable/disable appropriate Apache sites and modules:
+    a2enmod rewrite
+    a2enmod ssl
+    a2enmod shib2
+    a2enmod wsgi
+    a2dissite 000-default.conf
+    a2dissite default-ssl.conf
+    a2ensite site_mentat.conf
+
+    # !!!!! THIS STEP IS SUPER IMPORTANT !!!!!
+    # Step 5: Change default SECRET_KEY in /etc/mentat/mentat-hawat.py.conf:
+    # !!!!! THIS STEP IS SUPER IMPORTANT !!!!!
+    vim /etc/mentat/mentat-hawat.py.conf
+    # Or version for installations with git:
+    vim ./conf/mentat-hawat.py.conf
+
+    # Step 6: Restart Apache service:
+    systemctl restart apache2.service
+
+    # Step 7: Create initial user account/s in Mentat`s metadata database
+    # (please adjust the attributes, do not simply copy and paste):
+    mentat-dbmngr.py --command user-add login=superman "fullname=Clark Kent" email=kent@dailyplanet.com "organization=Daily Planet, inc." roles=user,admin
+
+
 Directory layout
 --------------------------------------------------------------------------------
 

packaging/debian/postinst

@@ -111,3 +111,9 @@ find /etc/mentat/cron/ -name "*.cron" -exec chmod 644 {} \;
 # Make sure all executable scripts have proper permissions
 find /etc/mentat/scripts/ -name "*.py" -exec chmod 755 {} \;
 find /etc/mentat/scripts/ -name "*.sh" -exec chmod 755 {} \;
+
+# Create PostgreSQL database
+/etc/mentat/scripts/sqldb-init.sh
+
+# Fetch fresh IP geolocation databases
+/etc/mentat/scripts/fetch-geoipdb.sh