From 8ea53be9ce323be0820d7f713737e4bf997debc3 Mon Sep 17 00:00:00 2001
From: Jan Mach <jan.mach@cesnet.cz>
Date: Fri, 15 Feb 2019 19:31:38 +0100
Subject: [PATCH] Huge revision of master makefile and development
documentation page.
Documentation, build procedures...almost everything was revised and rethought. (Redmine issue: #4216,#3387,#3361)
---
Makefile | 90 +++++++-
doc/sphinx/_doclib/development.rst | 356 +++++++++++++++++++++++------
2 files changed, 363 insertions(+), 83 deletions(-)
diff --git a/Makefile b/Makefile
index 437b6b5eb..7a8e66e59 100644
--- a/Makefile
+++ b/Makefile
@@ -39,8 +39,11 @@ include Makefile.cfg
# Color code definitions for colored terminal output.
#
# Resource:
-# https://stackoverflow.com/questions/5947742/how-to-change-the-output-color-of-echo-in-linux
+# https://stackoverflow.com/questions/5947742/how-to-change-the-output-color-of-echo-in-linux
+# https://www.tldp.org/HOWTO/Bash-Prompt-HOWTO/x405.html
+# https://linux.die.net/man/1/tput
#
+
RED=\033[0;31m
GREEN=\033[0;32m
ORANGE=\033[0;33m
@@ -69,7 +72,7 @@ help:
@echo " â–â–â•‘â•šâ–â–╔╝â–â–â•‘â–â–â•”â•â•â•ť â–â–â•‘â•šâ–â–â•—â–â–â•‘ â–â–â•‘ â–â–â•”â•â•â–â–â•‘ â–â–â•‘"
@echo " â–â–â•‘ â•šâ•â•ť â–â–â•‘â–â–â–â–â–â–â–â•—â–â–â•‘ â•šâ–â–â–â–â•‘ â–â–â•‘ â–â–â•‘ â–â–â•‘ â–â–â•‘"
@echo " â•šâ•â•ť â•šâ•â•ťâ•šâ•â•â•â•â•â•â•ťâ•šâ•â•ť â•šâ•â•â•â•ť â•šâ•â•ť â•šâ•â•ť â•šâ•â•ť â•šâ•â•ť"
- @echo " $(FAINT)Copyright (C) since 2011 CESNET, z.s.p.o$(NC)"
+ @echo " $(FAINT)Copyright (C) since 2011 CESNET, z.s.p.o$(NC)"
@echo ""
@echo " $(GREEN)$(BOLD)â•”â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•—$(NC)"
@echo " $(GREEN)$(BOLD)â•‘ LIST OF AVAILABLE MAKE TARGETS â•‘$(NC)"
@@ -80,11 +83,12 @@ help:
@echo " * $(GREEN)default$(NC): alias for help, you have to pick a target"
@echo " * $(GREEN)help$(NC): print this help message and exit"
@echo " * $(GREEN)show-version$(NC): show current project version"
+ @echo " * $(GREEN)show-envstamp$(NC): show information about current development environment"
@echo " * $(GREEN)develop$(NC): install and configure project locally for development"
@echo " * $(GREEN)deps$(NC): install project dependencies"
@echo " * $(GREEN)clean$(NC): cleanup development and build environment"
@echo " * $(GREEN)docs$(NC): generate project documentation"
- @echo " * $(GREEN)check$(NC): perform extensive checks and tests"
+ @echo " * $(GREEN)check$(NC): perform all checks and tests"
@echo " * $(GREEN)build-whl$(NC): perform local build of Python distribution package"
@echo " * $(GREEN)build-deb$(NC): perform local build of Debian distribution package"
@echo ""
@@ -112,16 +116,17 @@ help:
@echo " * $(GREEN)presentations$(NC): project presentations"
@echo ""
@echo " * $(GREEN)pybabel-patch$(NC): patch babel library"
- @echo " * ${GREEN}hpybabel-init INIT_LOCALE=lc${NC}: init Hawat translations for given locale lc"
+ @echo " * ${GREEN}hpybabel-init INIT_LOCALE=lc${NC}: init Hawat translations for given locale $(FAINT)lc$(NC)"
@echo " * $(GREEN)hpybabel-pull$(NC): extract and update Hawat translations"
@echo " - $(ORANGE)hpybabel-extract$(NC): extract Hawat translations"
@echo " - $(ORANGE)hpybabel-update$(NC): update Hawat translations"
@echo " * $(GREEN)hpybabel-compile$(NC): compile Hawat translations"
- @echo " * ${GREEN}mpybabel-init INIT_LOCALE=lc${NC}: init Mentat translations for given locale lc"
+ @echo " * ${GREEN}mpybabel-init INIT_LOCALE=lc${NC}: init Mentat translations for given locale $(FAINT)lc$(NC)"
@echo " * $(GREEN)mpybabel-pull$(NC): extract and update Mentat translations"
@echo " - $(ORANGE)mpybabel-extract$(NC): extract Mentat translations"
@echo " - $(ORANGE)mpybabel-update$(NC): update Mentat translations"
@echo " * $(GREEN)mpybabel-compile$(NC): compile Mentat translations"
+ @echo " * $(GREEN)translations-compile$(NC): compile all available translations"
@echo ""
@echo " * $(GREEN)pyflakes$(NC): check project with pyflakes"
@echo " * $(GREEN)pylint$(NC): check project with pylint"
@@ -145,6 +150,19 @@ help:
show-version: FORCE
@PYTHONPATH=lib $(PYTHON) -c "import mentat; print(mentat.__version__);"
+#
+# Show information about current development environment. This may be very handy
+# be executed on some automated build systems to determine the state of the build
+# environment prior to the build.
+#
+show-envstamp: FORCE
+ @echo "System info: `uname -a`"
+ @echo "Pip version: `$(PIP) --version`"
+ @echo "Python version: `$(PYTHON) --version`"
+ @echo "Project version: `PYTHONPATH=lib $(PYTHON) -c 'import mentat; print(mentat.__version__);'`"
+ @echo "Pip libraries:"
+ @$(PIP) freeze
+
#
# Install and configure project locally for development. This target will perform
# following tasks for you:
@@ -194,7 +212,7 @@ develop: FORCE
#
# Install and configure project dependencies.
#
-deps: deps-prerequisites deps-python deps-python-dev deps-webui deps-geoip deps-negistry deps-postgresql deps-translations
+deps: deps-prerequisites deps-python deps-python-dev deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
#
# Cleanup development and build environment.
@@ -225,6 +243,14 @@ build-deb: clean build-webui build-package-deb
#===============================================================================
+#
+# Check for development prerequisites. The prerequisites are certain commands and
+# applications that have to be already installed on local system, otherwise the
+# installation can not proceed further. These prerequisites ussually require more
+# complex installation process, or their installation is not straightforward, or
+# there are multiple installation procedures and it is not possible to choose the
+# best option. In any case, it is best to leave the installation to the user.
+#
deps-prerequisites: FORCE
@echo "\n$(GREEN)*** Checking for development prerequisites ***$(NC)\n"
@for prereq in $(PYTHON) $(PIP) yarn grunt psql ; do \
@@ -238,63 +264,99 @@ deps-prerequisites: FORCE
done
@echo ""
+#
+# Install project`s Python dependencies using pip requirements file. The dependencies
+# are already listed in setup.py file and pip can install them automatically. It
+# is however better to use requirements file directly, because its syntax enables
+# users to provide addditonal options to pip binary and thus enable for example
+# an installation of binary packages. It is much more powerfull than simple syntax
+# of 'install_requires' keyword of 'setup.py'.
+#
deps-python: FORCE
@echo "\n$(GREEN)*** Installing Python dependencies ***$(NC)\n"
@$(PIP) --version
@$(PIP) install -r conf/requirements.pip
@echo ""
+#
+# Install project`s Python development dependencies using pip requirements file.
+# These dependencies are essential for development, but not required for production
+# deployment. For more information on why to use pip requirements file explicitly
+# instead of letting it install the dependencies from the list in 'setup.py' file
+# please see the documentation of 'deps-python' target above.
+#
deps-python-dev: FORCE
@echo "\n$(GREEN)*** Installing Python development dependencies ***$(NC)\n"
@$(PIP) --version
@$(PIP) install -r conf/requirements-dev.pip
@echo ""
+#
+# Upgrade project`s Python dependencies using pip requirements file to latest
+# versions.
+#
deps-python-upgrade: FORCE
@echo "\n$(GREEN)*** Upgrading Python dependencies to latest versions ***$(NC)\n"
@$(PIP) --version
@$(PIP) install -r conf/requirements-latest.pip --upgrade
@echo ""
+#
+# Upgrade project`s Python development dependencies using pip requirements file
+# to latest versions.
+#
deps-python-upgrade-dev: FORCE
@echo "\n$(GREEN)*** Upgrading Python development dependencies to latest versions ***$(NC)\n"
@$(PIP) --version
@$(PIP) install -r conf/requirements-latest-dev.pip --upgrade
@echo ""
+#
+# Install project`s web interface dependencies using yarn.
+#
deps-webui: FORCE
@echo "\n$(GREEN)*** Installing web interface dependencies ***$(NC)\n"
@yarn install
@echo ""
+#
+# Upgrade project`s web interface dependencies using yarn to latest versions.
+#
deps-webui-upgrade: FORCE
@echo "\n$(GREEN)*** Upgrading web interface dependencies ***$(NC)\n"
@yarn upgrade
@echo ""
+#
+# Fetch and install IP geolocation databases.
+#
deps-geoip: FORCE
@echo "\n$(GREEN)*** Installing IP geolocation databases ***$(NC)\n"
@./scripts/fetch-geoipdb.sh
@echo ""
+#
+# Fetch and install Negistry whois database.
+#
deps-negistry: FORCE
@echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n"
@./scripts/fetch-negistry.sh --stub
@echo ""
+#
+# Create and configure required PostgreSQL user accounts and databases.
+#
deps-postgresql: FORCE
@echo "\n$(GREEN)*** Installing and configuring PostgreSQL database ***$(NC)\n"
@./scripts/sqldb-init.sh
@echo ""
-deps-translations: hpybabel-compile mpybabel-compile
-
#-------------------------------------------------------------------------------
clean-pycs: FORCE
- @echo "\n$(GREEN)*** Cleaning up Python compiled files ***$(NC)\n"
+ @echo "\n$(GREEN)*** Cleaning up Python precompiled files ***$(NC)\n"
@find . -name '*.pyc' -delete
@find . -name '*.pyo' -delete
@find . -name '*~' -delete
@@ -362,7 +424,7 @@ presentations: FORCE
#
# This patch solves following issue: https://github.com/python-babel/flask-babel/issues/43
# Apply when necessary.
-
+#
pybabel-patch: FORCE
@echo "\n$(GREEN)*** Patching babel library ***$(NC)\n"
@cp util/babel.messages.frontend.py.patch /var/tmp/
@@ -375,7 +437,6 @@ pybabel-patch: FORCE
# 1. Enter directory containing the messages.pot file.
# 2. pybabel init -i messages.pot -d translations -l [locale]
#
-
hpybabel-extract: FORCE
@echo "\n$(GREEN)*** Extracting babel translations for Hawat user interface ***$(NC)\n"
@cd $(DIR_LIB_HAWAT) && $(PYBABEL) extract -F babel.cfg -o messages.pot -k lazy_gettext -k tr_ .
@@ -425,6 +486,11 @@ mpybabel-compile: FORCE
@$(PYBABEL) compile -d $(DIR_TEMPLATES_REPORTER)/translations
@$(PYBABEL) compile -d $(DIR_TEMPLATES_UTEST)/translations
+#
+# Compile all available translations.
+#
+translations-compile: hpybabel-compile mpybabel-compile
+
#-------------------------------------------------------------------------------
@@ -474,7 +540,7 @@ build-package-whl: FORCE
build-package-deb: FORCE
@echo "\n${GREEN}*** Building Debian packages ***${NC}\n"
- @$(PYTHON) grunt build
+ @grunt deb-build
@echo ""
# Empty rule as dependency will force make to always perform target
diff --git a/doc/sphinx/_doclib/development.rst b/doc/sphinx/_doclib/development.rst
index 897fa7c03..23d2f23fb 100644
--- a/doc/sphinx/_doclib/development.rst
+++ b/doc/sphinx/_doclib/development.rst
@@ -18,6 +18,20 @@ Key information
* `Automated build system (Alchemist) <https://alchemist.cesnet.cz>`__
+Getting the code
+--------------------------------------------------------------------------------
+
+We are using the `Git <https://git-scm.com/>`__ SCM system to manage our codebase.
+Use following command to clone the repository without the push privileges::
+
+ $ git clone --recurse-submodules https://homeproj.cesnet.cz/git/mentat-ng.git/ mentat-ng
+ $ cd mentat-ng
+
+Please note the use of ``--recurse-submodules`` option. We are using submodules
+in our project and some options may be unavailable to you without them present
+(like generating documentation locally).
+
+
General guidelines
--------------------------------------------------------------------------------
@@ -49,19 +63,25 @@ commands::
$ make
$ make help
+Of course you need to have *make* utility installed on your system, on Debian-based
+system you can use following command::
-Preparing development environment
+ $ aptitude install make
+
+
+Development prerequisites
````````````````````````````````````````````````````````````````````````````````
There are several development prerequisites, that already have to be present on
-your development machine. These prerequisites are not installed automatically
-for you, because the installation is too complex. There prerequisites are:
+your development machine. These prerequisites are not installed automatically for
+you, because the installation is too complex, too customizable or simply best to
+be performed by the user himself. These prerequisites currently are:
-* `Python 3 <https://www.python.org/>`__: Please use version similar to current stable Python3 release on current stable Debian release.
-* `Pip <https://pip.pypa.io/en/stable/>`__: Python package manager, we recommend installation with `get-pip.py <https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py>`__.
-* `Yarn <https://yarnpkg.com/en/>`__: NPM package manager for web interface libraries.
-* `Grunt <https://gruntjs.com/>`__: JavaScript task runner for web interface development.
-* `PostgreSQL 11 <https://www.postgresql.org/>`__: Relational database, please use version 11 wherever possible.
+* `Python 3 <https://www.python.org/>`__: Please use version similar to current stable Python3 release on current stable Debian release (`current <https://packages.debian.org/stretch/python3>`__).
+* `Pip <https://pip.pypa.io/en/stable/>`__: Python package manager, we recommend installation with `get-pip.py <https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py>`__ to get the latest release, you may sometimes encounter weird bugs when installing newer packages with old versions of pip.
+* `Yarn <https://yarnpkg.com/en/>`__: NPM package manager for web interface libraries. This package manager is responsible for managing frontend development libraries like jQuery, Bootstrap, D3 etc.
+* `Grunt <https://gruntjs.com/>`__: JavaScript task runner for web interface development. It is responsible for taking care of frontend related tasks like JavaScript and CSS minification
+* `PostgreSQL 11 <https://www.postgresql.org/>`__: Relational database, used for persistent data storage. Please use version 11 wherever possible.
You can check for presence of all of these dependencies with this handy make target:
@@ -70,25 +90,210 @@ You can check for presence of all of these dependencies with this handy make tar
# Check for presence of all prerequisites:
$ make deps-prerequisites
-Now please execute the following command from the root directory of the repository to
-initialize correct Python virtual environment, install all requirements (including
-those only for development) and finally install the project locally in editable mode:
+In case you get any errors please follow the official documentation to install the
+missing prerequisite.
+
+
+Preparing development environment
+````````````````````````````````````````````````````````````````````````````````
+
+If you want to participate on the development of this project there is a handy
+make target that will make all preparations for you to get you started as soon as
+possible. Please execute the following command from the root directory of the
+cloned repository to initialize correct Python virtual environment, install all
+requirements (including those required only for development) and finally install
+the project locally in editable mode:
.. code-block:: shell
- # Perform all installations:
+ # Perform the installation magic:
$ make develop
# Activate virtual environment before any development work:
$ . venv/bin/activate
- # Now from within the virtual environment install all required dependencies:
+ # Now from within the virtual environment install all required dependencies.
+ # This step is required, because project can have some additional dependencies
+ # or there might be some initialization steps that need to be taken, that are
+ # not possible to do with pip:
(venv) $ make deps
- # Deactivate virtual environment when not needed with:
+ # Deactivate virtual environment when it is not needed anymore with:
(venv) $ deactivate
+Dependencies
+````````````````````````````````````````````````````````````````````````````````
+
+There is a number of makefile targets called ``deps`` and ``deps-`` that are responsible
+for helping with dependency management. Please study the ``make help`` output to
+view the list of available targets.
+
+If your code requires some additional third party dependencies please follow these
+procedures:
+
+* Debian dependencies
+
+ * Debian dependencies must be specified in the ``deploy/mentat/ctrl/control.tmpl``
+ file. Please study the `documentation <https://www.debian.org/doc/debian-policy/ch-controlfields.html#binary-package-control-files-debian-control>`__ to correctly
+ use Debian dependency system.
+
+* Python dependencies
+
+ * Use `pip <https://pip.pypa.io/en/stable/reference/>`__ command to manage Python
+ dependencies.
+ * Dependencies that are always required must be listed in ``conf/requirements.pip``
+ file (including version) and in ``conf/requirements-latest.pip`` file (without
+ version).
+ * Dependencies that are required only for development must be listed in ``conf/requirements-dev.pip``
+ file (including version) and in ``conf/requirements-latest-dev.pip`` file (without
+ version).
+ * In this project it is preferred to use pip for native Python dependencies. In
+ some cases you may however choose to use Debian package for pulling in the
+ required library instead.
+
+* Web interface dependencies
+
+ * Use `yarn <https://yarnpkg.com/en/docs/usage>`_ command to manage frontend
+ dependencies.
+
+Example workflow for adding Python dependency::
+
+ # Install library locally:
+ (venv) $ pip install flask
+
+ # Get the version of the library:
+ (venv) $ pip freeze | grep -i flask
+
+ # Now write the library name with version to `conf/requirements.pip` and
+ # without version to `conf/requirements-latest.pip`.
+
+ # In case the library is required only for development write the library name
+ # with version to `conf/requirements-dev.pip` and without version to
+ # `conf/requirements-latest-dev.pip`.
+
+ # Make sure the dependency gets installed also using the makefile target:
+ (venv) $ make deps-python
+ (venv) $ make deps-python-dev
+
+Example workflow for adding frontend dependency::
+
+ # Install dependency with yarn:
+ (venv) $ yarn add jquery
+
+ # Install development dependency with yarn:
+ (venv) $ yarn add grunt --dev
+
+ # Make sure the dependency gets installed also using the makefile target:
+ (venv) $ make deps-webui
+
+For upgrading all the dependencies to latest versions you may use following make
+targets::
+
+ # Activate virtual environment before any development work:
+ $ . venv/bin/activate
+
+ (venv) $ make deps-python-upgrade
+ (venv) $ make deps-python-upgrade-dev
+ (venv) $ make deps-webui-upgrade
+
+
+Documentation
+````````````````````````````````````````````````````````````````````````````````
+
+The project documentation consists of the part generated directly from the source
+code docstrings and of the part written manually. It is generated using the
+`Sphinx-doc <http://www.sphinx-doc.org/en/stable/contents.html>`__ tool into various
+formats. Please use `RST <http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`__
+markup features where appropriate to increase readability and cross-reference to
+related content. It should however still be possible to view the documentation of
+all Python modules in *Pythonic* way via `pydoc3 <https://docs.python.org/3/library/pydoc.html>`__
+and the result should still be more or less readable. Please test it immediately with:
+
+.. code-block:: shell
+
+ # Always make sure your virtual environment is activated:
+ $ . venv/bin/activate
+
+ # Run tests:
+ (venv) $ pydoc3 ./path/to/module.py
+
+You may generate and review the documentation locally by executing the following
+command:
+
+.. code-block:: shell
+
+ # Always make sure your virtual environment is activated:
+ $ . venv/bin/activate
+
+ # Run tests:
+ (venv) $ make docs
+
+ # View the documentation in your default web browser:
+ (venv) $ make docs-view
+
+ # When necessary you may also remove all documentation related artifacts and
+ # rebuild:
+ (venv) $ make clean-build-docs
+ (venv) $ make docs
+ (venv) $ make docs-view
+
+Documentation will be generated into ``doc/sphinx/_build/html/manual.html``.
+
+Important resources for you to study:
+
+* `pydoc3 <https://docs.python.org/3/library/pydoc.html>`__
+* `Sphinx-doc <http://www.sphinx-doc.org/en/stable/contents.html>`__
+
+ * `reStructuredText Primer <http://www.sphinx-doc.org/en/stable/rest.html>`__
+ * `Sphinx markup constructs <http://www.sphinx-doc.org/en/stable/markup/index.html>`__
+ * `The Python domain <http://www.sphinx-doc.org/en/stable/domains.html#the-python-domain>`__
+ * `Documenting functions and methods <http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists>`__
+
+
+Internationalization and translations
+````````````````````````````````````````````````````````````````````````````````
+
+The web interface and some other parts of the system are localized to provide best
+experience for target user. Following libraries are used to accomplish this task:
+
+* `pybabel <http://babel.pocoo.org/en/latest/index.html>`__
+* `flask-babel <https://pythonhosted.org/Flask-Babel/>`__
+
+The web interface translations are included in the :py:mod:`hawat` module. The most
+important files are following:
+
+* ``lib/hawat/babel.cfg`` - Babel configuration file
+* ``lib/hawat/messages.pot`` - Extracted translations, generated automatically
+* ``lib/hawat/translations/`` - Directory containing translations to various languages
+
+Strings in the python source code are marked for translation when you wrap them
+in one of the following functions: ``gettext()``, ``lazy_gettext()``, ``tr_()``.
+The last one is defined internally and is used for translating constants or enums.
+Strings in the Jinja2 templates are marked for translation when you wrap them with
+``gettext()`` or ``_()`` functions.
+
+After adding new strings into the web interface that will need translating please
+follow this procedure::
+
+ # Pull (extract and update) all translation strings into message catalogs:
+ (venv) $ make hpybabel-pull
+
+ # Now please edit the translation files. For example for czech locale please
+ # edit file ``lib/hawat/translations/cs/messages.po``.
+
+ # When you are happy with your translations compile the message catalogs with:
+ (venv) $ make hpybabel-compile
+
+Similarly the ``make mpybabel-pull`` and ``make hpybabel-compile`` targets can be
+used to compile translations of various Mentat reports. These are located in
+following directories:
+
+* ``conf/templates/informant/``: Report templates for :ref:`section-bin-mentat-informant`.
+* ``conf/templates/reporter/``: Report templates for :ref:`section-bin-mentat-reporter`.
+* ``conf/templates/utest/``: Templates for library unit tests.
+
+
Checking code with Pyflakes
````````````````````````````````````````````````````````````````````````````````
@@ -168,15 +373,13 @@ Important resources:
* `nosetests <http://nose.readthedocs.io/en/latest/>`__
-Documentation
+Building web interface
````````````````````````````````````````````````````````````````````````````````
-Project documentation is generated using the `Sphinx-doc <http://www.sphinx-doc.org/en/stable/contents.html>`__
-tool into various formats. Please use `RST <http://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html>`__
-markup features where appropriate to increase readability and cross-reference to
-related content. It should however still be possible to view the documentation of
-all Python modules in *Pythonic* way via `pydoc3 <https://docs.python.org/3/library/pydoc.html>`__
-and the result should still be more or less readable. Please test it immediately with:
+The web interface development requires certain specific tasks like copying third
+party libraries from ``node_modules`` directory to correct locations, JavaScript
+and CSS minifications etc. When you are developing web interface following makefile
+target will be very handy to you:
.. code-block:: shell
@@ -184,30 +387,57 @@ and the result should still be more or less readable. Please test it immediately
$ . venv/bin/activate
# Run tests:
- (venv) $ pydoc3 ./path/to/module.py
+ (venv) $ make build-webui
-You may generate and review the documentation locally by executing the following
-command:
-.. code-block:: shell
+Database schema migrations
+--------------------------------------------------------------------------------
- # Always make sure your virtual environment is activated:
- $ . venv/bin/activate
+Event database migrations
+````````````````````````````````````````````````````````````````````````````````
- # Run tests:
- (venv) $ make docs
+Due to the performance reasons the event database abstraction layer is implemented
+directly on top of the `psycopg2 <http://initd.org/psycopg/>`__ driver. To be
+consistent with metadata database migrations we are using separate configured instance
+of `Alembic <https://alembic.sqlalchemy.org/en/latest/index.html>`__ database
+migration utility. The migration environment is located in ``migrations-events``
+subdirectory.
-Documentation will be generated into ``doc/sphinx/_build/html/manual.html``.
+To create new migration during development follow these steps::
+
+ cd migrations-events
+ alembic revision -m "revision description"
+
+Now edit the generated revision file to suit your needs. You may wish to use
+following resources as reference:
+
+* `Operation Reference <https://alembic.sqlalchemy.org/en/latest/ops.html>`__
+* `Cookbook <https://alembic.sqlalchemy.org/en/latest/cookbook.html>`__
+
+Migration can be then invoked locally from within the migration environment directory::
+
+ cd migrations-events
+ alembic upgrade head
+ alembic history
+
+To enable execution of database migrations on target systems after installation
+from package there is a simple wrapper script ``/etc/mentat/scripts/sqldb-migrate-e.sh``::
+
+ /etc/mentat/scripts/sqldb-migrate-e.sh upgrade head
+ /etc/mentat/scripts/sqldb-migrate-e.sh history
Important resources:
-* `pydoc3 <https://docs.python.org/3/library/pydoc.html>`__
-* `Sphinx-doc <http://www.sphinx-doc.org/en/stable/contents.html>`__
+* `Alembic <https://alembic.sqlalchemy.org/en/latest/index.html>`__
- * `reStructuredText Primer <http://www.sphinx-doc.org/en/stable/rest.html>`__
- * `Sphinx markup constructs <http://www.sphinx-doc.org/en/stable/markup/index.html>`__
- * `The Python domain <http://www.sphinx-doc.org/en/stable/domains.html#the-python-domain>`__
- * `Documenting functions and methods <http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists>`__
+
+Metadata database migrations
+````````````````````````````````````````````````````````````````````````````````
+
+Important resources:
+
+* `Alembic <https://alembic.sqlalchemy.org/en/latest/index.html>`__
+* `Flask-Migrate <https://flask-migrate.readthedocs.io/en/latest/>`__
Versioning
@@ -258,54 +488,38 @@ the tag messages.
$ git log --numstat --pretty="%H" v1.0.0..v0.0.1 | awk 'NF==3 {plus+=$1; minus+=$2} END {printf("+%d, -%d\n", plus, minus)}'
-Database schema migrations
+Building Python packages
--------------------------------------------------------------------------------
-Event database migrations
-````````````````````````````````````````````````````````````````````````````````
+If you want to build native Python packages locally please use following makefile
+target:
-Due to the performance reasons the event database abstraction layer is implemented
-directly on top of the `psycopg2 <http://initd.org/psycopg/>`__ driver. To be
-consistent with metadata database migrations we are using separate configured instance
-of `Alembic <https://alembic.sqlalchemy.org/en/latest/index.html>`__ database
-migration utility. The migration environment is located in ``migrations-events``
-subdirectory.
-
-To create new migration during development follow these steps::
-
- cd migrations-events
- alembic revision -m "revision description"
-
-Now edit the generated revision file to suit your needs. You may wish to use
-following resources as reference:
-
-* `Operation Reference <https://alembic.sqlalchemy.org/en/latest/ops.html>`__
-* `Cookbook <https://alembic.sqlalchemy.org/en/latest/cookbook.html>`__
+.. code-block:: shell
-Migration can be then invoked locally from within the migration environment directory::
+ # Always make sure your virtual environment is activated:
+ $ . venv/bin/activate
- cd migrations-events
- alembic upgrade head
- alembic history
+ # Run tests:
+ (venv) $ make build-whl
-To enable execution of database migrations on target systems after installation
-from package there is a simple wrapper script ``/etc/mentat/scripts/sqldb-migrate-e.sh``::
+Generated packages will be placed into ``./dist`` subdirectory.
- /etc/mentat/scripts/sqldb-migrate-e.sh upgrade head
- /etc/mentat/scripts/sqldb-migrate-e.sh history
-Important resources:
+Building Debian packages
+--------------------------------------------------------------------------------
-* `Alembic <https://alembic.sqlalchemy.org/en/latest/index.html>`__
+If you want to build native Debian packages locally please use following makefile
+target:
+.. code-block:: shell
-Metadata database migrations
-````````````````````````````````````````````````````````````````````````````````
+ # Always make sure your virtual environment is activated:
+ $ . venv/bin/activate
-Important resources:
+ # Run tests:
+ (venv) $ make build-deb
-* `Alembic <https://alembic.sqlalchemy.org/en/latest/index.html>`__
-* `Flask-Migrate <https://flask-migrate.readthedocs.io/en/latest/>`__
+Generated packages will be placed into ``./deploy/mentat/`` subdirectory.
Examples
--
GitLab