Make the whole project more developer friendly.

Following big improvements were made to make the development more easier:

* Created the 'setup.py' script to support native Python packaging with setuptools and pip.
* There is a new target 'make develop', that will bootstrap the Python virtual environment, install all production and development requirements and install the whole project in editable mode.
* Revised the dependency installation targets.
* Big facelift of makefile help target.
* Added target for build environment cleanup.
* Added target for initialization of new translations of Mentat and Hawat.
* Added target for building locally Python packages.
* Added target for building locally Debian packages.
* Updated documentation page for developers.

(Redmine issue: #4216,#3387,#3361)

.gitignore

@@ -28,6 +28,27 @@
 *.pyc
 *.egg-info
 __pycache__
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+*.manifest
+*.spec
+pip-log.txt
+pip-delete-this-directory.txt
 
 # Ignore compiled translations catalogs.
 messages.mo

Gruntfile.js

+//------------------------------------------------------------------------------
+// This file is part of Mentat system (https://mentat.cesnet.cz/).
+//
+// Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+// Author: Jan Mach <jan.mach@cesnet.cz>
+// Use of this source is governed by the MIT license, see LICENSE file.
+//------------------------------------------------------------------------------
+
 module.exports = function(grunt) {
 
     // ---------------------------------------------------------------------
@@ -573,10 +581,6 @@ module.exports = function(grunt) {
     // Setup custom task(s).
     // ---------------------------------------------------------------------
 
-    grunt.registerTask('deb-build',
-                       '(RUN) Build Debian packages for Mentat-ng system.',
-                       ['check-version', 'shell:pyclean', 'clean:build', 'shell:deb_archive', 'webui', 'copy:deb', 'chmod', 'shell:dir_size', 'template:deb-control', 'shell:deb_build', 'clean:build']
-        );
     grunt.registerTask('deb-buildbot',
                        '(RUN) Build Debian packages for Mentat-ng system by Buildbot automation system.',
                        ['check-version', 'shell:pyclean', 'clean:build', 'shell:deb_archive', 'shell:pybabel_mentat', 'webui', 'copy:deb', 'chmod', 'shell:dir_size', 'template:deb-control', 'shell:deb_build', 'clean:build']
@@ -613,6 +617,10 @@ module.exports = function(grunt) {
                        '(RUN) Build and install web user interface dependencies.',
                        ['shell:yarn_install', 'shell:pybabel_hawat', 'clean:webui', 'copy:webui', 'comments', 'minify-cldrs']
         );
+    grunt.registerTask('build',
+                       '(RUN) Build Debian packages for Mentat-ng system.',
+                       ['check-version', 'shell:pyclean', 'clean:build', 'shell:deb_archive', 'shell:pybabel_mentat', 'webui', 'copy:deb', 'chmod', 'shell:dir_size', 'template:deb-control', 'shell:deb_build', 'clean:build']
+        );
     grunt.registerTask('default',
                        '(RUN) Alias for deb-build, only build Debian packages.',
                        ['deb-build']

MANIFEST.in

+include LICENSE.txt
+include README.rst
+recursive-include lib/hawat/static *
+recursive-include lib/hawat/templates *
+recursive-include lib/hawat/translations *
+recursive-include lib/hawat/blueprints *
+global-exclude *.pyc

Makefile.cfg

+#-------------------------------------------------------------------------------
+# LOCAL MAKEFILE CUSTOMIZATION FILE
+#
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Author: Jan Mach <jan.mach@cesnet.cz>
+# Use of this source is governed by an MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------

README.rst

+Mentat-ng - README
+================================================================================
+
+.. note::
+
+    This readme file is work in progress.

conf/requirements-dev.pip

+nose==1.3.7
+pyflakes==2.1.0
+pylint==2.2.2
+sphinx==1.8.4
+sphinx-rtd-theme==0.4.2

conf/requirements-latest-dev.pip

+nose
+pyflakes
+pylint
+sphinx
+sphinx-rtd-theme

doc/sphinx/_doclib/development.rst

@@ -4,17 +4,18 @@ Development
 ================================================================================
 
 This is the documentation for developers of the Mentat library itself, or developers
-of components and modules usable by or pluggable into Mentat system.
+of components and modules usable by or pluggable into the Mentat system.
 
 
 Key information
 --------------------------------------------------------------------------------
 
-* `Project issue tracking system <https://homeproj.cesnet.cz/projects/mentat>`__
-* `Primary source code repository <https://homeproj.cesnet.cz/git/mentat-ng.git/>`__
 * `Project Mentat: official website <https://mentat.cesnet.cz/en/index>`__
 * `Project Warden: official website <https://warden.cesnet.cz/en/index>`__
 * `IDEA: official website <https://idea.cesnet.cz/en/index>`__
+* `Project issue tracking system (Redmine) <https://homeproj.cesnet.cz/projects/mentat>`__
+* `Primary source code repository (Git) <https://homeproj.cesnet.cz/git/mentat-ng.git/>`__
+* `Automated build system (Alchemist) <https://alchemist.cesnet.cz>`__
 
 
 General guidelines
@@ -37,58 +38,51 @@ General guidelines
 * Reuse existing (even soft) dependencies. There is no need to use three competing IP address libraries. However, do not prevent application developer to use different one in his app, should he need to.
 
 
-Versioning
+Development essentials
 --------------------------------------------------------------------------------
 
-This project uses the `semantic versioning <https://semver.org/>`__. When the
-**production** level packages are being built and deployed, the automated build
-system takes the project version directly from following files (paths are relative
-to project root):
+There is a project master *Makefile* in the root of the project repository which
+can perform various essential or useful development tasks. You can get the full
+list of all available make commands/targets by executing one of the following
+commands::
 
-* ``lib/mentat/__init__.py``
-* ``package.json``
+	$ make
+	$ make help
 
-Sadly you have to adjust the version string on both of these places, currently
-there is no way how to do it on one.
 
-When building the **release** or **development** level packages, the automated
-build system appends an internal build number as additional subversion. This way
-each build produces unique version string and unique package. This feature can
-be used during development to reduce the need for incrementing the version numbers
-manually between each builds.
+Preparing development environment
+````````````````````````````````````````````````````````````````````````````````
 
+There are several development prerequisites, that already have to be present on
+your development machine. These prerequisites are not installed automatically
+for you, because the installation is too complex. There prerequisites are:
 
-Tagging
---------------------------------------------------------------------------------
+* `Python 3 <https://www.python.org/>`__: Please use version similar to current stable Python3 release on current stable Debian release.
+* `Pip <https://pip.pypa.io/en/stable/>`__: Python package manager, we recommend installation with `get-pip.py <https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py>`__.
+* `Yarn <https://yarnpkg.com/en/>`__: NPM package manager for web interface libraries.
+* `PostgreSQL 11 <https://www.postgresql.org/>`__: Relational database, please use version 11 wherever possible.
 
-Each major and minor version release must be tagged within the repository. Please
-use only annotated or signed tags and provide short comment for the release. Before
-tagging please view existing tags so that you can attempt to maintain the style of
-the tag messages.
+You can check for presence of all of these dependencies with this handy make target:
 
 .. code-block:: shell
 
-	# List all existing tags
-	git tag -l -n999
-
-	# Create new annotated tag and provide message
-	git tag -a v2.0.0
+	# Check for presence of all prerequisites:
+	$ make deps-prerequisites
 
-	# Push tags to remote servers (if you are permitted to do so)
-	git push origin v2.0.0
-	git push buildbot v2.0.0
+Now please execute the following command from the root directory of the repository to
+initialize correct Python virtual environment, install all requirements (including
+those only for development) and finally install the project locally in editable mode:
 
+.. code-block:: shell
 
-Development essentials
---------------------------------------------------------------------------------
+	# Perform all installations:
+	$ make develop
 
-There is a project master *Makefile* in the root of the project repository which
-can perform various usefull or essential development tasks. You can get the full
-list of all available make commands/targets by executing one of the following
-commands::
+	# Activate virtual environment before any development work:
+	$ . venv/bin/activate
 
-	make
-	make help
+	# Deactivate virtual environment when not needed with:
+	(venv) $ deactivate
 
 
 Checking code with Pyflakes
@@ -99,25 +93,22 @@ tool by executing following command:
 
 .. code-block:: shell
 
-	make pyflakes
+	# Always make sure your virtual environment is activated:
+	$ . venv/bin/activate
+
+	# Run tests:
+	(venv) $ make pyflakes
 
 Or you may check just the single file by executing following command:
 
 .. code-block:: shell
 
-	cd lib
-	pyflakes path/to/module.py
-
-You have to be inside the lib project subdirectory, otherwise Python interpreter
-would not be able to find required libraries. You may fix that by providing correct
-value to PYTHONPATH environment variable.
-
-Make sure, that the `pyflakes <https://pypi.org/project/pyflakes/>`__ library is
-already installed on your system. You may install it by executing following command:
-
-.. code-block:: shell
+	# Always make sure your virtual environment is activated:
+	$ . venv/bin/activate
 
-	pip3 install pyflakes
+	# Run tests:
+	(venv) $ cd lib
+	(venv) $ pyflakes path/to/module.py
 
 
 Checking code with Pylint
@@ -128,25 +119,22 @@ tool by executing following command:
 
 .. code-block:: shell
 
-	make pylint
+	# Always make sure your virtual environment is activated:
+	$ . venv/bin/activate
+
+	# Run tests:
+	(venv) $ make pylint
 
 Or you may check just the single file by executing following command:
 
 .. code-block:: shell
 
-	cd lib
-	pylint --rcfile=../.pylintrc-lib path/to/module.py
-
-You have to be inside the lib project subdirectory, otherwise Python interpreter
-would not be able to find required libraries. You may fix that by providing correct
-value to PYTHONPATH environment variable.
-
-Make sure, that the `pylint <https://pypi.org/project/pylint/>`__ library is already
-installed on your system. You may install it by executing following command:
-
-.. code-block:: shell
+	# Always make sure your virtual environment is activated:
+	$ . venv/bin/activate
 
-	pip3 install pylint
+	# Run tests:
+	(venv) $ cd lib
+	(venv) $ pylint --rcfile=../.pylintrc-lib path/to/module.py
 
 
 Running unit tests
@@ -157,14 +145,11 @@ command:
 
 .. code-block:: shell
 
-	make test
+	# Always make sure your virtual environment is activated:
+	$ . venv/bin/activate
 
-Make sure, that the `nose <https://pypi.org/project/nose/>`__ library is already
-installed on your system. You may install it by executing following command:
-
-.. code-block:: shell
-
-	pip3 install nose
+	# Run tests:
+	(venv) $ make test
 
 
 Documentation
@@ -179,24 +164,22 @@ and the result should still be more or less readable. Please test it immediately
 
 .. code-block:: shell
 
-	pydoc3 ./path/to/module.py
+	# Always make sure your virtual environment is activated:
+	$ . venv/bin/activate
+
+	# Run tests:
+	(venv) $ pydoc3 ./path/to/module.py
 
 You may generate and review the documentation locally by executing the following
 command:
 
 .. code-block:: shell
 
-	make docs
-
-Make sure, that the `Sphinx <https://pypi.org/project/sphinx/>`__ and
-`sphinx-rtd-theme <https://pypi.org/project/sphinx-rtd-theme/>`__ libraries are
-already installed on your system. You may install them by executing following
-commands:
-
-.. code-block:: shell
+	# Always make sure your virtual environment is activated:
+	$ . venv/bin/activate
 
-	pip3 install sphinx
-	pip3 install sphinx_rtd_theme
+	# Run tests:
+	(venv) $ make docs
 
 Documentation will be generated into ``doc/sphinx/_build/html/manual.html``.
 
@@ -216,6 +199,48 @@ Important resources
   * `Documenting functions and methods <http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists>`__
 
 
+Versioning
+--------------------------------------------------------------------------------
+
+This project uses the `semantic versioning <https://semver.org/>`__. When the
+**production** level packages are being built and deployed, the automated build
+system takes the project version directly from following files (paths are relative
+to project root):
+
+* ``lib/mentat/__init__.py``
+* ``package.json``
+
+Sadly you have to adjust the version string on both of these places, currently
+there is no way how to do it on one.
+
+When building the **release** or **development** level packages, the automated
+build system appends an internal build number as additional subversion. This way
+each build produces unique version string and unique package. This feature can
+be used during development to reduce the need for incrementing the version numbers
+manually between each builds.
+
+
+Tagging
+--------------------------------------------------------------------------------
+
+Each major and minor version release must be tagged within the repository. Please
+use only annotated or signed tags and provide short comment for the release. Before
+tagging please view existing tags so that you can attempt to maintain the style of
+the tag messages.
+
+.. code-block:: shell
+
+	# List all existing tags
+	git tag -l -n999
+
+	# Create new annotated tag and provide message
+	git tag -a v2.0.0
+
+	# Push tags to remote servers (if you are permitted to do so)
+	git push origin v2.0.0
+	git push buildbot v2.0.0
+
+
 Database schema migrations
 --------------------------------------------------------------------------------
 

setup.py

+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Author: Jan Mach <jan.mach@cesnet.cz>
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+Usage
+--------------------------------------------------------------------------------
+
+Install package locally for development:
+
+    pip install -e .[dev]
+
+Resources:
+--------------------------------------------------------------------------------
+
+* https://packaging.python.org/en/latest/
+* https://python-packaging.readthedocs.io/en/latest/index.html
+* https://setuptools.readthedocs.io/en/latest/setuptools.html
+
+"""
+
+
+import sys
+import os
+
+# To use a consistent encoding
+from codecs import open
+# Always prefer setuptools over distutils
+from setuptools import setup, find_packages
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+sys.path.insert(0, os.path.abspath('lib'))
+import mentat
+
+#-------------------------------------------------------------------------------
+
+def read_file(file_name):
+    """Read file and return its contents."""
+    with open(file_name, 'r') as fhd:
+        return fhd.read()
+
+def read_requirements(file_name):
+    """Read requirements file as a list."""
+    reqs = read_file(file_name).splitlines()
+    if not reqs:
+        raise RuntimeError(
+            "Unable to read requirements from the {} file.".format(
+                file_name
+            )
+        )
+    reqs = [req.split(' ')[0] for req in reqs]
+    return reqs
+
+#-------------------------------------------------------------------------------
+
+# Get the long description from the README file
+with open(os.path.join(here, 'README.rst'), encoding='utf-8') as fhnd:
+    long_description = fhnd.read()
+
+setup(
+    name = 'mentat-ng',
+    version = mentat.__version__,
+    description = 'Distributed modular SIEM system designed to monitor networks of all sizes',
+    long_description = long_description,
+    classifiers = [
+        'Development Status :: 4 - Beta',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 3 :: Only'
+    ],
+    keywords = 'library',
+    url = 'https://homeproj.cesnet.cz/git/mentat-ng.git',
+    author = 'CESNET-CERTS Development Team',
+    author_email = 'csirt@cesnet.cz',
+    license = 'MIT',
+    package_dir = {'': 'lib'},
+    packages = find_packages('lib'),
+    test_suite = 'nose.collector',
+    tests_require = [
+        'nose'
+    ],
+    install_requires = read_requirements('conf/requirements.pip'),
+    # Add development requirements as extras. This way it is possible to install
+    # the package for development locally with following command:
+    #
+    #   pip install -e .[dev]
+    #
+    # Resources:
+    #   https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies
+    #   https://stackoverflow.com/a/28842733
+    extras_require = {
+        'dev': [
+            'nose',
+            'pyflakes',
+            'pylint',
+            'sphinx',
+            'sphinx-rtd-theme'
+        ]
+    },
+    scripts = [
+        'bin/mentat-backup.py',
+        'bin/mentat-cleanup.py',
+        'bin/mentat-controller.py',
+        'bin/mentat-dbmngr.py',
+        'bin/mentat-enricher.py',
+        'bin/mentat-hawat.py',
+        'bin/mentat-hawat.wsgi',
+        'bin/mentat-ideagen.py',
+        'bin/mentat-informant.py',
+        'bin/mentat-inspector.py',
+        'bin/mentat-netmngr.py',
+        'bin/mentat-precache.py',
+        'bin/mentat-reporter.py',
+        'bin/mentat-sampler.py',
+        'bin/mentat-statistician.py',
+        'bin/mentat-storage.py'
+    ],
+    # Add entry point to custom command line interface.
+    #
+    # Resources:
+    #   http://flask.pocoo.org/docs/1.0/cli/#custom-commands
+    #entry_points={
+    #    'console_scripts': [
+    #        'hawat-cli=hawat:cli'
+    #    ],
+    #},
+    include_package_data = True,
+    zip_safe = False
+)