diff --git a/.gitignore b/.gitignore index 66e2f8002210d17b609d82abe3a05a30c62b74b0..a6e089a9146001868b36def7bb4f0ce2932f2dab 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,27 @@ *.pyc *.egg-info __pycache__ +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +venv/ +*.egg-info/ +.installed.cfg +*.egg +*.manifest +*.spec +pip-log.txt +pip-delete-this-directory.txt # Ignore compiled translations catalogs. messages.mo diff --git a/Gruntfile.js b/Gruntfile.js index 75e63e6e24601df81a214236d1480a96931af1fd..43bee360b1fa917f2575f3e6d6f3483c7d88f723 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -1,3 +1,11 @@ +//------------------------------------------------------------------------------ +// This file is part of Mentat system (https://mentat.cesnet.cz/). +// +// Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/) +// Author: Jan Mach <jan.mach@cesnet.cz> +// Use of this source is governed by the MIT license, see LICENSE file. +//------------------------------------------------------------------------------ + module.exports = function(grunt) { // --------------------------------------------------------------------- @@ -573,10 +581,6 @@ module.exports = function(grunt) { // Setup custom task(s). // --------------------------------------------------------------------- - grunt.registerTask('deb-build', - '(RUN) Build Debian packages for Mentat-ng system.', - ['check-version', 'shell:pyclean', 'clean:build', 'shell:deb_archive', 'webui', 'copy:deb', 'chmod', 'shell:dir_size', 'template:deb-control', 'shell:deb_build', 'clean:build'] - ); grunt.registerTask('deb-buildbot', '(RUN) Build Debian packages for Mentat-ng system by Buildbot automation system.', ['check-version', 'shell:pyclean', 'clean:build', 'shell:deb_archive', 'shell:pybabel_mentat', 'webui', 'copy:deb', 'chmod', 'shell:dir_size', 'template:deb-control', 'shell:deb_build', 'clean:build'] @@ -613,6 +617,10 @@ module.exports = function(grunt) { '(RUN) Build and install web user interface dependencies.', ['shell:yarn_install', 'shell:pybabel_hawat', 'clean:webui', 'copy:webui', 'comments', 'minify-cldrs'] ); + grunt.registerTask('build', + '(RUN) Build Debian packages for Mentat-ng system.', + ['check-version', 'shell:pyclean', 'clean:build', 'shell:deb_archive', 'shell:pybabel_mentat', 'webui', 'copy:deb', 'chmod', 'shell:dir_size', 'template:deb-control', 'shell:deb_build', 'clean:build'] + ); grunt.registerTask('default', '(RUN) Alias for deb-build, only build Debian packages.', ['deb-build'] diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..9e71067c474c4f3b4483ba2fcf84aca0d28d9144 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,7 @@ +include LICENSE.txt +include README.rst +recursive-include lib/hawat/static * +recursive-include lib/hawat/templates * +recursive-include lib/hawat/translations * +recursive-include lib/hawat/blueprints * +global-exclude *.pyc diff --git a/Makefile b/Makefile index 37fa95b5ad21aa42395d73595b145addabb19b65..7dbd64f797e291ffbd5729d148042e569dc0260f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ #------------------------------------------------------------------------------- # MASTER MAKEFILE FOR MENTAT-NG PROJECT # -# Copyright (C) since 2016, CESNET, z. s. p. o. +# This file is part of Mentat system (https://mentat.cesnet.cz/). +# +# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/) # Author: Jan Mach <jan.mach@cesnet.cz> # Use of this source is governed by an MIT license, see LICENSE file. #------------------------------------------------------------------------------- @@ -18,17 +20,26 @@ DIR_TEMPLATES_UTEST = conf/templates/utest BIN_FILES := $(wildcard bin/mentat-*.py) LIB_FILES := $(shell find $(DIR_LIB) -name '*.py' | grep -v 'test_') -PYTHON = python3 -PIP = pip3 -NOSETESTS = nosetests -TWINE = twine -PYBABEL = pybabel +VENV_PYTHON = python3 +VENV_PATH = venv +PYTHON = python +PIP = pip +NOSETESTS = nosetests +TWINE = twine +PYBABEL = pybabel CURRENT_DIR = $(shell pwd) # -# Color code definitions for colored terminal output -# https://stackoverflow.com/questions/5947742/how-to-change-the-output-color-of-echo-in-linux +# Include local customized configurations. +# +include Makefile.cfg + +# +# Color code definitions for colored terminal output. +# +# Resource: +# https://stackoverflow.com/questions/5947742/how-to-change-the-output-color-of-echo-in-linux # RED=\033[0;31m GREEN=\033[0;32m @@ -37,8 +48,10 @@ BLUE=\033[0;34m PURPLE=\033[0;35m CYAN=\033[0;36m NC=\033[0m +BOLD=\033[1m +FAINT=\033[2m -#------------------------------------------------------------------------------- +#=============================================================================== # # Default make target, alias for 'help', you must explicitly choose the target. @@ -46,91 +59,232 @@ NC=\033[0m default: help # -# Check the project code. +# Display extensive help information page. # -check: pyflakes pylint test - -#------------------------------------------------------------------------------- - help: - @echo " __ __ _ _" - @echo " | \\/ | | | | |" - @echo " | \\ / | ___ _ __ | |_ __ _| |_" - @echo " | |\\/| |/ _ \\ '_ \\| __/ _\` | __|" - @echo " | | | | __/ | | | || (_| | |_" - @echo " |_| |_|\\___|_| |_|\\__\\__,_|\\__|" - @echo "" - @echo " ${GREEN}────────────────────────────────────────────────────────────────────────────────${NC}" - @echo " ${GREEN} LIST OF AVAILABLE MAKE TARGETS${NC}" - @echo " ${GREEN}────────────────────────────────────────────────────────────────────────────────${NC}" - @echo "" - @echo " * ${GREEN}default${NC}: alias for help, you have to pick a target" - @echo " * ${GREEN}help${NC}: print this help message and exit" - @echo " * ${GREEN}show-version${NC}: show current project version" - @echo " * ${GREEN}deps${NC}: install various dependencies" - @echo " = ${ORANGE}deps-python${NC}: install Python dependencies with pip3" - @echo " = ${ORANGE}deps-geoip${NC}: install geolocation databases" - @echo " = ${ORANGE}deps-negistry${NC}: install negistry whois database" - @echo " = ${ORANGE}deps-postgresql${NC}: install and configure PostgreSQL database" - @echo " = ${ORANGE}deps-translations${NC}: compile all available translations" - @echo " * ${GREEN}docs${NC}: generate local project documentation" - @echo " = ${ORANGE}docs-sync${NC}: synchronize documentation from submodules" - @echo " = ${ORANGE}docs-sphinx${NC}: generate local project documentation" - @echo " * ${GREEN}presentations${NC}: project presentations" - @echo " * ${GREEN}pybabel-patch${NC}: patch babel library" - @echo " * ${GREEN}hpybabel-extract${NC}: extract Hawat user interface translations" - @echo " * ${GREEN}hpybabel-update${NC}: update Hawat user interface translations" - @echo " * ${GREEN}hpybabel-pull${NC}: extract and update Hawat user interface translations" - @echo " * ${GREEN}hpybabel-compile${NC}: compile Hawat user interface translations" - @echo " * ${GREEN}mpybabel-extract${NC}: extract Mentat translations" - @echo " * ${GREEN}mpybabel-update${NC}: update Mentat translations" - @echo " * ${GREEN}mpybabel-pull${NC}: extract and update Mentat translations" - @echo " * ${GREEN}mpybabel-compile${NC}: compile Mentat translations" - @echo " * ${GREEN}check${NC}: perform extensive project checking" - @echo " = ${ORANGE}pyflakes${NC}: check project with pyflakes" - @echo " - pyflakes-bin: check executables with pyflakes" - @echo " - pyflakes-lib: check library with pyflakes, exclude test files" - @echo " - pyflakes-lib-all: check library with pyflakes including test files" - @echo " = ${ORANGE}pylint${NC}: check project with pylint" - @echo " - pylint-bin: check executables with pylint" - @echo " - pylint-lib: check library with pylint, exclude test files" - @echo " - pylint-lib-all: check library with pylint including test files" - @echo " = ${ORANGE}test${NC}: run unit tests with nosetest" - @echo "" - @echo " ${GREEN}────────────────────────────────────────────────────────────────────────────────${NC}" + @echo "" + @echo " â–â–â–â•— â–â–â–â•—â–â–â–â–â–â–â–â•—â–â–â–â•— â–â–â•—â–â–â–â–â–â–â–â–â•— â–â–â–â–â–â•— â–â–â–â–â–â–â–â–â•—" + @echo " â–â–â–â–â•— â–â–â–â–â•‘â–â–â•”â•â•â•â•â•ťâ–â–â–â–â•— â–â–â•‘â•šâ•â•â–â–â•”â•â•â•ťâ–â–â•”â•â•â–â–â•—â•šâ•â•â–â–â•”â•â•â•ť" + @echo " â–â–â•”â–â–â–â–â•”â–â–â•‘â–â–â–â–â–â•— â–â–â•”â–â–â•— â–â–â•‘ â–â–â•‘ â–â–â–â–â–â–â–â•‘ â–â–â•‘" + @echo " â–â–â•‘â•šâ–â–╔╝â–â–â•‘â–â–â•”â•â•â•ť â–â–â•‘â•šâ–â–â•—â–â–â•‘ â–â–â•‘ â–â–â•”â•â•â–â–â•‘ â–â–â•‘" + @echo " â–â–â•‘ â•šâ•â•ť â–â–â•‘â–â–â–â–â–â–â–â•—â–â–â•‘ â•šâ–â–â–â–â•‘ â–â–â•‘ â–â–â•‘ â–â–â•‘ â–â–â•‘" + @echo " â•šâ•â•ť â•šâ•â•ťâ•šâ•â•â•â•â•â•â•ťâ•šâ•â•ť â•šâ•â•â•â•ť â•šâ•â•ť â•šâ•â•ť â•šâ•â•ť â•šâ•â•ť" + @echo " $(FAINT)Copyright (C) since 2011 CESNET, z.s.p.o$(NC)" + @echo "" + @echo " $(GREEN)$(BOLD)â•”â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•—$(NC)" + @echo " $(GREEN)$(BOLD)â•‘ LIST OF AVAILABLE MAKE TARGETS â•‘$(NC)" + @echo " $(GREEN)$(BOLD)â•šâ•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•ť$(NC)" + @echo "" + @echo " $(BLUE)$(BOLD)MAIN TARGETS$(NC)" + @echo " $(BLUE)$(BOLD)────────────$(NC)" + @echo " * $(GREEN)default$(NC): alias for help, you have to pick a target" + @echo " * $(GREEN)help$(NC): print this help message and exit" + @echo " * $(GREEN)show-version$(NC): show current project version" + @echo " * $(GREEN)develop$(NC): install and configure project locally for development" + @echo " * $(GREEN)deps$(NC): install project dependencies" + @echo " * $(GREEN)clean$(NC): cleanup development and build environment" + @echo " * $(GREEN)docs$(NC): generate project documentation" + @echo " * $(GREEN)check$(NC): perform extensive checks and tests" + @echo " * $(GREEN)build-whl$(NC): perform local build of Python distribution package" + @echo " * $(GREEN)build-deb$(NC): perform local build of Debian distribution package" + @echo "" + @echo " $(BLUE)$(BOLD)HELPER TARGETS$(NC)" + @echo " $(BLUE)$(BOLD)──────────────$(NC)" + @echo " * $(GREEN)deps-prerequisites$(NC): check for development prerequisites" + @echo " * $(GREEN)deps-python$(NC): install Python dependencies" + @echo " * $(GREEN)deps-python-dev$(NC): install Python development dependencies" + @echo " * $(GREEN)deps-python-upgrade$(NC): upgrade Python dependencies to latest versions" + @echo " * $(GREEN)deps-python-upgrade-dev$(NC): upgrade Python development dependencies to latest versions" + @echo " * $(GREEN)deps-webui$(NC): install web interface dependencies" + @echo " * $(GREEN)deps-webui-upgrade$(NC): upgrade web interface dependencies" + @echo " * $(GREEN)deps-geoip$(NC): install geolocation databases" + @echo " * $(GREEN)deps-negistry$(NC): install negistry whois database" + @echo " * $(GREEN)deps-postgresql$(NC): install and configure PostgreSQL database" + @echo " * $(GREEN)deps-translations$(NC): compile all available translations" + @echo "" + @echo " * $(GREEN)clean-pyc$(NC): clean up Python compiled files" + @echo " * $(GREEN)clean-build-python$(NC): clean up Python build directories" + @echo " * $(GREEN)clean-build-debian$(NC): clean up Debian build directories" + @echo "" + @echo " * $(GREEN)docs-sync$(NC): synchronize documentation from submodules" + @echo " * $(GREEN)docs-sphinx$(NC): generate local project documentation" + @echo " * $(GREEN)presentations$(NC): project presentations" + @echo "" + @echo " * $(GREEN)pybabel-patch$(NC): patch babel library" + @echo " * $(GREEN)hpybabel-pull$(NC): extract and update Hawat translations" + @echo " - $(ORANGE)hpybabel-extract$(NC): extract Hawat translations" + @echo " - $(ORANGE)hpybabel-update$(NC): update Hawat translations" + @echo " * $(GREEN)hpybabel-compile$(NC): compile Hawat translations" + @echo " * $(GREEN)mpybabel-pull$(NC): extract and update Mentat translations" + @echo " - $(ORANGE)mpybabel-extract$(NC): extract Mentat translations" + @echo " - $(ORANGE)mpybabel-update$(NC): update Mentat translations" + @echo " * $(GREEN)mpybabel-compile$(NC): compile Mentat translations" + @echo "" + @echo " * $(GREEN)pyflakes$(NC): check project with pyflakes" + @echo " - $(ORANGE)pyflakes-bin$(NC): check executables with pyflakes" + @echo " - $(ORANGE)pyflakes-lib$(NC): check library with pyflakes, exclude test files" + @echo " - $(ORANGE)pyflakes-lib-all$(NC): check library with pyflakes including test files" + @echo " * $(GREEN)pylint$(NC): check project with pylint" + @echo " - $(ORANGE)pylint-bin$(NC): check executables with pylint" + @echo " - $(ORANGE)pylint-lib$(NC): check library with pylint, exclude test files" + @echo " - $(ORANGE)pylint-lib-all$(NC): check library with pylint including test files" + @echo " * $(GREEN)test$(NC): run unit tests with nosetest" + @echo "" + @echo " * $(GREEN)build-webui$(NC): setup web interface locally" + @echo " * $(GREEN)build-package-whl$(NC): actually generate Python package" + @echo " * $(GREEN)build-package-deb$(NC): actually generate Debian package" + @echo "" + @echo " $(GREEN)â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•â•$(NC)" @echo "" #------------------------------------------------------------------------------- +# +# Show current project version. This can be used by various automated systems to +# verify/mark the version that is actually being built. +# show-version: FORCE @PYTHONPATH=lib $(PYTHON) -c "import mentat; print(mentat.__version__);" -#------------------------------------------------------------------------------- +# +# Install and configure project locally for development. This target will perform +# following tasks for you: +# - bootstrap the Python virtual environment into 'venv' subdirectory +# - install all requirements (conf/requirements.pip) +# - install all development requirements (conf/requirements-dev.pip) +# - install the project in editable mode +# +# NOTE: This target is calling 'venv/bin/activate' on its own to install the +# requirements into newly created/existing virtual environment. When using all +# other makefile targets you must enable the environment youself! +# +develop: FORCE + @echo "\n$(GREEN)*** Installing Python virtual environment for local development ***$(NC)\n" + @echo "Requested version: $(VENV_PYTHON)" + @echo "Path to binary: `which $(VENV_PYTHON)`" + @echo "Path to venv: $(VENV_PATH)" + @echo "" + @if [ -d $(VENV_PATH) ]; then\ + echo "$(CYAN)Virtual environment already exists in '$(VENV_PATH)'.$(NC)";\ + else\ + $(VENV_PYTHON) -m venv $(VENV_PATH);\ + echo "$(CYAN)Virtual environment successfully created in '$(VENV_PATH)'.$(NC)";\ + fi + @echo "" + @echo "Venv path: `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`" + @echo "Python stuff versions:" + @echo "" + @ls -al $(VENV_PATH)/bin | grep python + @ls -al $(VENV_PATH)/bin | grep pip + + @echo "\n$(GREEN)*** Installing project requirements ***$(NC)\n" + @. $(VENV_PATH)/bin/activate && $(PIP) install -r conf/requirements.pip + + @echo "\n$(GREEN)*** Installing project development requirements ***$(NC)\n" + @. $(VENV_PATH)/bin/activate && $(PIP) install -r conf/requirements-dev.pip + + @echo "\n$(GREEN)*** Installing project into virtual environment in editable mode ***$(NC)\n" + @. $(VENV_PATH)/bin/activate && $(PIP) install -e ".[dev]" + + @echo "\n$(CYAN)Your development environment is ready in `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`.$(NC)\n" + @echo "Please activate it manually with following command:\n" + @echo "\t$(ORANGE). $(VENV_PATH)/bin/activate$(NC)\n" + @echo "!!! Please keep in mind, that all makefile targets leave it up to you to activate the correct virtual environment !!!" + @echo "" + +# +# Install and configure project dependencies. +# +deps: deps-prerequisites deps-python deps-python-dev deps-webui deps-geoip deps-negistry deps-postgresql deps-translations + +# +# Cleanup development and build environment. +# +clean: clean-pyc clean-build-python clean-build-debian + +# +# Generate project documentation. +# +docs: docs-sync docs-sphinx + +# +# Check the project code. +# +check: pyflakes pylint test +# +# Perform local build of Python distribution package. +# +build-whl: clean build-webui build-package-whl + +# +# Perform local build of Debian distribution package. +# +build-deb: clean build-webui build-package-deb + + +#=============================================================================== -deps: deps-python deps-geoip deps-negistry deps-postgresql deps-translations + +deps-prerequisites: FORCE + @echo "\n$(GREEN)*** Checking for development prerequisites ***$(NC)\n" + @for prereq in $(PYTHON) $(PIP) yarn psql ; do \ + if command -v $$prereq >/dev/null 2>&1; then \ + echo "Prerequisite: $$prereq"; \ + else \ + echo "$(RED)PREREQUISITE: $$prereq (missing).$(NC)\n"; \ + echo "You have to install this prerequisite manually.\n"; \ + exit 1; \ + fi \ + done + @echo "" deps-python: FORCE - @echo "\n${GREEN}*** Installing Python dependencies ***${NC}\n" + @echo "\n$(GREEN)*** Installing Python dependencies ***$(NC)\n" @$(PIP) install -r conf/requirements.pip + @echo "" + +deps-python-dev: FORCE + @echo "\n$(GREEN)*** Installing Python development dependencies ***$(NC)\n" + @$(PIP) install -r conf/requirements-dev.pip + @echo "" -deps-python-latest: FORCE - @echo "\n${GREEN}*** Upgrading Python dependencies to latest versions ***${NC}\n" +deps-python-upgrade: FORCE + @echo "\n$(GREEN)*** Upgrading Python dependencies to latest versions ***$(NC)\n" @$(PIP) install -r conf/requirements-latest.pip --upgrade + @echo "" + +deps-python-upgrade-dev: FORCE + @echo "\n$(GREEN)*** Upgrading Python development dependencies to latest versions ***$(NC)\n" + @$(PIP) install -r conf/requirements-latest-dev.pip --upgrade + @echo "" + +deps-webui: FORCE + @echo "\n$(GREEN)*** Installing web interface dependencies ***$(NC)\n" + @yarn install + @echo "" + +deps-webui-upgrade: FORCE + @echo "\n$(GREEN)*** Upgrading web interface dependencies ***$(NC)\n" + @yarn upgrade + @echo "" deps-geoip: FORCE - @echo "\n${GREEN}*** Installing geolocation databases ***${NC}\n" + @echo "\n$(GREEN)*** Installing IP geolocation databases ***$(NC)\n" @./scripts/fetch-geoipdb.sh + @echo "" deps-negistry: FORCE - @echo "\n${GREEN}*** Installing negistry whois database ***${NC}\n" + @echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n" @./scripts/fetch-negistry.sh --stub + @echo "" deps-postgresql: FORCE - @echo "\n${GREEN}*** Installing and configuring PostgreSQL database ***${NC}\n" + @echo "\n$(GREEN)*** Installing and configuring PostgreSQL database ***$(NC)\n" @./scripts/sqldb-init.sh + @echo "" deps-translations: hpybabel-compile mpybabel-compile @@ -138,25 +292,42 @@ deps-translations: hpybabel-compile mpybabel-compile #------------------------------------------------------------------------------- -docs: docs-sync docs-sphinx +clean-pyc: FORCE + @echo "\n$(GREEN)*** Cleaning up Python compiled files ***$(NC)\n" + @find . -name '*.pyc' -exec rm --force {} + + @find . -name '*.pyo' -exec rm --force {} + + @find . -name '*~' -exec rm --force {} + @echo "" + +clean-build-python: FORCE + @echo "\n$(GREEN)*** Cleaning up Python build directories ***$(NC)\n" + @rm --force --recursive build/ + @rm --force --recursive dist/ + @rm --force --recursive *.egg-info + @echo "" + +clean-build-debian: FORCE + @echo "\n$(GREEN)*** Cleaning up Debian build directories ***$(NC)\n" + @find ./deploy/mentat -name '*.deb' -exec rm --force {} + + @echo "" + + +#------------------------------------------------------------------------------- + docs-sync: FORCE - @echo "\n${GREEN}*** Synchronizing documentation source code from submodules ***${NC}\n" + @echo "\n$(GREEN)*** Synchronizing documentation source code from submodules ***$(NC)\n" @rsync -r --progress --archive --update --delete --force ./submodules/pyzenkit/doc/_doclib/apidoc/pyzenkit.*.rst ./doc/sphinx/_doclib/apidoc/ @rsync -r --progress --archive --update --delete --force ./submodules/pyzenkit/doc/_doclib/_inc*.rst ./doc/sphinx/_doclib/ @rsync -r --progress --archive --update --delete --force ./submodules/pynspect/doc/_doclib/api_*.rst ./doc/sphinx/_doclib/ @#find ./ -type f -name *.pyc -exec rm -f {} \; docs-sphinx: FORCE - @echo "\n${GREEN}*** Generating project documentation ***${NC}\n" + @echo "\n$(GREEN)*** Generating project documentation ***$(NC)\n" @cd $(DIR_DOC)/sphinx && make html - -#------------------------------------------------------------------------------- - - presentations: FORCE - @echo "\n${GREEN}*** Generating project presentations ***${NC}\n" + @echo "\n$(GREEN)*** Generating project presentations ***$(NC)\n" @for presdir in $(DIR_DOC)/presentations/*; do \ if [ -d $$presdir ]; then \ echo "────────────────────────────────────────────────────────────────────────────────"; \ @@ -169,15 +340,16 @@ presentations: FORCE fi; \ done + #------------------------------------------------------------------------------- # # This patch solves following issue: https://github.com/python-babel/flask-babel/issues/43 -# +# Apply when necessary. pybabel-patch: FORCE - @echo "\n${GREEN}*** Patching babel library ***${NC}\n" + @echo "\n$(GREEN)*** Patching babel library ***$(NC)\n" @cp util/babel.messages.frontend.py.patch /var/tmp/ @cd / && patch -p0 -i /var/tmp/babel.messages.frontend.py.patch @@ -190,28 +362,42 @@ pybabel-patch: FORCE # hpybabel-extract: FORCE - @echo "\n${GREEN}*** Extracting babel translations for Hawat user interface ***${NC}\n" + @echo "\n$(GREEN)*** Extracting babel translations for Hawat user interface ***$(NC)\n" @cd $(DIR_LIB_HAWAT) && $(PYBABEL) extract -F babel.cfg -o messages.pot -k lazy_gettext -k tr_ . +hpybabel-init: FORCE + @echo "\n$(GREEN)*** Initializing translations for new locale for Hawat user interface ***$(NC)\n" + @echo "Locale name: $(INIT_LOCALE)" + @cd $(DIR_LIB_HAWAT) && $(PYBABEL) init -i messages.pot -d translations -l $(INIT_LOCALE) + @echo "" + hpybabel-update: FORCE - @echo "\n${GREEN}*** Updating translations for Hawat user interface ***${NC}\n" + @echo "\n$(GREEN)*** Updating translations for Hawat user interface ***$(NC)\n" @cd $(DIR_LIB_HAWAT) && $(PYBABEL) update -i messages.pot -d translations -l cs hpybabel-pull: hpybabel-extract hpybabel-update hpybabel-compile: FORCE - @echo "\n${GREEN}*** Compiling translations for Hawat user interface ***${NC}\n" + @echo "\n$(GREEN)*** Compiling translations for Hawat user interface ***$(NC)\n" @cd $(DIR_LIB_HAWAT) && $(PYBABEL) compile -d translations mpybabel-extract: FORCE - @echo "\n${GREEN}*** Extracting babel translations for Mentat ***${NC}\n" + @echo "\n$(GREEN)*** Extracting babel translations for Mentat ***$(NC)\n" @$(PYBABEL) extract -F $(DIR_TEMPLATES_INFORMANT)/babel.cfg -o $(DIR_TEMPLATES_INFORMANT)/messages.pot -k lazy_gettext -k tr_ . @$(PYBABEL) extract -F $(DIR_TEMPLATES_REPORTER)/babel.cfg -o $(DIR_TEMPLATES_REPORTER)/messages.pot -k lazy_gettext -k tr_ . @$(PYBABEL) extract -F $(DIR_TEMPLATES_UTEST)/babel.cfg -o $(DIR_TEMPLATES_UTEST)/messages.pot -k lazy_gettext -k tr_ . +mpybabel-init: FORCE + @echo "\n$(GREEN)*** Initializing translations for new locale for Mentat ***$(NC)\n" + @echo "Locale name: $(INIT_LOCALE)" + @$(PYBABEL) init -i $(DIR_TEMPLATES_INFORMANT)/messages.pot -d $(DIR_TEMPLATES_INFORMANT)/translations -l $(INIT_LOCALE) + @$(PYBABEL) init -i $(DIR_TEMPLATES_REPORTER)/messages.pot -d $(DIR_TEMPLATES_REPORTER)/translations -l $(INIT_LOCALE) + @$(PYBABEL) init -i $(DIR_TEMPLATES_UTEST)/messages.pot -d $(DIR_TEMPLATES_UTEST)/translations -l $(INIT_LOCALE) + @echo "" + mpybabel-update: FORCE - @echo "\n${GREEN}*** Updating translations for Mentat ***${NC}\n" + @echo "\n$(GREEN)*** Updating translations for Mentat ***$(NC)\n" @$(PYBABEL) update -i $(DIR_TEMPLATES_INFORMANT)/messages.pot -d $(DIR_TEMPLATES_INFORMANT)/translations -l cs @$(PYBABEL) update -i $(DIR_TEMPLATES_REPORTER)/messages.pot -d $(DIR_TEMPLATES_REPORTER)/translations -l cs @$(PYBABEL) update -i $(DIR_TEMPLATES_UTEST)/messages.pot -d $(DIR_TEMPLATES_UTEST)/translations -l cs @@ -219,7 +405,7 @@ mpybabel-update: FORCE mpybabel-pull: mpybabel-extract mpybabel-update mpybabel-compile: FORCE - @echo "\n${GREEN}*** Compiling translations for Mentat ***${NC}\n" + @echo "\n$(GREEN)*** Compiling translations for Mentat ***$(NC)\n" @$(PYBABEL) compile -d $(DIR_TEMPLATES_INFORMANT)/translations @$(PYBABEL) compile -d $(DIR_TEMPLATES_REPORTER)/translations @$(PYBABEL) compile -d $(DIR_TEMPLATES_UTEST)/translations @@ -231,55 +417,72 @@ mpybabel-compile: FORCE pyflakes: pyflakes-bin pyflakes-lib pyflakes-bin: - @echo "\n${GREEN}*** Checking executables with pyflakes ***${NC}\n" + @echo "\n$(GREEN)*** Checking executables with pyflakes ***$(NC)\n" @$(PYTHON) --version @echo "" - -@PYTHONPATH=$(DIR_LIB) $(PYTHON) -m pyflakes $(DIR_BIN) + @$(PYTHON) -m pyflakes $(DIR_BIN) pyflakes-lib: - @echo "\n${GREEN}*** Checking library with pyflakes - test files excluded ***${NC}\n" + @echo "\n$(GREEN)*** Checking library with pyflakes - test files excluded ***$(NC)\n" @$(PYTHON) --version @echo "" - -@for l in ${LIB_FILES}; do\ - PYTHONPATH=$(DIR_LIB) $(PYTHON) -m pyflakes $$l; \ + -@for l in $(LIB_FILES); do\ + $(PYTHON) -m pyflakes $$l; \ done pyflakes-lib-all: - @echo "\n${GREEN}*** Checking library with pyflakes - all files ***${NC}\n" + @echo "\n$(GREEN)*** Checking library with pyflakes - all files ***$(NC)\n" @$(PYTHON) --version @echo "" - -@PYTHONPATH=$(DIR_LIB) $(PYTHON) -m pyflakes $(DIR_LIB) + @$(PYTHON) -m pyflakes $(DIR_LIB) pylint: pylint-bin pylint-lib pylint-bin: - @echo "\n${GREEN}*** Checking executables with pylint ***${NC}\n" + @echo "\n$(GREEN)*** Checking executables with pylint ***$(NC)\n" @$(PYTHON) --version @echo "" -@PYTHONPATH=$(DIR_LIB) $(PYTHON) -m pylint --rcfile .pylintrc-bin $(DIR_BIN) pylint-lib: - @echo "\n${GREEN}*** Checking library with pylint - test files excluded ***${NC}\n" + @echo "\n$(GREEN)*** Checking library with pylint - test files excluded ***$(NC)\n" @$(PYTHON) --version @echo "" - -@for l in ${LIB_FILES}; do\ + -@for l in $(LIB_FILES); do\ PYTHONPATH=$(DIR_LIB) $(PYTHON) -m pylint --rcfile .pylintrc-lib $$l; \ done pylint-lib-all: - @echo "\n${GREEN}*** Checking library with pylint - all files ***${NC}\n" + @echo "\n$(GREEN)*** Checking library with pylint - all files ***$(NC)\n" @$(PYTHON) --version @echo "" -@PYTHONPATH=$(DIR_LIB) $(PYTHON) -m pylint --rcfile .pylintrc-lib $(DIR_LIB) test: - @echo "\n${GREEN}*** Checking code with nosetests ***${NC}\n" + @echo "\n$(GREEN)*** Checking code with nosetests ***$(NC)\n" @PYTHONPATH=$(DIR_LIB) $(NOSETESTS) #------------------------------------------------------------------------------- +build-webui: + @echo "\n${GREEN}*** Building web interface environment ***${NC}\n" + @grunt webui + @echo "" + +build-package-whl: FORCE + @echo "\n${GREEN}*** Building Python packages ***${NC}\n" + @echo "Using Python version: `$(PYTHON) --version`" + @echo "" + @$(PYTHON) setup.py sdist bdist_wheel + @echo "" + +build-package-deb: FORCE + @echo "\n${GREEN}*** Building Debian packages ***${NC}\n" + @$(PYTHON) grunt build + @echo "" + # Empty rule as dependency will force make to always perform target # Source: https://www.gnu.org/software/make/manual/html_node/Force-Targets.html FORCE: diff --git a/Makefile.cfg b/Makefile.cfg new file mode 100644 index 0000000000000000000000000000000000000000..3b1ae19b287d8c67a9e305a12ed55e9a04242072 --- /dev/null +++ b/Makefile.cfg @@ -0,0 +1,9 @@ +#------------------------------------------------------------------------------- +# LOCAL MAKEFILE CUSTOMIZATION FILE +# +# This file is part of Mentat system (https://mentat.cesnet.cz/). +# +# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/) +# Author: Jan Mach <jan.mach@cesnet.cz> +# Use of this source is governed by an MIT license, see LICENSE file. +#------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..66cbf0a413876221d6393b9c6b771024360b8f9c --- /dev/null +++ b/README.rst @@ -0,0 +1,6 @@ +Mentat-ng - README +================================================================================ + +.. note:: + + This readme file is work in progress. diff --git a/conf/requirements-dev.pip b/conf/requirements-dev.pip new file mode 100644 index 0000000000000000000000000000000000000000..7eca31beba4a8eafe0fe549c74a1abafd74bbff9 --- /dev/null +++ b/conf/requirements-dev.pip @@ -0,0 +1,5 @@ +nose==1.3.7 +pyflakes==2.1.0 +pylint==2.2.2 +sphinx==1.8.4 +sphinx-rtd-theme==0.4.2 diff --git a/conf/requirements-latest-dev.pip b/conf/requirements-latest-dev.pip new file mode 100644 index 0000000000000000000000000000000000000000..6bb055546d762f925712349f6cff0c02f7e695c5 --- /dev/null +++ b/conf/requirements-latest-dev.pip @@ -0,0 +1,5 @@ +nose +pyflakes +pylint +sphinx +sphinx-rtd-theme diff --git a/doc/sphinx/_doclib/development.rst b/doc/sphinx/_doclib/development.rst index 6c3b0ab2ca21e77fbc691f7d39bf2d048ec63ea0..9c7422b5bc2f0c8f7f1bbd7556e0127691e128d4 100644 --- a/doc/sphinx/_doclib/development.rst +++ b/doc/sphinx/_doclib/development.rst @@ -4,17 +4,18 @@ Development ================================================================================ This is the documentation for developers of the Mentat library itself, or developers -of components and modules usable by or pluggable into Mentat system. +of components and modules usable by or pluggable into the Mentat system. Key information -------------------------------------------------------------------------------- -* `Project issue tracking system <https://homeproj.cesnet.cz/projects/mentat>`__ -* `Primary source code repository <https://homeproj.cesnet.cz/git/mentat-ng.git/>`__ * `Project Mentat: official website <https://mentat.cesnet.cz/en/index>`__ * `Project Warden: official website <https://warden.cesnet.cz/en/index>`__ * `IDEA: official website <https://idea.cesnet.cz/en/index>`__ +* `Project issue tracking system (Redmine) <https://homeproj.cesnet.cz/projects/mentat>`__ +* `Primary source code repository (Git) <https://homeproj.cesnet.cz/git/mentat-ng.git/>`__ +* `Automated build system (Alchemist) <https://alchemist.cesnet.cz>`__ General guidelines @@ -37,58 +38,51 @@ General guidelines * Reuse existing (even soft) dependencies. There is no need to use three competing IP address libraries. However, do not prevent application developer to use different one in his app, should he need to. -Versioning +Development essentials -------------------------------------------------------------------------------- -This project uses the `semantic versioning <https://semver.org/>`__. When the -**production** level packages are being built and deployed, the automated build -system takes the project version directly from following files (paths are relative -to project root): +There is a project master *Makefile* in the root of the project repository which +can perform various essential or useful development tasks. You can get the full +list of all available make commands/targets by executing one of the following +commands:: -* ``lib/mentat/__init__.py`` -* ``package.json`` + $ make + $ make help -Sadly you have to adjust the version string on both of these places, currently -there is no way how to do it on one. -When building the **release** or **development** level packages, the automated -build system appends an internal build number as additional subversion. This way -each build produces unique version string and unique package. This feature can -be used during development to reduce the need for incrementing the version numbers -manually between each builds. +Preparing development environment +```````````````````````````````````````````````````````````````````````````````` +There are several development prerequisites, that already have to be present on +your development machine. These prerequisites are not installed automatically +for you, because the installation is too complex. There prerequisites are: -Tagging --------------------------------------------------------------------------------- +* `Python 3 <https://www.python.org/>`__: Please use version similar to current stable Python3 release on current stable Debian release. +* `Pip <https://pip.pypa.io/en/stable/>`__: Python package manager, we recommend installation with `get-pip.py <https://pip.pypa.io/en/stable/installing/#installing-with-get-pip-py>`__. +* `Yarn <https://yarnpkg.com/en/>`__: NPM package manager for web interface libraries. +* `PostgreSQL 11 <https://www.postgresql.org/>`__: Relational database, please use version 11 wherever possible. -Each major and minor version release must be tagged within the repository. Please -use only annotated or signed tags and provide short comment for the release. Before -tagging please view existing tags so that you can attempt to maintain the style of -the tag messages. +You can check for presence of all of these dependencies with this handy make target: .. code-block:: shell - # List all existing tags - git tag -l -n999 - - # Create new annotated tag and provide message - git tag -a v2.0.0 + # Check for presence of all prerequisites: + $ make deps-prerequisites - # Push tags to remote servers (if you are permitted to do so) - git push origin v2.0.0 - git push buildbot v2.0.0 +Now please execute the following command from the root directory of the repository to +initialize correct Python virtual environment, install all requirements (including +those only for development) and finally install the project locally in editable mode: +.. code-block:: shell -Development essentials --------------------------------------------------------------------------------- + # Perform all installations: + $ make develop -There is a project master *Makefile* in the root of the project repository which -can perform various usefull or essential development tasks. You can get the full -list of all available make commands/targets by executing one of the following -commands:: + # Activate virtual environment before any development work: + $ . venv/bin/activate - make - make help + # Deactivate virtual environment when not needed with: + (venv) $ deactivate Checking code with Pyflakes @@ -99,25 +93,22 @@ tool by executing following command: .. code-block:: shell - make pyflakes + # Always make sure your virtual environment is activated: + $ . venv/bin/activate + + # Run tests: + (venv) $ make pyflakes Or you may check just the single file by executing following command: .. code-block:: shell - cd lib - pyflakes path/to/module.py - -You have to be inside the lib project subdirectory, otherwise Python interpreter -would not be able to find required libraries. You may fix that by providing correct -value to PYTHONPATH environment variable. - -Make sure, that the `pyflakes <https://pypi.org/project/pyflakes/>`__ library is -already installed on your system. You may install it by executing following command: - -.. code-block:: shell + # Always make sure your virtual environment is activated: + $ . venv/bin/activate - pip3 install pyflakes + # Run tests: + (venv) $ cd lib + (venv) $ pyflakes path/to/module.py Checking code with Pylint @@ -128,25 +119,22 @@ tool by executing following command: .. code-block:: shell - make pylint + # Always make sure your virtual environment is activated: + $ . venv/bin/activate + + # Run tests: + (venv) $ make pylint Or you may check just the single file by executing following command: .. code-block:: shell - cd lib - pylint --rcfile=../.pylintrc-lib path/to/module.py - -You have to be inside the lib project subdirectory, otherwise Python interpreter -would not be able to find required libraries. You may fix that by providing correct -value to PYTHONPATH environment variable. - -Make sure, that the `pylint <https://pypi.org/project/pylint/>`__ library is already -installed on your system. You may install it by executing following command: - -.. code-block:: shell + # Always make sure your virtual environment is activated: + $ . venv/bin/activate - pip3 install pylint + # Run tests: + (venv) $ cd lib + (venv) $ pylint --rcfile=../.pylintrc-lib path/to/module.py Running unit tests @@ -157,14 +145,11 @@ command: .. code-block:: shell - make test + # Always make sure your virtual environment is activated: + $ . venv/bin/activate -Make sure, that the `nose <https://pypi.org/project/nose/>`__ library is already -installed on your system. You may install it by executing following command: - -.. code-block:: shell - - pip3 install nose + # Run tests: + (venv) $ make test Documentation @@ -179,24 +164,22 @@ and the result should still be more or less readable. Please test it immediately .. code-block:: shell - pydoc3 ./path/to/module.py + # Always make sure your virtual environment is activated: + $ . venv/bin/activate + + # Run tests: + (venv) $ pydoc3 ./path/to/module.py You may generate and review the documentation locally by executing the following command: .. code-block:: shell - make docs - -Make sure, that the `Sphinx <https://pypi.org/project/sphinx/>`__ and -`sphinx-rtd-theme <https://pypi.org/project/sphinx-rtd-theme/>`__ libraries are -already installed on your system. You may install them by executing following -commands: - -.. code-block:: shell + # Always make sure your virtual environment is activated: + $ . venv/bin/activate - pip3 install sphinx - pip3 install sphinx_rtd_theme + # Run tests: + (venv) $ make docs Documentation will be generated into ``doc/sphinx/_build/html/manual.html``. @@ -216,6 +199,48 @@ Important resources * `Documenting functions and methods <http://www.sphinx-doc.org/en/stable/domains.html#info-field-lists>`__ +Versioning +-------------------------------------------------------------------------------- + +This project uses the `semantic versioning <https://semver.org/>`__. When the +**production** level packages are being built and deployed, the automated build +system takes the project version directly from following files (paths are relative +to project root): + +* ``lib/mentat/__init__.py`` +* ``package.json`` + +Sadly you have to adjust the version string on both of these places, currently +there is no way how to do it on one. + +When building the **release** or **development** level packages, the automated +build system appends an internal build number as additional subversion. This way +each build produces unique version string and unique package. This feature can +be used during development to reduce the need for incrementing the version numbers +manually between each builds. + + +Tagging +-------------------------------------------------------------------------------- + +Each major and minor version release must be tagged within the repository. Please +use only annotated or signed tags and provide short comment for the release. Before +tagging please view existing tags so that you can attempt to maintain the style of +the tag messages. + +.. code-block:: shell + + # List all existing tags + git tag -l -n999 + + # Create new annotated tag and provide message + git tag -a v2.0.0 + + # Push tags to remote servers (if you are permitted to do so) + git push origin v2.0.0 + git push buildbot v2.0.0 + + Database schema migrations -------------------------------------------------------------------------------- diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..c5cbf96f75069001ccf3b74336697829b9acd456 --- /dev/null +++ b/setup.py @@ -0,0 +1,136 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +#------------------------------------------------------------------------------- +# This file is part of Mentat system (https://mentat.cesnet.cz/). +# +# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/) +# Author: Jan Mach <jan.mach@cesnet.cz> +# Use of this source is governed by the MIT license, see LICENSE file. +#------------------------------------------------------------------------------- + + +""" +Usage +-------------------------------------------------------------------------------- + +Install package locally for development: + + pip install -e .[dev] + +Resources: +-------------------------------------------------------------------------------- + +* https://packaging.python.org/en/latest/ +* https://python-packaging.readthedocs.io/en/latest/index.html +* https://setuptools.readthedocs.io/en/latest/setuptools.html + +""" + + +import sys +import os + +# To use a consistent encoding +from codecs import open +# Always prefer setuptools over distutils +from setuptools import setup, find_packages + +here = os.path.abspath(os.path.dirname(__file__)) + +sys.path.insert(0, os.path.abspath('lib')) +import mentat + +#------------------------------------------------------------------------------- + +def read_file(file_name): + """Read file and return its contents.""" + with open(file_name, 'r') as fhd: + return fhd.read() + +def read_requirements(file_name): + """Read requirements file as a list.""" + reqs = read_file(file_name).splitlines() + if not reqs: + raise RuntimeError( + "Unable to read requirements from the {} file.".format( + file_name + ) + ) + reqs = [req.split(' ')[0] for req in reqs] + return reqs + +#------------------------------------------------------------------------------- + +# Get the long description from the README file +with open(os.path.join(here, 'README.rst'), encoding='utf-8') as fhnd: + long_description = fhnd.read() + +setup( + name = 'mentat-ng', + version = mentat.__version__, + description = 'Distributed modular SIEM system designed to monitor networks of all sizes', + long_description = long_description, + classifiers = [ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3 :: Only' + ], + keywords = 'library', + url = 'https://homeproj.cesnet.cz/git/mentat-ng.git', + author = 'CESNET-CERTS Development Team', + author_email = 'csirt@cesnet.cz', + license = 'MIT', + package_dir = {'': 'lib'}, + packages = find_packages('lib'), + test_suite = 'nose.collector', + tests_require = [ + 'nose' + ], + install_requires = read_requirements('conf/requirements.pip'), + # Add development requirements as extras. This way it is possible to install + # the package for development locally with following command: + # + # pip install -e .[dev] + # + # Resources: + # https://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-extras-optional-features-with-their-own-dependencies + # https://stackoverflow.com/a/28842733 + extras_require = { + 'dev': [ + 'nose', + 'pyflakes', + 'pylint', + 'sphinx', + 'sphinx-rtd-theme' + ] + }, + scripts = [ + 'bin/mentat-backup.py', + 'bin/mentat-cleanup.py', + 'bin/mentat-controller.py', + 'bin/mentat-dbmngr.py', + 'bin/mentat-enricher.py', + 'bin/mentat-hawat.py', + 'bin/mentat-hawat.wsgi', + 'bin/mentat-ideagen.py', + 'bin/mentat-informant.py', + 'bin/mentat-inspector.py', + 'bin/mentat-netmngr.py', + 'bin/mentat-precache.py', + 'bin/mentat-reporter.py', + 'bin/mentat-sampler.py', + 'bin/mentat-statistician.py', + 'bin/mentat-storage.py' + ], + # Add entry point to custom command line interface. + # + # Resources: + # http://flask.pocoo.org/docs/1.0/cli/#custom-commands + #entry_points={ + # 'console_scripts': [ + # 'hawat-cli=hawat:cli' + # ], + #}, + include_package_data = True, + zip_safe = False +)