diff --git a/.gitignore b/.gitignore index 5b94c50a4239718956389ea988bc09c1a0c9ab09..b2ea7be0b10236568201e0a54fa9b50492441b8e 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,9 @@ parsetab.py /doc/sphinx/_doclib/_inc_bin.cfg-daemon.rst /doc/sphinx/_doclib/_inc_bin.cfg-script.rst +# Ignore local lightweight chroot. +/chroot/ + # Ignore local .dotenv configuration file .env diff --git a/Makefile b/Makefile index a4745849d8a75c33b12e8f11b0d7c0fcbc67933d..f2b236000eda10148c133197e455be062eb34030 100644 --- a/Makefile +++ b/Makefile @@ -35,8 +35,8 @@ DIR_EVENT_CLASSES = $(shell pwd)/conf/event_classes BIN_FILES := $(wildcard bin/mentat-*.py) LIB_FILES := $(shell find $(DIR_LIB) -name '*.py' | grep -v 'test_') -VENV_PYTHON ?= python3.7 -VENV_PATH ?= /var/mentat/venv +VENV_PYTHON ?= python3 +VENV_PATH ?= venv PYTHON = python3 PIP = pip @@ -166,7 +166,39 @@ help: # Install and configure Python virtual environment for local project development. # venv: FORCE - @./conf/scripts/init-venv.sh + @echo "\n$(GREEN)*** Installing Python virtual environment for local development ***$(NC)\n" + @echo "Requested version: $(VENV_PYTHON)" + @echo "Path to binary: `which $(VENV_PYTHON)`" + @echo "Path to venv: $(VENV_PATH)" + @echo "" + @if [ -d $(VENV_PATH) ]; then\ + echo "$(CYAN)Virtual environment already exists in '$(VENV_PATH)'.$(NC)";\ + else\ + $(VENV_PYTHON) -m venv $(VENV_PATH);\ + echo "$(CYAN)Virtual environment successfully created in '$(VENV_PATH)'.$(NC)";\ + fi + @echo "" + @echo "Upgrading 'pip' and 'wheel' within the virtual environment to latest versions" + @echo "" + @$(VENV_PATH)/bin/pip install --upgrade pip + @$(VENV_PATH)/bin/pip install --upgrade wheel + @echo "" + @echo "Venv path: `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`" + @echo "Python stuff versions:" + @echo "" + @$(VENV_PATH)/bin/python --version + @$(VENV_PATH)/bin/pip --version + @echo "" + @ls -al $(VENV_PATH)/bin | grep python + @ls -al $(VENV_PATH)/bin | grep pip + + @echo "\n$(CYAN)Your development environment is ready in `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`.$(NC)\n" + @echo "Please activate it manually with following command:\n" + @echo "\t$(ORANGE). $(VENV_PATH)/bin/activate$(NC)\n" + @echo "Consider adding following alias to your ~/.bashrc file for easier environment activation:\n" + @echo "\t$(ORANGE)alias entervenv='. venv/bin/activate'$(NC)\n" + @echo "$(BOLD)!!! Please keep in mind, that all makefile targets except this one ('develop') leave it up to you to activate the correct virtual environment !!!$(NC)" + @echo "" # # Install and configure project locally for development. This target will perform @@ -183,7 +215,11 @@ venv: FORCE # develop: deps build-webui -deps: deps-prerequisites deps-runenv deps-python-dev deps-python deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile +develop-vagrant: deps-vagrant build-webui + +deps: deps-prerequisites deps-lwchroot deps-python-dev deps-python deps-editable deps-webui deps-geoip deps-negistry deps-postgresql translations-compile + +deps-vagrant: deps-prerequisites deps-runenv deps-python-dev deps-python deps-editable deps-webui deps-geoip-vagrant deps-negistry-vagrant deps-postgresql translations-compile clean: clean-pycs clean-build-docs clean-build-python clean-build-debian @@ -211,7 +247,7 @@ release-whl: FORCE #=============================================================================== -deps-build: deps-prerequisites deps-runenv deps-python deps-python-dev deps-editable deps-webui translations-compile +deps-build: deps-prerequisites deps-lwchroot deps-python deps-python-dev deps-editable deps-webui translations-compile deps-build-whl: deps-build deps-build-deb: deps-build @@ -240,6 +276,10 @@ deps-prerequisites: FORCE # # Install project`s runtime environment. # +deps-lwchroot: FORCE + @echo "\n$(GREEN)*** Creating local lightweight chroot subdirectory structure ***$(NC)\n" + @./conf/scripts/lwchroot-init.sh + deps-runenv: FORCE @echo "\n$(GREEN)*** Creating local runtime environment subdirectory structure ***$(NC)\n" @./conf/scripts/init-runenv.sh @@ -318,6 +358,12 @@ deps-webui-upgrade: FORCE # Fetch and install IP geolocation databases. # deps-geoip: FORCE + @echo "\n$(GREEN)*** Installing IP geolocation databases ***$(NC)\n" + @mkdir -p ./chroot/usr/share/GeoIP + @geoipupdate --verbose --database-directory $(shell realpath ./chroot/usr/share/GeoIP) + @echo "" + +deps-geoip-vagrant: FORCE @echo "\n$(GREEN)*** Installing IP geolocation databases ***$(NC)\n" @mkdir -p $(shell realpath ./data/geoip) @geoipupdate --verbose --database-directory $(shell realpath ./data/geoip) @@ -327,6 +373,11 @@ deps-geoip: FORCE # Fetch and install Negistry whois database. # deps-negistry: FORCE + @echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n" + @./conf/scripts/fetch-negistry.sh --stub --target=$(shell realpath ./chroot/var/mentat/whois-negistry.json) + @echo "" + +deps-negistry-vagrant: FORCE @echo "\n$(GREEN)*** Installing Negistry whois database ***$(NC)\n" @./conf/scripts/fetch-negistry.sh --stub --target=/var/mentat/whois-negistry.json @echo "" @@ -344,6 +395,11 @@ deps-postgresql: FORCE run-webui-dev: + @echo "\n$(GREEN)*** Running development web server with development configuration on $(DEV_SERVER):$(DEV_PORT) ***$(NC)\n" + #APP_ROOT_PATH=$(shell realpath ./chroot) hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT) + HAWAT_CONFIG_FILE=$(shell realpath ./hawat.local.conf) hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT) + +run-webui-vagrant: @echo "\n$(GREEN)*** Running development web server with development configuration on $(DEV_SERVER):$(DEV_PORT) ***$(NC)\n" FLASK_ENV=development HAWAT_CONFIG_FILE=$(shell realpath ./conf/mentat-hawat-dev.py.conf) hawat-cli run --host "0.0.0.0" --port $(DEV_PORT) diff --git a/bin/mentat-hawat-dev.wsgi b/bin/mentat-hawat-dev.wsgi index 8ae7e84bf25afe806bd09cabfa399ef2d1671bfb..2b0ae7de970bf507cc84460853c0313fe070a4f2 100644 --- a/bin/mentat-hawat-dev.wsgi +++ b/bin/mentat-hawat-dev.wsgi @@ -6,8 +6,12 @@ #------------------------------------------------------------------------------- +from dotenv import load_dotenv + import hawat +load_dotenv(dotenv_path = '/home/mentat/mentat-ng/.env') + # # Use prepared factory function to create application instance. The factory # function takes number of arguments, that can be used to fine tune configuration @@ -15,8 +19,5 @@ import hawat # capabilities or for purposes of testing. Please refer to the documentation # for more information. # -application = hawat.create_app_full( - config_object = 'hawat.config.DevelopmentConfig', - config_file = '/etc/mentat/mentat-hawat-dev.py.conf', - config_env = 'FLASK_CONFIG_FILE' -) + +application = hawat.create_app() diff --git a/bin/mentat-hawat-vagrant.wsgi b/bin/mentat-hawat-vagrant.wsgi new file mode 100644 index 0000000000000000000000000000000000000000..8ae7e84bf25afe806bd09cabfa399ef2d1671bfb --- /dev/null +++ b/bin/mentat-hawat-vagrant.wsgi @@ -0,0 +1,22 @@ +#------------------------------------------------------------------------------- +# This file is part of Mentat system (https://mentat.cesnet.cz/). +# +# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/) +# Use of this source is governed by the MIT license, see LICENSE file. +#------------------------------------------------------------------------------- + + +import hawat + +# +# Use prepared factory function to create application instance. The factory +# function takes number of arguments, that can be used to fine tune configuration +# of the application. This is can be very usefull when extending applications` +# capabilities or for purposes of testing. Please refer to the documentation +# for more information. +# +application = hawat.create_app_full( + config_object = 'hawat.config.DevelopmentConfig', + config_file = '/etc/mentat/mentat-hawat-dev.py.conf', + config_env = 'FLASK_CONFIG_FILE' +) diff --git a/conf/apache/site_mentat_vagrant.conf b/conf/apache/site_mentat_vagrant.conf index 6e37cb3c92f3f5112c4f2c48c3ae9cf2cd4a71a5..bc13ddc4149bb99461de897963ddfd280ed1aa72 100644 --- a/conf/apache/site_mentat_vagrant.conf +++ b/conf/apache/site_mentat_vagrant.conf @@ -35,7 +35,7 @@ WSGIPythonHome /var/mentat/venv DocumentRoot /var/www WSGIDaemonProcess hawat user=mentat group=mentat threads=5 python-home=/var/mentat/venv - WSGIScriptAlias /mentat /vagrant/bin/mentat-hawat-dev.wsgi + WSGIScriptAlias /mentat /vagrant/bin/mentat-hawat-vagrant.wsgi <Location /> Require all granted diff --git a/conf/scripts/lwchroot-init.sh b/conf/scripts/lwchroot-init.sh new file mode 100755 index 0000000000000000000000000000000000000000..85144021e9eb1940fc7c081fcabd01bb3b9381fb --- /dev/null +++ b/conf/scripts/lwchroot-init.sh @@ -0,0 +1,36 @@ +#!/bin/bash +#------------------------------------------------------------------------------- +# Utility script for creating local lightweight chroot directory structure. +# +# Copyright (C) since 2011 CESNET, z.s.p.o +# Use of this source is governed by the MIT license, see LICENSE file. +#------------------------------------------------------------------------------- + +mkdir -p ./chroot +mkdir -p ./chroot/etc +mkdir -p ./chroot/etc/cron.d +mkdir -p ./chroot/var +mkdir -p ./chroot/var/mentat/backups +mkdir -p ./chroot/var/mentat/cache +mkdir -p ./chroot/var/mentat/charts +mkdir -p ./chroot/var/mentat/log +mkdir -p ./chroot/var/mentat/reports +mkdir -p ./chroot/var/mentat/rrds +mkdir -p ./chroot/var/mentat/run +mkdir -p ./chroot/var/mentat/spool +mkdir -p ./chroot/usr/share/GeoIP + +if [ ! -L ./chroot/etc/mentat ]; then + ln -s $(realpath ./conf) $(realpath ./chroot/etc)/mentat; +fi + +if [ ! -f .env ]; then + echo "APP_ROOT_PATH=$(realpath ./chroot)" >> .env + echo "FLASK_ENV=development" >> .env + echo "FLASK_CONFIG=development" >> .env + echo "FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf)" >> .env +fi + +if [ ! -L ./conf/migrations-events/.env ]; then + ln -s $(realpath ./.env) $(realpath ./conf/migrations-events)/.env; +fi diff --git a/etc/default/mentat b/etc/default/mentat index ec29614b23b35c87e9a381b0571d91312fd05284..4d2e4aecbab0fa7b0bca77805579c963fe5a1532 100644 --- a/etc/default/mentat +++ b/etc/default/mentat @@ -8,7 +8,7 @@ PG_MAJOR=13 NODE_MAJOR="12" -PYTHON_VERSION="python3.7" +PYTHON_VERSION="python3" # Master switch for Debian system scripts. MENTAT_IS_ENABLED=yes diff --git a/hawat.local.conf b/hawat.local.conf new file mode 100644 index 0000000000000000000000000000000000000000..9628b947444be71b2598752c923a6f0c2df547c8 --- /dev/null +++ b/hawat.local.conf @@ -0,0 +1,36 @@ +SECRET_KEY = 'local-secret-key' +MAIL_SERVER = 'localhost' +MAIL_PORT = 1025 +MAIL_DEFAULT_SENDER = 'mentat@cesnet.cz' +HAWAT_LOG_DEFAULT_LEVEL = 'debug' +HAWAT_LOG_FILE = '/var/tmp/mentat-hawat.py.log' +HAWAT_LOG_FILE_LEVEL = 'debug' +ENABLED_BLUEPRINTS = [ + 'vial.blueprints.auth', + 'vial.blueprints.auth_api', + 'vial.blueprints.design_bs3', + 'vial.blueprints.devtools', + 'vial.blueprints.changelogs', + + 'hawat.blueprints.auth_env', + 'hawat.blueprints.auth_dev', + 'hawat.blueprints.auth_pwd', + 'hawat.blueprints.home', + 'hawat.blueprints.reports', + 'hawat.blueprints.events', + 'hawat.blueprints.hosts', + 'hawat.blueprints.timeline', + 'hawat.blueprints.dnsr', + #'hawat.blueprints.pdnsr', + 'hawat.blueprints.geoip', + #'hawat.blueprints.nerd', + 'hawat.blueprints.whois', + 'hawat.blueprints.performance', + 'hawat.blueprints.status', + 'hawat.blueprints.dbstatus', + 'hawat.blueprints.users', + 'hawat.blueprints.groups', + 'hawat.blueprints.settings_reporting', + 'hawat.blueprints.filters', + 'hawat.blueprints.networks', +] diff --git a/vagrantenv/provision.sh b/vagrantenv/provision.sh index 22e9775190e17abe4b1772a394286a35a323e40f..ae1fdd7e60b3318d8c3d73a3d27a762a88140ce3 100644 --- a/vagrantenv/provision.sh +++ b/vagrantenv/provision.sh @@ -130,11 +130,14 @@ if [ ! -f /etc/mentat/install.pip ] ; then fi echo "==========> Preparing Python virtual environment" -cd /vagrant/ && make venv +/etc/mentat/scripts/init-venv.sh source ${MENTAT_VENV}/bin/activate -echo "==========> Preparing Python development environment" -cd /vagrant/ && make develop +echo "==========> Preparing application runtime environment" +/etc/mentat/scripts/init-runenv.sh + +echo "==========> Preparing development environment" +cd /vagrant/ && make develop-vagrant echo "==========> Configuring email services for development" ensure_link /vagrant/vagrantenv/postfix.main.cf /etc/postfix/main.cf @@ -149,9 +152,6 @@ systemctl daemon-reload systemctl start sendria.service systemctl enable sendria.service -echo "==========> Bootstraping runtime environment" -/etc/mentat/scripts/init-runenv.sh - echo "==========> Tweaking database users" for dbuname in root mentat vagrant watchdog do