diff --git a/.gitignore b/.gitignore
index bd147dd7cb723722356fe4fd64495a5383e41190..96c13d2fc0fc2445f70197f7a293f133dcb179c9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -67,3 +67,7 @@ parsetab.py
/doc/sphinx/_doclib/_inc_bin.cfg-app.rst
/doc/sphinx/_doclib/_inc_bin.cfg-daemon.rst
/doc/sphinx/_doclib/_inc_bin.cfg-script.rst
+
+# Ignore local lightweight chroot.
+/chroot/
+
diff --git a/Makefile b/Makefile
index 3cf7b0812051cb7c943286a62ea669a21918f063..e7898b2bc2c3700b7c6384580f948a180072f9d3 100644
--- a/Makefile
+++ b/Makefile
@@ -96,6 +96,7 @@ help:
@echo " $(BLUE)$(BOLD)HELPER TARGETS$(NC)"
@echo " $(BLUE)$(BOLD)──────────────$(NC)"
@echo " * $(GREEN)deps-prerequisites$(NC): check for development prerequisites"
+ @echo " * $(GREEN)deps-lwchroot$(NC): creating local lightweight chroot subdirectory structure"
@echo " * $(GREEN)deps-python$(NC): install Python dependencies"
@echo " * $(GREEN)deps-python-dev$(NC): install Python development dependencies"
@echo " * $(GREEN)deps-python-upgrade$(NC): upgrade Python dependencies to latest versions"
@@ -107,9 +108,12 @@ help:
@echo " * $(GREEN)deps-postgresql$(NC): configure required PostgreSQL user accounts and databases"
@echo " * $(GREEN)deps-translations$(NC): compile all available translations"
@echo ""
- @echo " * $(GREEN)run-webui-dev$(NC): run development web server with development configuration on on $(DEV_SERVER):$(DEV_PORT)"
+ @echo " * $(GREEN)run-mentat-dev$(NC): run development version of Mentat system"
+ @echo " * $(GREEN)run-webui-dev$(NC): run development web server with development configuration on $(DEV_SERVER):$(DEV_PORT)"
@echo " * $(GREEN)run-mailserver-dev$(NC): run development mail server on $(DEV_MAIL_SERVER):$(DEV_MAIL_PORT)"
@echo ""
+ @echo " * $(GREEN)ctrl-mentat-dev COMMAND=cmd$(NC): execute mentat-controller.py with given command $(FAINT)cmd$(NC)"
+ @echo ""
@echo " * $(GREEN)clean-pycs$(NC): clean up Python compiled files"
@echo " * $(GREEN)clean-build-python$(NC): clean up Python build directories"
@echo " * $(GREEN)clean-build-debian$(NC): clean up Debian build directories"
@@ -187,6 +191,18 @@ develop: FORCE
@echo "\n$(GREEN)*** Installing project into virtual environment in editable mode ***$(NC)\n"
@. $(VENV_PATH)/bin/activate && $(PIP) install -e ".[dev]"
+ @echo "\n$(GREEN)*** Creating local lightweight chroot directory structure ***$(NC)\n"
+ mkdir -p ./chroot/etc
+ mkdir -p ./chroot/var/mentat/backups
+ mkdir -p ./chroot/var/mentat/cache
+ mkdir -p ./chroot/var/mentat/charts
+ mkdir -p ./chroot/var/mentat/log
+ mkdir -p ./chroot/var/mentat/reports
+ mkdir -p ./chroot/var/mentat/rrds
+ mkdir -p ./chroot/var/mentat/run
+ mkdir -p ./chroot/var/mentat/spool;
+ -ln -s $(shell realpath ./conf) $(shell realpath ./chroot/etc)/mentat
+
@echo "\n$(CYAN)Your development environment is ready in `. $(VENV_PATH)/bin/activate && python -c 'import sys; print(sys.prefix)'`.$(NC)\n"
@echo "Please activate it manually with following command:\n"
@echo "\t$(ORANGE). $(VENV_PATH)/bin/activate$(NC)\n"
@@ -196,7 +212,7 @@ develop: FORCE
#
# Install and configure project dependencies.
#
-deps: deps-prerequisites deps-python deps-python-dev deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
+deps: deps-prerequisites deps-lwchroot deps-python deps-python-dev deps-webui deps-geoip deps-negistry deps-postgresql translations-compile
#
# Cleanup development and build environment.
@@ -248,6 +264,22 @@ deps-prerequisites: FORCE
done
@echo ""
+#
+# Install project`s lightweight chroot.
+#
+deps-lwchroot: FORCE
+ @echo "\n$(GREEN)*** Creating local lightweight chroot subdirectory structure ***$(NC)\n"
+ mkdir -p ./chroot/etc
+ mkdir -p ./chroot/var/mentat/backups
+ mkdir -p ./chroot/var/mentat/cache
+ mkdir -p ./chroot/var/mentat/charts
+ mkdir -p ./chroot/var/mentat/log
+ mkdir -p ./chroot/var/mentat/reports
+ mkdir -p ./chroot/var/mentat/rrds
+ mkdir -p ./chroot/var/mentat/run
+ mkdir -p ./chroot/var/mentat/spool;
+ -ln -s $(shell realpath ./conf) $(shell realpath ./chroot/etc)/mentat
+
#
# Install project`s Python dependencies using pip requirements file. The dependencies
# are already listed in setup.py file and pip can install them automatically. It
@@ -339,15 +371,23 @@ deps-postgresql: FORCE
#-------------------------------------------------------------------------------
+run-mentat-dev:
+ @echo "\n$(GREEN)*** Running development version of Mentat system ***$(NC)\n"
+ APP_ROOT_PATH=$(shell realpath ./chroot) mentat-controller.py --command start
+
run-webui-dev:
@echo "\n$(GREEN)*** Running development web server with development configuration on $(DEV_SERVER):$(DEV_PORT) ***$(NC)\n"
- FLASK_ENV=development FLASK_CONFIG=development FLASK_CONFIG_FILE=$(shell realpath ./hawat.local.conf) hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT)
+ APP_ROOT_PATH=$(shell realpath ./chroot) FLASK_ENV=development FLASK_CONFIG=development FLASK_CONFIG_FILE=$(shell realpath ./hawat.local.conf) hawat-cli run --host $(DEV_SERVER) --port $(DEV_PORT)
run-mailserver-dev:
@echo "\n$(GREEN)*** Running development mail server on $(DEV_MAIL_SERVER):$(DEV_MAIL_PORT) ***$(NC)\n"
$(PYTHON) -m smtpd -n -c DebuggingServer $(DEV_MAIL_SERVER):$(DEV_MAIL_PORT)
+ctrl-mentat-dev:
+ -APP_ROOT_PATH=$(shell realpath ./chroot) mentat-controller.py --command $(COMMAND)
+
+
#-------------------------------------------------------------------------------
diff --git a/conf/core/common.json.conf b/conf/core/common.json.conf
new file mode 100644
index 0000000000000000000000000000000000000000..4dbf4b6a1e014b6594e44165fd25f445811d2048
--- /dev/null
+++ b/conf/core/common.json.conf
@@ -0,0 +1,23 @@
+#-------------------------------------------------------------------------------
+#
+# COMMON MENTAT SYSTEM CONFIGURATIONS
+#
+#-------------------------------------------------------------------------------
+
+{
+ # Name/uid of the user.
+ # default: null
+ # type: string or integer
+ #
+ #"user": "mentat",
+
+ # Name/gid of the group.
+ # default: null
+ # type: string or integer
+ #
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
+}
diff --git a/conf/mentat-backup.py.conf b/conf/mentat-backup.py.conf
index b0a9b354e9b92eaa4a1938a45644cfb045323e45..3f7afab15550ebb7a802a0f304745b26aee9d0e9 100644
--- a/conf/mentat-backup.py.conf
+++ b/conf/mentat-backup.py.conf
@@ -15,19 +15,19 @@
#
"no_upload": true,
- # Mame of the mount point directory.
- # default: "/media/du-store"
+ # Name of the local mount point directory for remote storage.
+ # default: "/media/ds-store"
# type: string
#
- #"mount_point": "/media/du-store",
+ #"mount_point": "/media/ds-store",
- # Name of the temporary file directory.
- # default: "/var/tmp"
+ # Name of the local temporary file directory.
+ # default: "/var/mentat/tmp"
# type: string
#
- #"temp_dir": "/var/tmp",
+ #"temp_dir": "/var/mentat/tmp",
- # Name of the database backup directory.
+ # Name of the local database backup directory.
# default: "/var/mentat/backups"
# type: string
#
@@ -159,11 +159,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-cleanup.py.conf b/conf/mentat-cleanup.py.conf
index 23c70cc2a5ce11af89e5f6fc4998389813355004..6a3f3c1e09e3d359664a7c45174ca260ef2da23f 100644
--- a/conf/mentat-cleanup.py.conf
+++ b/conf/mentat-cleanup.py.conf
@@ -236,11 +236,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-controller.py.conf b/conf/mentat-controller.py.conf
index 669b60d7c4300815dd35cdbeb8b89225e297b82a..6688d42a3e8528e88547499741ccce06f5e9f092 100644
--- a/conf/mentat-controller.py.conf
+++ b/conf/mentat-controller.py.conf
@@ -86,6 +86,23 @@
]
}
+ #
+ # [CHAIN A|ENTRY]: Message inspection module - event classifications
+ #
+ #{
+ # "exec": "mentat-inspector.py",
+ # # Enable multiple instances working the same queue directory
+ # #"paralel": true,
+ # # In case of paralel mode, you MUST set the required number of instances
+ # #"count": 3,
+ # "args": [
+ # # Enable debug information before daemonization
+ # #"--debug"
+ # # Force logging level ['debug', 'info', 'warning', 'error', 'critical']
+ # #"--log-level=debug"
+ # ]
+ #}
+
#
# [CHAIN B|ENTRY|FINAL]: Additional message inspection module
#
@@ -133,5 +150,9 @@
# Check database of Mentat system for various anomalies.
{ "name": "mentat-watchdog-events-py" }
- ]
+ ],
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-dbmngr.py.conf b/conf/mentat-dbmngr.py.conf
index 8ee74e04e2c83d6590bdd718c7f64514ef0434bf..c38a5676c8d819b1b7e83a0968dbb1dc1a91362c 100644
--- a/conf/mentat-dbmngr.py.conf
+++ b/conf/mentat-dbmngr.py.conf
@@ -153,11 +153,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-enricher.py.conf b/conf/mentat-enricher.py.conf
index 4cec0c8c23e07a619f15a21d9899e323b5cac930..10305f52cdf38beb0d5df29d2247cd1acedbe2ed 100644
--- a/conf/mentat-enricher.py.conf
+++ b/conf/mentat-enricher.py.conf
@@ -141,16 +141,16 @@
#---------------------------------------------------------------------------
# Name of the input queue directory.
- # default: "/var/mentat/spool/mentat-enricher.py"
+ # default: "mentat-enricher.py"
# type: string
#
- #"queue_in_dir": "/var/mentat/spool/mentat-enricher.py",
+ #"queue_in_dir": "mentat-enricher.py",
# Name of the output queue directory.
# default: null
# type: string
#
- "queue_out_dir": "/var/mentat/spool/mentat-storage.py",
+ "queue_out_dir": "mentat-storage.py",
# Limit on the number of the files for the output queue directory.
# default: 10000
@@ -290,11 +290,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-ideagen.py.conf b/conf/mentat-ideagen.py.conf
index fe99a116fff3076dc27d23d6c71c73aa620dce46..dd28ea661959e51568ac2bcc3ccebd664238840c 100644
--- a/conf/mentat-ideagen.py.conf
+++ b/conf/mentat-ideagen.py.conf
@@ -35,16 +35,16 @@
#"back_off": 60,
# Name of the target queue directory.
- # default: "/var/tmp"
+ # default: "/var/mentat/spool/mentat-inspector.py/incoming"
# type: string
#
- "queue_dir": "/var/mentat/spool/mentat-inspector.py/incoming",
+ #"queue_dir": "/var/mentat/spool/mentat-inspector.py/incoming",
# Name of the temporary file directory.
- # default: "/var/tmp"
+ # default: "/var/mentat/spool/mentat-inspector.py/tmp"
# type: string
#
- #"temp_dir": "/var/tmp",
+ #"temp_dir": "/var/mentat/spool/mentat-inspector.py/tmp",
# Name of the template file for generating messages.
# default: "msg.01.idea.j2"
@@ -172,13 +172,13 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat",
+ #"group": "mentat",
#---------------------------------------------------------------------------
# Lists for IDEA message pseudo random generation.
@@ -415,5 +415,9 @@
"Tarpit",
"Recon",
"Monitor"
- ]
+ ],
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-informant.py.conf b/conf/mentat-informant.py.conf
index 49c1aba81389d597a6e31a80c38a294987ba9880..8ed6dbc78fa749f21fea9db2e5dc654315b865a7 100644
--- a/conf/mentat-informant.py.conf
+++ b/conf/mentat-informant.py.conf
@@ -201,11 +201,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-inspector-b.py.conf b/conf/mentat-inspector-b.py.conf
index f9670040b6de7534443303f89c611f2eebcdf32b..f9427ec823982d4b7c3f63b4947aac859c0b2876 100644
--- a/conf/mentat-inspector-b.py.conf
+++ b/conf/mentat-inspector-b.py.conf
@@ -154,16 +154,16 @@
#---------------------------------------------------------------------------
# Name of the input queue directory.
- # default: "/var/mentat/spool/mentat-inspector.py"
+ # default: "mentat-inspector-b.py"
# type: string
#
- "queue_in_dir": "/var/mentat/spool/mentat-inspector-b.py",
+ #"queue_in_dir": "mentat-inspector-b.py",
# Name of the output queue directory.
# default: null
# type: string
#
- "queue_out_dir": "/var/mentat/spool/mentat-enricher.py",
+ "queue_out_dir": "mentat-enricher.py",
# Limit on the number of the files for the output queue directory.
# default: 10000
@@ -303,11 +303,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-inspector-c.py.conf b/conf/mentat-inspector-c.py.conf
index c4ae8eda713d18201ce3879d8dde209d333dded1..7afffddf5b7ccaaff50737e404a25b74df977870 100644
--- a/conf/mentat-inspector-c.py.conf
+++ b/conf/mentat-inspector-c.py.conf
@@ -117,16 +117,16 @@
#---------------------------------------------------------------------------
# Name of the input queue directory.
- # default: "/var/mentat/spool/mentat-inspector.py"
+ # default: "mentat-inspector-c.py"
# type: string
#
- #"queue_in_dir": "/var/mentat/spool/mentat-inspector.py",
+ #"queue_in_dir": "mentat-inspector-c.py",
# Name of the output queue directory.
# default: null
# type: string
#
- "queue_out_dir": "/var/mentat/spool/mentat-enricher.py",
+ "queue_out_dir": "mentat-enricher.py",
# Limit on the number of the files for the output queue directory.
# default: 10000
@@ -266,11 +266,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-inspector.py.conf b/conf/mentat-inspector.py.conf
index a675a7db3503089d9d4c380e20ee4e14372df250..139066abf028f50e98029cbbafbc25c89bc6345c 100644
--- a/conf/mentat-inspector.py.conf
+++ b/conf/mentat-inspector.py.conf
@@ -217,16 +217,16 @@
#---------------------------------------------------------------------------
# Name of the input queue directory.
- # default: "/var/mentat/spool/mentat-inspector.py"
+ # default: "mentat-inspector.py"
# type: string
#
- "queue_in_dir": "/var/mentat/spool/mentat-inspector.py",
+ #"queue_in_dir": "mentat-inspector.py",
# Name of the output queue directory.
# default: null
# type: string
#
- "queue_out_dir": "/var/mentat/spool/mentat-inspector-b.py",
+ "queue_out_dir": "mentat-inspector-b.py",
# Limit on the number of the files for the output queue directory.
# default: 10000
@@ -366,11 +366,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-netmngr.py.conf b/conf/mentat-netmngr.py.conf
index 35511083214039383dde2980ca1fedbf8fbfac77..ec4545baf14917482b476d5335e0b46c0b4791f2 100644
--- a/conf/mentat-netmngr.py.conf
+++ b/conf/mentat-netmngr.py.conf
@@ -140,11 +140,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-precache.py.conf b/conf/mentat-precache.py.conf
index eccfc6524703bc605018af9b706df820a85e8f67..aaab73035d7663e2fb07370e31f9b1e0056d9729 100644
--- a/conf/mentat-precache.py.conf
+++ b/conf/mentat-precache.py.conf
@@ -150,11 +150,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-reporter.py.conf b/conf/mentat-reporter.py.conf
index 087c72fbe18c4397ee49ce8be2302fbf815199a0..5bbc6241756dd17bff86f08e265a838f8b10d924 100644
--- a/conf/mentat-reporter.py.conf
+++ b/conf/mentat-reporter.py.conf
@@ -246,11 +246,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-sampler.py.conf b/conf/mentat-sampler.py.conf
index 2d06fb24ecfffea3cf92013db92f7c5e24aa373c..8629a05654dd7076c14129a08324e827d7b19c60 100644
--- a/conf/mentat-sampler.py.conf
+++ b/conf/mentat-sampler.py.conf
@@ -22,16 +22,16 @@
#---------------------------------------------------------------------------
# Name of the input queue directory.
- # default: "/var/mentat/spool/mentat-sampler.py"
+ # default: "mentat-sampler.py"
# type: string
#
- #"queue_in_dir": "/var/mentat/spool/mentat-sampler.py",
+ #"queue_in_dir": "mentat-sampler.py",
# Name of the output queue directory.
# default: null
# type: string
#
- "queue_out_dir": "/var/mentat/spool/mentat-inspector.py",
+ "queue_out_dir": "mentat-inspector.py",
# Limit on the number of the files for the output queue directory.
# default: 10000
@@ -171,11 +171,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-statistician.py.conf b/conf/mentat-statistician.py.conf
index 92910dd1737082d3cd7fc3893231ec554b0887b7..a479183bf04031ab14eae61faa846436697447bf 100644
--- a/conf/mentat-statistician.py.conf
+++ b/conf/mentat-statistician.py.conf
@@ -119,11 +119,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/conf/mentat-storage.py.conf b/conf/mentat-storage.py.conf
index ea66cec2910e97ea95a08fb7b84636f85940c610..e24e84bc8173040622fe56803aa04cfd780248a8 100644
--- a/conf/mentat-storage.py.conf
+++ b/conf/mentat-storage.py.conf
@@ -10,10 +10,10 @@
#---------------------------------------------------------------------------
# Name of the input queue directory.
- # default: "/var/mentat/spool/mentat-storage.py"
+ # default: "mentat-storage.py"
# type: string
#
- #"queue_in_dir": "/var/mentat/spool/mentat-storage.py",
+ #"queue_in_dir": "mentat-storage.py",
# Name of the output queue directory.
# default: null
@@ -159,11 +159,15 @@
# default: null
# type: string or integer
#
- "user": "mentat",
+ #"user": "mentat",
# Name/gid of the group.
# default: null
# type: string or integer
#
- "group": "mentat"
+ #"group": "mentat",
+
+ # This is a dummy last configuration so that the user does not have to fix
+ # the commas in the whole configuration file after each change.
+ "_dummy_": "_dummy_"
}
diff --git a/doc/sphinx/_doclib/development.rst b/doc/sphinx/_doclib/development.rst
index 6f44c97772999c0a4df65cb5709574388f23430d..0cfa511626b96ae8d06ec306a9778da35d81e994 100644
--- a/doc/sphinx/_doclib/development.rst
+++ b/doc/sphinx/_doclib/development.rst
@@ -198,6 +198,49 @@ targets::
(venv) $ make deps-webui-upgrade
+Running development version of Mentat system
+````````````````````````````````````````````````````````````````````````````````
+
+Mentat system can be executed from within the Git repository. We worked hard to
+make sure it will not wander around your filesystem. The key to make everything
+work is the environment variable ``APP_ROOT_PATH``, which controls the base root
+of the application. It is basically a soft implementation of a very lightweight
+chroot. Application can adjust various filesystem paths to the value of this
+variable. During the preparation of development environment a local ``./chroot``
+subdirectory was prepared automatically for you. You may use it in a following
+fashion::
+
+ # A: Use it manually, be carefull to provide the APP_ROOR_PATH as absolute path:
+ (venv) $ APP_ROOT_PATH=$(realpath ./chroot) mentat-controller.py --command start
+
+ # B: For your convenience there is also a makefile target:
+ (venv) $ make run-mentat-dev
+
+For your convenience there is a very handy makefile target ``ctrl-mentat-dev``,
+which is capable of passing commands to your local development instance of
+:ref:`section-bin-mentat-controller`. You may use it in a following fashion::
+
+ # This:
+ (venv) $ make ctrl-mentat-dev COMMAND=start
+ # Is same as this:
+ (venv) $ APP_ROOT_PATH=$(realpath ./chroot) mentat-controller.py --command start
+
+ # This:
+ (venv) $ make ctrl-mentat-dev COMMAND=status
+ # Is same as this:
+ (venv) $ APP_ROOT_PATH=$(realpath ./chroot) mentat-controller.py --command status
+
+ # This:
+ (venv) $ make ctrl-mentat-dev COMMAND=stop
+ # Is same as this:
+ (venv) $ APP_ROOT_PATH=$(realpath ./chroot) mentat-controller.py --command stop
+
+Make sure to read following documentation sections to understand the usage of the
+Mentat system and its various components:
+
+* :ref:`section-quickstart`
+
+
Running development web server
````````````````````````````````````````````````````````````````````````````````
@@ -206,11 +249,11 @@ microframework, that comes with built-in webserver for development. It can be
launched in following ways::
# A: You may use the Flask built-in command in a following way:
- (venv) $ FLASK_APP=hawat FLASK_ENV=development FLASK_CONFIG=development FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf) flask run
+ (venv) $ APP_ROOT_PATH=$(realpath ./chroot) FLASK_APP=hawat FLASK_ENV=development FLASK_CONFIG=development FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf) flask run
# B: You may custom command line interface to launch webserver in development
# mode and with development configuration:
- (venv) $ FLASK_ENV=development FLASK_CONFIG=development FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf) hawat-cli run
+ (venv) $ APP_ROOT_PATH=$(realpath ./chroot) FLASK_ENV=development FLASK_CONFIG=development FLASK_CONFIG_FILE=$(realpath ./hawat.local.conf) hawat-cli run
# C: Use following makefile target to do the same as the three above with less
# typing:
diff --git a/hawat.local.conf b/hawat.local.conf
index ee83f97575bee4d08608eec0f208f757c1a68e64..633a68e4e0587296452cbc4e19f3b722ecdbf22b 100644
--- a/hawat.local.conf
+++ b/hawat.local.conf
@@ -3,5 +3,4 @@ MAIL_SERVER = 'localhost'
MAIL_PORT = 8025
MAIL_DEFAULT_SENDER = 'mentat@cesnet.cz'
HAWAT_LOG_DEFAULT_LEVEL = 'debug'
-HAWAT_LOG_FILE = '/var/tmp/mentat-hawat.py.log'
HAWAT_LOG_FILE_LEVEL = 'debug'
diff --git a/lib/hawat/config.py b/lib/hawat/config.py
index 3a83cafb21166504a8f7a1d5541a9d91c6139f4c..eba27cc61b3046060af2fdd09f3d2b91d92d7c6a 100644
--- a/lib/hawat/config.py
+++ b/lib/hawat/config.py
@@ -43,6 +43,7 @@ __author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+import os
import socket
import collections
@@ -55,6 +56,8 @@ from flask_babel import lazy_gettext
# Custom modules.
#
import pyzenkit.jsonconf
+import pyzenkit.utils
+import mentat.const
import hawat.const
@@ -208,23 +211,29 @@ class Config: # pylint: disable=locally-disabled,too-few-public-methods
]
"""Configuration of application menu skeleton."""
- MENTAT_CORE = pyzenkit.jsonconf.config_load_dir('/etc/mentat/core')
+ MENTAT_CORE = pyzenkit.jsonconf.config_load_dir(
+ pyzenkit.utils.get_resource_path(mentat.const.PATH_CFG_CORE)
+ )
"""Mentat core system configurations."""
MENTAT_PATHS = {
- 'path_bin': '/usr/local/bin',
- 'path_cfg': '/etc/mentat',
- 'path_log': '/var/mentat/log',
- 'path_run': '/var/mentat/run',
- 'path_tmp': '/tmp',
- 'path_crn': '/etc/cron.d'
+ 'path_crn': pyzenkit.utils.get_resource_path(mentat.const.PATH_CRN),
+ 'path_cfg': pyzenkit.utils.get_resource_path(mentat.const.PATH_CFG),
+ 'path_var': pyzenkit.utils.get_resource_path(mentat.const.PATH_VAR),
+ 'path_log': pyzenkit.utils.get_resource_path(mentat.const.PATH_LOG),
+ 'path_run': pyzenkit.utils.get_resource_path(mentat.const.PATH_RUN),
+ 'path_tmp': pyzenkit.utils.get_resource_path(mentat.const.PATH_TMP),
}
"""Paths to various directories."""
- MENTAT_CACHE_DIR = '/var/mentat/cache'
+ MENTAT_CACHE_DIR = pyzenkit.utils.get_resource_path(
+ os.path.join(mentat.const.PATH_VAR, 'cache')
+ )
"""Path to Mentat cache directory."""
- MENTAT_CONTROLLER_CFG = '/etc/mentat/mentat-controller.py.conf'
+ MENTAT_CONTROLLER_CFG = pyzenkit.utils.get_resource_path(
+ os.path.join(mentat.const.PATH_CFG, 'mentat-controller.py.conf')
+ )
"""path to configuration file of mentat-controller.py module."""
HAWAT_ADMINS = ['root@{}'.format(socket.getfqdn())]
@@ -233,7 +242,9 @@ class Config: # pylint: disable=locally-disabled,too-few-public-methods
HAWAT_LOG_DEFAULT_LEVEL = 'info'
"""Default logging level, case insensitive. One of the values ``DEBUG``, ``INFO``, ``WARNING``, ``ERROR``, ``CRITICAL``."""
- HAWAT_LOG_FILE = '/var/mentat/log/mentat-hawat.py.log'
+ HAWAT_LOG_FILE = pyzenkit.utils.get_resource_path(
+ os.path.join(mentat.const.PATH_LOG, 'mentat-hawat.py.log')
+ )
"""Log file settings for logging framework."""
HAWAT_LOG_FILE_LEVEL = 'info'
@@ -301,9 +312,6 @@ class DevelopmentConfig(Config): # pylint: disable=locally-disabled,too-few-pub
HAWAT_LOGIN_VIEW = 'auth_dev.login'
"""Overwrite default :py:const:`hawat.config.Config.HAWAT_LOGIN_VIEW`."""
- HAWAT_LOG_FILE = '/var/tmp/mentat-hawat.py.log'
- """Overwrite default :py:const:`hawat.config.Config.HAWAT_LOG_FILE`."""
-
class TestingConfig(Config): # pylint: disable=locally-disabled,too-few-public-methods
"""
diff --git a/lib/mentat/const.py b/lib/mentat/const.py
index 50204f7eca031d325db717704cc3b2e6acb35ef1..58704987ccfd1dbb0dc0ab8c2ba5674d183bf6fd 100644
--- a/lib/mentat/const.py
+++ b/lib/mentat/const.py
@@ -32,14 +32,25 @@ def tr_(val):
#-------------------------------------------------------------------------------
+PATH_BIN = 'usr/local/bin'
+PATH_CRN = 'etc/cron.d'
+PATH_CFG = 'etc/mentat'
+PATH_CFG_CORE = os.path.join(PATH_CFG, 'core')
+PATH_VAR = 'var/mentat'
+PATH_LOG = os.path.join(PATH_VAR, 'log')
+PATH_RUN = os.path.join(PATH_VAR, 'run')
+PATH_TMP = os.path.join(PATH_VAR, 'tmp')
+
+#-------------------------------------------------------------------------------
+
DFLT_EVENT_START = 'start'
"""Default name for the *start* event."""
-DFLT_EVENT_RELOAD = 'reload'
-"""Default name for the *reload* event."""
DFLT_EVENT_LOG_STATISTICS = 'log_statistics'
"""Default name for the *log_statistics* event."""
DFLT_EVENT_SAVE_RUNLOG = 'save_runlog'
"""Default name for the *save_runlog* event."""
+DFLT_EVENT_RELOAD = 'reload'
+"""Default name for the *reload* event."""
DFLT_INTERVAL_RELOAD = 300
"""Default time interval in seconds for reloading internal caches."""
@@ -196,9 +207,11 @@ NAGIOS_PLUGIN_RC_WARNING = 1
NAGIOS_PLUGIN_RC_CRITICAL = 2
NAGIOS_PLUGIN_RC_UNKNOWN = 3
+
RE_REPORT_FILE_TIMESTAMP = re.compile(r"^[^\d]*([\d]{8})")
"""Regular expression for parsing out the report timestamp from attachment file name."""
+
def construct_report_dirpath(basedirpath, filename, check = False):
r"""
Construct target report directory path based on given base path and file name.
diff --git a/lib/mentat/daemon/base.py b/lib/mentat/daemon/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e78fc5c8f8f77c73ce075b840145a7e63b60b5a
--- /dev/null
+++ b/lib/mentat/daemon/base.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This module provides base implementation of all Mentat daemons.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+#
+# Custom libraries.
+#
+import pyzenkit.baseapp
+import pyzenkit.zendaemon
+import mentat.const
+
+
+DEFAULT_EVENTS_SCHEDULED = [
+ (mentat.const.DFLT_EVENT_START,)
+]
+DEFAULT_EVENTS_SCHEDULED_AFTER = [
+ (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS),
+ (mentat.const.DFLT_INTERVAL_RUNLOG, mentat.const.DFLT_EVENT_SAVE_RUNLOG)
+]
+
+class MentatBaseDaemon(pyzenkit.zendaemon.ZenDaemon):
+ """
+ This module provides base implementation of generic *pipe-like* message processing
+ daemon.
+ """
+
+ def __init__(self, **kwargs):
+ #
+ # Configure required script paths.
+ #
+ kwargs.setdefault('path_cfg', mentat.const.PATH_CFG)
+ kwargs.setdefault('path_var', mentat.const.PATH_VAR)
+ kwargs.setdefault('path_log', mentat.const.PATH_LOG)
+ kwargs.setdefault('path_run', mentat.const.PATH_RUN)
+ kwargs.setdefault('path_tmp', mentat.const.PATH_TMP)
+
+ #
+ # Override default configurations.
+ #
+ kwargs.setdefault(
+ 'default_config_dir',
+ self.get_resource_path(mentat.const.PATH_CFG_CORE)
+ )
+ kwargs.setdefault(
+ 'default_stats_interval',
+ mentat.const.DFLT_INTERVAL_STATISTICS
+ )
+ kwargs.setdefault(
+ 'default_runlog_interval',
+ mentat.const.DFLT_INTERVAL_RUNLOG
+ )
+
+ #
+ # Schedule initial events.
+ #
+ kwargs.setdefault('schedule', DEFAULT_EVENTS_SCHEDULED)
+ kwargs.setdefault('schedule_after', DEFAULT_EVENTS_SCHEDULED_AFTER)
+
+ super().__init__(**kwargs)
diff --git a/lib/mentat/daemon/piper.py b/lib/mentat/daemon/piper.py
index 2399004b9a10308feee19c2238ccd348b0a8081b..00566b2c9060d4282ded95572ba5f64b00d736fd 100644
--- a/lib/mentat/daemon/piper.py
+++ b/lib/mentat/daemon/piper.py
@@ -109,12 +109,13 @@ import os
#
import pyzenkit.baseapp
import pyzenkit.zendaemon
+import mentat.daemon.base
import mentat.const
import mentat.dirq
import mentat.daemon.component.filer
-class PiperDaemon(pyzenkit.zendaemon.ZenDaemon):
+class PiperDaemon(mentat.daemon.base.MentatBaseDaemon):
"""
This module provides base implementation of generic *pipe-like* message processing
daemon.
@@ -129,6 +130,14 @@ class PiperDaemon(pyzenkit.zendaemon.ZenDaemon):
CONFIG_QUEUE_OUT_LIMIT = 'queue_out_limit'
CONFIG_QUEUE_OUT_WAIT = 'queue_out_wait'
+ def __init__(self, **kwargs):
+ #
+ # Override default configurations.
+ #
+ kwargs.setdefault('default_queue_out_dir', None)
+
+ super().__init__(**kwargs)
+
def _init_argparser(self, **kwargs):
"""
Initialize piper daemon command line argument parser. This method overrides the
@@ -150,10 +159,30 @@ class PiperDaemon(pyzenkit.zendaemon.ZenDaemon):
#
arggroup_daemon = argparser.add_argument_group('common piper daemon arguments')
- arggroup_daemon.add_argument('--queue-in-dir', help='name of the input queue directory', type = str, default = None)
- arggroup_daemon.add_argument('--queue-out-dir', help='name of the output queue directory', type = str, default = None)
- arggroup_daemon.add_argument('--queue-out-limit', help='limit on the number of the files for the output queue directory', type = int, default = None)
- arggroup_daemon.add_argument('--queue-out-wait', help='waiting time when the output queue limit is reached in seconds', type = int, default = None)
+ arggroup_daemon.add_argument(
+ '--queue-in-dir',
+ type = str,
+ default = None,
+ help = 'name of the input queue directory'
+ )
+ arggroup_daemon.add_argument(
+ '--queue-out-dir',
+ type = str,
+ default = None,
+ help = 'name of the output queue directory',
+ )
+ arggroup_daemon.add_argument(
+ '--queue-out-limit',
+ type = int,
+ default = None,
+ help = 'limit on the number of the files for the output queue directory'
+ )
+ arggroup_daemon.add_argument(
+ '--queue-out-wait',
+ type = int,
+ default = None,
+ help = 'waiting time when the output queue limit is reached in seconds'
+ )
return argparser
@@ -172,10 +201,10 @@ class PiperDaemon(pyzenkit.zendaemon.ZenDaemon):
:rtype: dict
"""
cfgs = (
- (self.CONFIG_QUEUE_IN_DIR, os.path.join(self.paths['run'], "{}.queue".format(self.name))),
- (self.CONFIG_QUEUE_OUT_DIR, False),
- (self.CONFIG_QUEUE_OUT_LIMIT, 10000),
- (self.CONFIG_QUEUE_OUT_WAIT, 30),
+ (self.CONFIG_QUEUE_IN_DIR, self.name),
+ (self.CONFIG_QUEUE_OUT_DIR, None),
+ (self.CONFIG_QUEUE_OUT_LIMIT, mentat.const.DFLT_QUEUE_SIZE_LIMIT),
+ (self.CONFIG_QUEUE_OUT_WAIT, mentat.const.DFLT_QUEUE_CHECK_INTERVAL),
) + cfgs
return super()._init_config(cfgs, **kwargs)
@@ -203,12 +232,24 @@ class PiperDaemon(pyzenkit.zendaemon.ZenDaemon):
super()._configure_postprocess()
ccfg = {}
- ccfg[mentat.dirq.DirectoryQueue.CONFIG_DIR_QUEUE] = self.c(self.CONFIG_QUEUE_IN_DIR)
- ccfg[mentat.dirq.DirectoryQueue.CONFIG_DIR_NEXT_QUEUE] = self.c(self.CONFIG_QUEUE_OUT_DIR)
+ ccfg[mentat.dirq.DirectoryQueue.CONFIG_DIR_QUEUE] = self._get_queue_dir_path(
+ self.c(self.CONFIG_QUEUE_IN_DIR)
+ )
+ ccfg[mentat.dirq.DirectoryQueue.CONFIG_DIR_NEXT_QUEUE] = self._get_queue_dir_path(
+ self.c(self.CONFIG_QUEUE_OUT_DIR)
+ )
ccfg[mentat.daemon.component.filer.FilerDaemonComponent.CONFIG_QUEUE_OUT_LIMIT] = self.c(self.CONFIG_QUEUE_OUT_LIMIT)
ccfg[mentat.daemon.component.filer.FilerDaemonComponent.CONFIG_QUEUE_OUT_WAIT] = self.c(self.CONFIG_QUEUE_OUT_WAIT)
self.config[self.CORE][self.CORE_FILEQUEUE] = ccfg
+ def _get_queue_dir_path(self, queue_dir):
+ if not queue_dir:
+ return queue_dir
+ queue_dir = str(queue_dir)
+ if not os.path.dirname(queue_dir):
+ return os.path.join(self.paths[self.PATH_VAR], 'spool', queue_dir)
+ return queue_dir
+
class DemoPrintComponent(pyzenkit.zendaemon.ZenDaemonComponent):
"""
@@ -258,11 +299,12 @@ class DemoPiperDaemon(PiperDaemon):
#
# Configure required deamon paths.
#
- path_bin = '/tmp',
- path_cfg = '/tmp',
- path_log = '/tmp',
- path_tmp = '/tmp',
- path_run = '/tmp',
+ path_bin = 'tmp',
+ path_cfg = 'tmp',
+ path_var = 'tmp',
+ path_log = 'tmp',
+ path_tmp = 'tmp',
+ path_run = 'tmp',
#
# Override default configurations.
@@ -284,7 +326,7 @@ class DemoPiperDaemon(PiperDaemon):
],
schedule_after = [
(mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS),
- (mentat.const.DFLT_INTERVAL_RUNLOG, mentat.const.DFLT_EVENT_SAVE_RUNLOG)
+ (mentat.const.DFLT_INTERVAL_RUNLOG, mentat.const.DFLT_EVENT_SAVE_RUNLOG)
],
#
diff --git a/lib/mentat/dirq.py b/lib/mentat/dirq.py
index d26555f0e00d17dec59ef17490d07d1fb9d88425..8f0f38f705f138f094ec6c9960bda9da80bf5737 100644
--- a/lib/mentat/dirq.py
+++ b/lib/mentat/dirq.py
@@ -229,7 +229,7 @@ class DirectoryQueue:
Fetch next file from incoming queue.
"""
while True:
- if not len(self.queue):
+ if not self.queue:
self._rescan_queue()
try:
itemid = self.queue.pop()
diff --git a/lib/mentat/module/backup.py b/lib/mentat/module/backup.py
index 222ad4068f08d734ea0386259fc9157aba7f6e5a..b20faf42669fb0e277600f4eaf36ce1b07d8b485 100644
--- a/lib/mentat/module/backup.py
+++ b/lib/mentat/module/backup.py
@@ -132,9 +132,10 @@ import pprint
#
import pyzenkit.zenscript
import mentat.const
+import mentat.script.base
-class MentatBackupScript(pyzenkit.zenscript.ZenScript):
+class MentatBackupScript(mentat.script.base.MentatBaseScript):
"""
Implementation of Mentat module (script) providing database backup functions
and features.
@@ -161,22 +162,7 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
configuration values for parent contructor.
"""
super().__init__(
-
- description = 'mentat-backup.py - PostgreSQL database backup script for Mentat system',
-
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core'
+ description = 'mentat-backup.py - PostgreSQL database backup script for Mentat system'
)
def _init_argparser(self, **kwargs):
@@ -200,12 +186,42 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
#
arggroup_script = argparser.add_argument_group('custom script arguments')
- arggroup_script.add_argument('--no-upload', help = 'do not upload the backup file to remote storage (flag)', action='store_true', default = None)
- arggroup_script.add_argument('--mount-point', help = 'name of the mount point directory', type = str, default = None)
- arggroup_script.add_argument('--temp-dir', help = 'name of the temporary file directory', type = str, default = None)
- arggroup_script.add_argument('--backup-dir', help = 'name of the database backup directory', type = str, default = None)
- arggroup_script.add_argument('--remote-host', help = 'SSH connection string for the remote host to which to sychronize backups', type = str, default = None)
- arggroup_script.add_argument('--remote-dir', help = 'directory path on the remote host to which to sychronize backups', type = str, default = None)
+ arggroup_script.add_argument(
+ '--no-upload',
+ help = 'do not upload the backup file to remote storage (flag)',
+ action = 'store_true',
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--mount-point',
+ help = 'name of the local mount point directory for remote storage',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--temp-dir',
+ help = 'name of the local temporary file directory',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--backup-dir',
+ help = 'name of the local database backup directory',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--remote-host',
+ help = 'SSH connection string for the remote host to which to sychronize backups',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--remote-dir',
+ help = 'directory path on the remote host to which to sychronize backups',
+ type = str,
+ default = None
+ )
return argparser
@@ -225,9 +241,9 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
"""
cfgs = (
(self.CONFIG_NO_UPLOAD, False),
- (self.CONFIG_MOUNT_POINT, '/media/du-store'),
- (self.CONFIG_TEMP_DIR, '/var/tmp'),
- (self.CONFIG_BACKUP_DIR, '/var/mentat/backups'),
+ (self.CONFIG_MOUNT_POINT, self.get_resource_path('media/ds-store')),
+ (self.CONFIG_TEMP_DIR, self.paths[self.PATH_TMP]),
+ (self.CONFIG_BACKUP_DIR, os.path.join(self.paths[self.PATH_VAR], 'backups')),
(self.CONFIG_REMOTE_HOST, None),
(self.CONFIG_REMOTE_DIR, None),
) + cfgs
diff --git a/lib/mentat/module/cleanup.py b/lib/mentat/module/cleanup.py
index fe70cf7ce8fefb29aefddd54545c2c4efa4b413b..cfe196e860c4f6ff2403e5548be2771c310741cc 100644
--- a/lib/mentat/module/cleanup.py
+++ b/lib/mentat/module/cleanup.py
@@ -104,7 +104,7 @@ import json
#
import pydgets.widgets
import mentat.script.fetcher
-
+import mentat.const
#
# Global variables.
@@ -150,6 +150,7 @@ def json_dump_cbk(obj):
return obj.isoformat()
return repr(obj)
+
class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
"""
Implementation of Mentat module (script) providing database and cache cleanup
@@ -180,22 +181,7 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
self.sqlservice = None
super().__init__(
-
- description = 'mentat-cleanup.py - Mentat system database and cache cleanup script',
-
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core'
+ description = 'mentat-cleanup.py - Mentat system database and cache cleanup script'
)
def _init_argparser(self, **kwargs):
@@ -219,8 +205,18 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
#
arggroup_script = argparser.add_argument_group('custom script arguments')
- arggroup_script.add_argument('--db-path', help = 'filesystem path to database files', type = str, default = None)
- arggroup_script.add_argument('--simulate', help = 'perform simulation, do not remove anything (flag)', action='store_true', default = None)
+ arggroup_script.add_argument(
+ '--db-path',
+ help = 'filesystem path to database files',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--simulate',
+ help = 'perform simulation, do not remove anything (flag)',
+ action = 'store_true',
+ default = None
+ )
return argparser
@@ -239,7 +235,7 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
:rtype: dict
"""
cfgs = (
- (self.CONFIG_DB_PATH, '/var/lib/postgresql/10/main'),
+ (self.CONFIG_DB_PATH, '/var/lib/postgresql/11/main'),
(self.CONFIG_SIMULATE, False),
(self.CONFIG_EVENTS, []),
(self.CONFIG_TABLES, []),
diff --git a/lib/mentat/module/controller.py b/lib/mentat/module/controller.py
index f60159b2f2c3423bbd2cd7c74e0c5e3766071140..55a79d06616cb8dab6618753288976fc254f3ad0 100644
--- a/lib/mentat/module/controller.py
+++ b/lib/mentat/module/controller.py
@@ -170,10 +170,9 @@ import pprint
#
# Custom libraries
#
-import pyzenkit.baseapp
-import pyzenkit.zenscript
import mentat.const
import mentat.system
+import mentat.script.base
# Translation table to translate signal numbers to their names.
@@ -183,7 +182,7 @@ SIGNALS_TO_NAMES_DICT = dict((getattr(signal, n), n) \
CRON_SCRIPT_DIR = '/etc/cron.d'
-class MentatControllerScript(pyzenkit.zenscript.ZenScript):
+class MentatControllerScript(mentat.script.base.MentatBaseScript):
"""
Implementation of Mentat module (script) providing Mentat system control
@@ -208,23 +207,11 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
it aims to even more simplify the script object creation by providing
configuration values for parent contructor.
"""
- super().__init__(
+ self.modules = None
+ self.cronjobs = None
- description = 'mentat-controller.py - Mentat system control script',
-
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core'
+ super().__init__(
+ description = 'mentat-controller.py - Mentat system control script'
)
def _init_argparser(self, **kwargs):
@@ -251,8 +238,19 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
arggroup_script = argparser.add_argument_group('custom script arguments')
# Specify the target module(s) for the command
- arggroup_script.add_argument('--target', help='target module(s) for the current command (*repeatable*)', action='append', type = str, default = None)
- arggroup_script.add_argument('--nagios-plugin', help = 'execute as Nagios plugin (flag)', action='store_true', default = None)
+ arggroup_script.add_argument(
+ '--target',
+ help = 'target module(s) for the current command (*repeatable*)',
+ action = 'append',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--nagios-plugin',
+ help = 'execute in Nagios plugin compatible mode (flag)',
+ action = 'store_true',
+ default = None
+ )
return argparser
@@ -272,7 +270,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
"""
cfgs = (
- (self.CONFIG_TARGET, None),
+ (self.CONFIG_TARGET, None),
(self.CONFIG_NAGIOS_PLUGIN, False)
) + cfgs
return super()._init_config(cfgs, **kwargs)
@@ -284,8 +282,12 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
This method is called from the main setup method :py:func:`pyzenkit.baseapp.BaseApp._stage_setup`
as a part of the **setup** stage of application`s life cycle.
"""
- self.modules = mentat.system.make_module_list(self.c(self.CONFIG_MODULES)) # pylint: disable=locally-disabled,attribute-defined-outside-init
- self.cronjobs = mentat.system.make_cronjob_list(self.c(self.CONFIG_CRONJOBS)) # pylint: disable=locally-disabled,attribute-defined-outside-init
+ self.modules = mentat.system.make_module_list(
+ self.c(self.CONFIG_MODULES)
+ )
+ self.cronjobs = mentat.system.make_cronjob_list(
+ self.c(self.CONFIG_CRONJOBS)
+ )
#---------------------------------------------------------------------------
@@ -635,7 +637,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
if stat[0] == mentat.system.STATUS_RT_RUNNING_OK:
self._module_signal(self.modules[mname], proc, signal.SIGUSR1)
else:
- self.logger.error("Module '%s': Unable to send signal", mname)
+ self.logger.error("Module '%s': Unable to send SIGUSR1 signal", mname)
return self.RESULT_SUCCESS
@@ -659,7 +661,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
if stat[0] == mentat.system.STATUS_RT_RUNNING_OK:
self._module_signal(self.modules[mname], proc, signal.SIGUSR2)
else:
- self.logger.error("Module '%s': Unable to send signal", mname)
+ self.logger.error("Module '%s': Unable to send SIGUSR2 signal", mname)
return self.RESULT_SUCCESS
@@ -714,13 +716,17 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
#---------------------------------------------------------------------------
- def _prepare_command(self, mod_data):
+ @staticmethod
+ def _prepare_command(mod_data):
"""
Prepare system command for execution of given module.
"""
- path = os.path.join(self.paths.get(self.PATH_BIN), mod_data.executable)
+ path = mod_data.executable
+ # From documentation https://docs.python.org/3/library/os.html#os.execv:
+ # "...the arguments to the child process should start with the name of
+ # the command being run, but this is not enforced..."
args = [path,]
- if mod_data.name != mod_data.executable:
+ if mod_data.name != os.path.basename(mod_data.executable):
args = args + ['--name', mod_data.name]
if mod_data.paralel:
args = args + ['--paralel',]
@@ -736,14 +742,30 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
newpid = os.fork()
if newpid == 0:
- self.logger.info("Module '%s': Worker process '%d' ready, executing command '%s' with arguments '%s'", mod_data.name, os.getpid(), executable, pprint.pformat(arguments))
+ self.logger.info(
+ "Module '%s': Worker process '%d' ready, executing command '%s' with arguments '%s'",
+ mod_data.name,
+ os.getpid(),
+ executable,
+ pprint.pformat(arguments)
+ )
os.sync()
-
- os.execv(executable, arguments)
- self.logger.critical("Module '%s': Worker process '%d' was unable to execute module", mod_data.name, os.getpid())
- os._exit(0) # pylint: disable=locally-disabled,protected-access
+ os.execvp(executable, arguments)
+
+ # In case the os.execv() call was successfull we should never reach
+ # this point in code. If we are here, there was a critical error.
+ self.logger.critical(
+ "Module '%s': Worker process '%d' was unable to execute module",
+ mod_data.name,
+ os.getpid()
+ )
+ os._exit(1) # pylint: disable=locally-disabled,protected-access
else:
- self.logger.info("Module '%s': Waiting for worker process '%d'", mod_data.name, newpid)
+ self.logger.info(
+ "Module '%s': Waiting for worker process '%d'",
+ mod_data.name,
+ newpid
+ )
time.sleep(1)
def _module_start(self, mod_data, processes):
@@ -757,7 +779,11 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
instances = instances - len(processes.keys())
for i in range(0, instances):
- self.logger.info("Module '%s': Launching instance '%d'", mod_data.name, i)
+ self.logger.info(
+ "Module '%s': Launching instance '%d'",
+ mod_data.name,
+ i
+ )
self._execute(mod_data)
def _module_enable(self, mod_data, meta):
@@ -790,14 +816,19 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
def _signal(self, mod_data, pid, sig):
"""
- Send given signal to given process.
+ Send given signal to given system process.
"""
- self.logger.info("Module '%s': Sending signal '%s' to process '%d'", mod_data.name, SIGNALS_TO_NAMES_DICT.get(sig, sig), pid)
+ self.logger.info(
+ "Module '%s': Sending signal '%s' to process '%d'",
+ mod_data.name,
+ SIGNALS_TO_NAMES_DICT.get(sig, sig),
+ pid
+ )
os.kill(pid, sig)
def _module_signal(self, mod_data, processes, sig):
"""
- Stop given module.
+ Send given signal to given module.
"""
for pid in sorted(processes.keys()):
self._signal(mod_data, pid, sig)
diff --git a/lib/mentat/module/dbmngr.py b/lib/mentat/module/dbmngr.py
index 2b1994b87b14a9d99456f35d07426b7c235b46fa..51066318732828ab2f2899bae3cc7ba2d80ff45e 100644
--- a/lib/mentat/module/dbmngr.py
+++ b/lib/mentat/module/dbmngr.py
@@ -175,22 +175,7 @@ class MentatDbmngrScript(mentat.script.fetcher.FetcherScript):
self.sqlservice = None
super().__init__(
-
- description = 'mentat-dbmngr.py - Mentat database management script',
-
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core'
+ description = 'mentat-dbmngr.py - Mentat database management script'
)
def _init_argparser(self, **kwargs):
@@ -214,12 +199,41 @@ class MentatDbmngrScript(mentat.script.fetcher.FetcherScript):
#
arggroup_script = argparser.add_argument_group('custom script arguments')
- arggroup_script.add_argument('--watchdog-delta', type = int, default = None, help = 'time interval delta in seconds for watchdog checks')
- arggroup_script.add_argument('--mail-subject', type = str, default = None, help = 'subject for the database watchdog emails')
- arggroup_script.add_argument('--mail-from', type = str, default = None, help = 'source email address for the database watchdog emails')
- arggroup_script.add_argument('--mail-to', type = str, default = None, help = 'target email address for the database watchdog emails')
- arggroup_script.add_argument('--nagios-plugin', help = 'execute as Nagios plugin (flag)', action='store_true', default = None)
- arggroup_script.add_argument('additional_args', help = 'optional additional arguments', nargs='*')
+ arggroup_script.add_argument(
+ '--watchdog-delta',
+ type = int,
+ default = None,
+ help = 'time interval delta in seconds for watchdog checks'
+ )
+ arggroup_script.add_argument(
+ '--mail-subject',
+ type = str,
+ default = None,
+ help = 'subject for the database watchdog emails'
+ )
+ arggroup_script.add_argument(
+ '--mail-from',
+ type = str,
+ default = None,
+ help = 'source email address for the database watchdog emails'
+ )
+ arggroup_script.add_argument(
+ '--mail-to',
+ type = str,
+ default = None,
+ help = 'target email address for the database watchdog emails'
+ )
+ arggroup_script.add_argument(
+ '--nagios-plugin',
+ action='store_true',
+ default = None,
+ help = 'execute as Nagios plugin (flag)'
+ )
+ arggroup_script.add_argument(
+ 'additional_args',
+ nargs='*',
+ help = 'optional additional arguments'
+ )
return argparser
diff --git a/lib/mentat/module/enricher.py b/lib/mentat/module/enricher.py
index 35cc2a24fe1e5b50b27bd1ed7615f61c654b0506..4646739e5556cadb3b975fa59648f83222aa1aca 100644
--- a/lib/mentat/module/enricher.py
+++ b/lib/mentat/module/enricher.py
@@ -46,6 +46,7 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea
# Custom libraries.
#
import mentat.const
+import mentat.daemon.base
import mentat.daemon.piper
import mentat.plugin.app.sqlstorage
import mentat.daemon.component.parser
@@ -70,35 +71,16 @@ class MentatEnricherDaemon(mentat.daemon.piper.PiperDaemon):
description = 'mentat-enricher.py - IDEA message enrichment daemon',
- #
- # Configure required deamon paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/var/tmp',
-
#
# Override default configurations.
#
- default_config_dir = '/etc/mentat/core',
- default_queue_in_dir = '/var/mentat/spool/mentat-enricher.py',
- default_queue_out_dir = None,
default_reload_interval = mentat.const.DFLT_INTERVAL_RELOAD,
- default_stats_interval = mentat.const.DFLT_INTERVAL_STATISTICS,
- default_runlog_interval = mentat.const.DFLT_INTERVAL_RUNLOG,
#
# Schedule initial events.
#
- schedule = [
- (mentat.const.DFLT_EVENT_START,)
- ],
- schedule_after = [
- (mentat.const.DFLT_INTERVAL_RELOAD, mentat.const.DFLT_EVENT_RELOAD),
- (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS),
- (mentat.const.DFLT_INTERVAL_RUNLOG, mentat.const.DFLT_EVENT_SAVE_RUNLOG)
+ schedule_after = mentat.daemon.base.DEFAULT_EVENTS_SCHEDULED_AFTER + [
+ (mentat.const.DFLT_INTERVAL_RELOAD, mentat.const.DFLT_EVENT_RELOAD)
],
#
@@ -139,7 +121,12 @@ class MentatEnricherDaemon(mentat.daemon.piper.PiperDaemon):
#
arggroup_daemon = argparser.add_argument_group('custom daemon arguments')
- arggroup_daemon.add_argument('--reload-interval', type = int, default = None, help = 'time interval for reloading enrichment plugins in seconds')
+ arggroup_daemon.add_argument(
+ '--reload-interval',
+ type = int,
+ default = None,
+ help = 'time interval for reloading enrichment plugins in seconds'
+ )
return argparser
diff --git a/lib/mentat/module/ideagen.py b/lib/mentat/module/ideagen.py
index 66da7d5ed9010a27d182a5d209a916140bc7e603..a89b02a4dcd178620b040eee914b1cc0e85b8fef 100644
--- a/lib/mentat/module/ideagen.py
+++ b/lib/mentat/module/ideagen.py
@@ -127,13 +127,14 @@ from jinja2 import Environment, FileSystemLoader
#
# Custom libraries
#
-import pyzenkit.zenscript
+import mentat.const
+import mentat.script.base
#-------------------------------------------------------------------------------
-class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
+class MentatIdeagenScript(mentat.script.base.MentatBaseScript):
"""
Implementation of Mentat module (script) providing functions for generating
`IDEA <https://idea.cesnet.cz/en/index>`__ messages, mainly for testing or
@@ -174,23 +175,10 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
it aims to even more simplify the script object creation by providing
configuration values for parent contructor.
"""
- super().__init__(
+ self.templ_env = None
+ super().__init__(
description = 'mentat-ideagen.py - IDEA message generation script for Mentat system',
-
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core'
)
def _init_argparser(self, **kwargs):
@@ -216,15 +204,55 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
# Setup mutually exclusive group for regular counter vs. random counter.
group_a = arggroup_script.add_mutually_exclusive_group()
- group_a.add_argument('--count', help = 'number of messages to be generated in one batch', type = int, default = None)
- group_a.add_argument('--random-count', help = 'generate random number of messages', type = int, default = None)
+ group_a.add_argument(
+ '--count',
+ help = 'number of messages to be generated in one batch',
+ type = int,
+ default = None
+ )
+ group_a.add_argument(
+ '--random-count',
+ help = 'generate random number of messages',
+ type = int,
+ default = None
+ )
- arggroup_script.add_argument('--steady', help = 'generate messages continuously with given time backoff (flag)', action='store_true', default = None)
- arggroup_script.add_argument('--back-off', help = 'back-off time between message batches in seconds', type = int, default = None)
- arggroup_script.add_argument('--queue-dir', help = 'name of the target queue directory', type = str, default = None)
- arggroup_script.add_argument('--temp-dir', help = 'name of the temporary file directory', type = str, default = None)
- arggroup_script.add_argument('--template', help = 'name of template file for generating messages', type = str, default = None)
- arggroup_script.add_argument('--template-dir', help = 'name of the directory containing message templates', type = str, default = None)
+ arggroup_script.add_argument(
+ '--steady',
+ help = 'generate messages continuously with given time backoff (flag)',
+ action = 'store_true',
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--back-off',
+ help = 'back-off time between message batches in seconds',
+ type = int,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--queue-dir',
+ help = 'name of the target queue directory',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--temp-dir',
+ help = 'name of the temporary file directory',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--template',
+ help = 'name of template file for generating messages',
+ type = str,
+ default = None
+ )
+ arggroup_script.add_argument(
+ '--template-dir',
+ help = 'name of the directory containing message templates',
+ type = str,
+ default = None
+ )
return argparser
@@ -247,10 +275,10 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
(self.CONFIG_RANDOM_COUNT, None),
(self.CONFIG_STEADY, False),
(self.CONFIG_BACK_OFF, 60),
- (self.CONFIG_QUEUE_DIR, '/var/tmp'),
- (self.CONFIG_TEMP_DIR, '/var/tmp'),
+ (self.CONFIG_QUEUE_DIR, os.path.join(self.paths[self.PATH_VAR], 'spool', 'mentat-inspector.py', 'incoming')),
+ (self.CONFIG_TEMP_DIR, os.path.join(self.paths[self.PATH_VAR], 'spool', 'mentat-inspector.py', 'tmp')),
(self.CONFIG_TEMPLATE, 'msg.01.idea.j2'),
- (self.CONFIG_TEMPLATE_DIR, os.path.join(self.paths.get(self.PATH_CFG), 'templates', 'idea')),
+ (self.CONFIG_TEMPLATE_DIR, os.path.join(self.paths[self.PATH_CFG], 'templates', 'idea')),
(self.CONFIG_LIST_SOURCE_IP4S, []),
(self.CONFIG_LIST_TARGET_IP4S, []),
@@ -274,7 +302,9 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
This method is called from the main setup method :py:func:`pyzenkit.baseapp.BaseApp._stage_setup`
as a part of the **setup** stage of application`s life cycle.
"""
- self.templ_env = Environment(loader = FileSystemLoader(self.c(self.CONFIG_TEMPLATE_DIR)))
+ self.templ_env = Environment(
+ loader = FileSystemLoader(self.c(self.CONFIG_TEMPLATE_DIR))
+ )
#---------------------------------------------------------------------------
@@ -300,14 +330,18 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
"""
result = {'cnt_messages': 0, 'cnt_batches': 0}
- self.dbgout("Using template '{}' to generate message(s)".format(self.c(self.CONFIG_TEMPLATE)))
template = self.templ_env.get_template(self.c(self.CONFIG_TEMPLATE))
-
counter = self.c(self.CONFIG_RANDOM_COUNT)
if counter:
counter = random.randint(1, counter)
else:
counter = self.c(self.CONFIG_COUNT)
+ self.dbgout(
+ "Using template '{}' to generate {:d} message(s)".format(
+ self.c(self.CONFIG_TEMPLATE),
+ counter
+ )
+ )
try:
while True:
@@ -332,6 +366,12 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
)
time.sleep(backoff)
+ except FileNotFoundError as err:
+ self.logger.critical("%s", str(err))
+
+ except PermissionError as err:
+ self.logger.critical("%s", str(err))
+
except KeyboardInterrupt:
pass
diff --git a/lib/mentat/module/informant.py b/lib/mentat/module/informant.py
index 558540ee221fff5f0ee0b77e663b4f70f051f3cf..f1747fcfe9e89104887ba2d85519a5c13e43794e 100644
--- a/lib/mentat/module/informant.py
+++ b/lib/mentat/module/informant.py
@@ -91,20 +91,6 @@ class MentatInformantScript(mentat.script.fetcher.FetcherScript):
description = 'mentat-informant.py - Mentat system overall performance statistics',
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core',
-
#
# Load additional application-level plugins.
#
@@ -149,9 +135,24 @@ class MentatInformantScript(mentat.script.fetcher.FetcherScript):
#
arggroup_script = argparser.add_argument_group('custom script arguments')
- arggroup_script.add_argument('--force-template', type = str, default = None, help = 'force a template for generating reports')
- arggroup_script.add_argument('--force-locale', type = str, default = None, help = 'force a locale for generating reports')
- arggroup_script.add_argument('--force-timezone', type = str, default = None, help = 'force a timezone for generating reports')
+ arggroup_script.add_argument(
+ '--force-template',
+ type = str,
+ default = None,
+ help = 'force a template for generating reports'
+ )
+ arggroup_script.add_argument(
+ '--force-locale',
+ type = str,
+ default = None,
+ help = 'force a locale for generating reports'
+ )
+ arggroup_script.add_argument(
+ '--force-timezone',
+ type = str,
+ default = None,
+ help = 'force a timezone for generating reports'
+ )
return argparser
diff --git a/lib/mentat/module/inspector.py b/lib/mentat/module/inspector.py
index 04cfda0f5e7060200a2f690ed51c5cf0ba6ce0db..4ab94cd14ab4cf60ecf55068dab462b7d8c54968 100644
--- a/lib/mentat/module/inspector.py
+++ b/lib/mentat/module/inspector.py
@@ -72,35 +72,6 @@ class MentatInspectorDaemon(mentat.daemon.piper.PiperDaemon):
description = 'mentat-inspector.py - IDEA message inspection daemon',
- #
- # Configure required deamon paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/var/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core',
- default_queue_in_dir = '/var/mentat/spool/mentat-inspector.py',
- default_queue_out_dir = None,
- default_stats_interval = mentat.const.DFLT_INTERVAL_STATISTICS,
- default_runlog_interval = mentat.const.DFLT_INTERVAL_RUNLOG,
-
- #
- # Schedule initial events.
- #
- schedule = [
- (mentat.const.DFLT_EVENT_START,)
- ],
- schedule_after = [
- (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS),
- (mentat.const.DFLT_INTERVAL_RUNLOG, mentat.const.DFLT_EVENT_SAVE_RUNLOG)
- ],
-
#
# Define required daemon components.
#
diff --git a/lib/mentat/module/netmngr.py b/lib/mentat/module/netmngr.py
index 8844a377c3bf66acbdb2c42180f783b19dd29f59..f4eda667837aa74ff2e6e60286cd6c5e84f846b2 100644
--- a/lib/mentat/module/netmngr.py
+++ b/lib/mentat/module/netmngr.py
@@ -236,22 +236,7 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
self.sqlservice = None
super().__init__(
-
description = 'mentat-netmngr.py - Abuse group network management script for Mentat database',
-
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core'
)
def _init_argparser(self, **kwargs):
@@ -275,7 +260,12 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
#
arggroup_script = argparser.add_argument_group('custom script arguments')
- arggroup_script.add_argument('--whois-file', type = str, default = None, help = 'path to reference whois file containing network data')
+ arggroup_script.add_argument(
+ '--whois-file',
+ type = str,
+ default = None,
+ help = 'path to reference whois file containing network data'
+ )
return argparser
diff --git a/lib/mentat/module/precache.py b/lib/mentat/module/precache.py
index ba36d10988ff6724b96c6fe062debac8c932021d..f055414921046220a29be70f16a85a99787db834 100644
--- a/lib/mentat/module/precache.py
+++ b/lib/mentat/module/precache.py
@@ -101,8 +101,8 @@ import datetime
# Custom libraries
#
import pyzenkit.jsonconf
-import mentat.script.fetcher
import mentat.const
+import mentat.script.fetcher
import mentat.system
#
@@ -138,22 +138,7 @@ class MentatPrecacheScript(mentat.script.fetcher.FetcherScript):
self.sqlservice = None
super().__init__(
-
description = 'mentat-precache.py - Mentat database pre-caching script',
-
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core'
)
def _init_argparser(self, **kwargs):
@@ -177,8 +162,18 @@ class MentatPrecacheScript(mentat.script.fetcher.FetcherScript):
#
arggroup_script = argparser.add_argument_group('custom script arguments')
- arggroup_script.add_argument('--allow-empty', help = 'allow storing empty item sets as valid result (flag)', action='store_true', default = None)
- arggroup_script.add_argument('--cache-dir', type = str, default = None, help = 'name of the cache directory')
+ arggroup_script.add_argument(
+ '--allow-empty',
+ action = 'store_true',
+ default = None,
+ help = 'allow storing empty item sets as valid result (flag)'
+ )
+ arggroup_script.add_argument(
+ '--cache-dir',
+ type = str,
+ default = None,
+ help = 'name of the cache directory'
+ )
return argparser
@@ -199,7 +194,7 @@ class MentatPrecacheScript(mentat.script.fetcher.FetcherScript):
cfgs = (
(self.CONFIG_ITEMSETS, None),
(self.CONFIG_ALLOW_EMPTY, False),
- (self.CONFIG_CACHE_DIR, '/var/mentat/cache')
+ (self.CONFIG_CACHE_DIR, os.path.join(self.paths[self.PATH_VAR], 'cache')),
) + cfgs
return super()._init_config(cfgs, **kwargs)
diff --git a/lib/mentat/module/reporter.py b/lib/mentat/module/reporter.py
index 209036fe2d460d6d8be496b5e7d0b6313d496c87..cb42c110d17159dac0ada5b6796060519cefad7a 100644
--- a/lib/mentat/module/reporter.py
+++ b/lib/mentat/module/reporter.py
@@ -183,20 +183,6 @@ class MentatReporterScript(mentat.script.fetcher.FetcherScript):
description = 'mentat-reporter.py - Mentat system event reporting',
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core',
-
#
# Load additional application-level plugins.
#
@@ -241,14 +227,48 @@ class MentatReporterScript(mentat.script.fetcher.FetcherScript):
#
arggroup_script = argparser.add_argument_group('custom script arguments')
- arggroup_script.add_argument('--force-mode', type = str, default = None, help = 'force a reporting mode setting')
- arggroup_script.add_argument('--force-attachments', type = str, default = None, help = 'force a report attachment setting')
- arggroup_script.add_argument('--force-template', type = str, default = None, help = 'force a template for generating reports')
- arggroup_script.add_argument('--force-locale', type = str, default = None, help = 'force a locale for generating reports')
- arggroup_script.add_argument('--force-timezone', type = str, default = None, help = 'force a timezone for generating reports')
- arggroup_script.add_argument('--force-max-attachment-size', type = int, default = None, help = 'force maximal size of email attachments in bytes')
-
- arggroup_script.add_argument('--test-data', help = 'use test data for reporting (flag)', action = 'store_true', default = None)
+ arggroup_script.add_argument(
+ '--force-mode',
+ type = str,
+ default = None,
+ help = 'force a reporting mode setting'
+ )
+ arggroup_script.add_argument(
+ '--force-attachments',
+ type = str,
+ default = None,
+ help = 'force a report attachment setting'
+ )
+ arggroup_script.add_argument(
+ '--force-template',
+ type = str,
+ default = None,
+ help = 'force a template for generating reports'
+ )
+ arggroup_script.add_argument(
+ '--force-locale',
+ type = str,
+ default = None,
+ help = 'force a locale for generating reports'
+ )
+ arggroup_script.add_argument(
+ '--force-timezone',
+ type = str,
+ default = None,
+ help = 'force a timezone for generating reports'
+ )
+ arggroup_script.add_argument(
+ '--force-max-attachment-size',
+ type = int,
+ default = None,
+ help = 'force maximal size of email attachments in bytes'
+ )
+ arggroup_script.add_argument(
+ '--test-data',
+ action = 'store_true',
+ default = None,
+ help = 'use test data for reporting (flag)'
+ )
return argparser
diff --git a/lib/mentat/module/sampler.py b/lib/mentat/module/sampler.py
index 6634ed040f43689762d3771c92ec03529d1039dd..b97e8ab62bc277e4b3cc15b6aa1ad7c83b800ebf 100644
--- a/lib/mentat/module/sampler.py
+++ b/lib/mentat/module/sampler.py
@@ -73,35 +73,6 @@ class MentatSamplerDaemon(mentat.daemon.piper.PiperDaemon):
description = 'mentat-sampler.py - IDEA message sampling daemon',
- #
- # Configure required deamon paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core',
- default_queue_in_dir = '/var/mentat/spool/mentat-sampler.py',
- default_queue_out_dir = None,
- default_stats_interval = mentat.const.DFLT_INTERVAL_STATISTICS,
- default_runlog_interval = mentat.const.DFLT_INTERVAL_RUNLOG,
-
- #
- # Schedule initial events.
- #
- schedule = [
- (mentat.const.DFLT_EVENT_START,)
- ],
- schedule_after = [
- (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS),
- (mentat.const.DFLT_INTERVAL_RUNLOG, mentat.const.DFLT_EVENT_SAVE_RUNLOG)
- ],
-
#
# Define required daemon components.
#
@@ -133,8 +104,18 @@ class MentatSamplerDaemon(mentat.daemon.piper.PiperDaemon):
#
arggroup_daemon = argparser.add_argument_group('custom daemon arguments')
- arggroup_daemon.add_argument('--sampling-limit', help='sampling limit', type = int, default = None)
- arggroup_daemon.add_argument('--sampling-policy', help='sampling policy', type = str, default = None)
+ arggroup_daemon.add_argument(
+ '--sampling-limit',
+ type = int,
+ default = None,
+ help = 'sampling limit'
+ )
+ arggroup_daemon.add_argument(
+ '--sampling-policy',
+ type = str,
+ default = None,
+ help = 'sampling policy'
+ )
return argparser
diff --git a/lib/mentat/module/statistician.py b/lib/mentat/module/statistician.py
index 07c580ae2d293370ada8f6ff87883f5047d12a36..f33d4ee10ef45804778425edbba02675eac9c7a2 100644
--- a/lib/mentat/module/statistician.py
+++ b/lib/mentat/module/statistician.py
@@ -89,25 +89,11 @@ class MentatStatisticianScript(mentat.script.fetcher.FetcherScript):
configuration values for parent contructor.
"""
# Declare private attributes.
+ self.stats_rrd = None
self.sqlservice = None
super().__init__(
-
description = 'mentat-statistician.py - Mentat system statistical script',
-
- #
- # Configure required script paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core'
)
def _sub_stage_init(self, **kwargs):
diff --git a/lib/mentat/module/storage.py b/lib/mentat/module/storage.py
index 822af08c1c31a8c9f4ff7ab8201cd04cfd64b768..c412a9e22f2b4ccb25d75442bb894405d1ff4ec9 100644
--- a/lib/mentat/module/storage.py
+++ b/lib/mentat/module/storage.py
@@ -71,35 +71,6 @@ class MentatStorageDaemon(mentat.daemon.piper.PiperDaemon):
description = 'mentat-storage.py - IDEA message storing daemon',
- #
- # Configure required deamon paths.
- #
- path_bin = '/usr/local/bin',
- path_cfg = '/etc/mentat',
- path_log = '/var/mentat/log',
- path_run = '/var/mentat/run',
- path_tmp = '/var/tmp',
-
- #
- # Override default configurations.
- #
- default_config_dir = '/etc/mentat/core',
- default_queue_in_dir = '/var/mentat/spool/mentat-storage.py',
- default_queue_out_dir = None,
- default_stats_interval = mentat.const.DFLT_INTERVAL_STATISTICS,
- default_runlog_interval = mentat.const.DFLT_INTERVAL_RUNLOG,
-
- #
- # Schedule initial events.
- #
- schedule = [
- (mentat.const.DFLT_EVENT_START,)
- ],
- schedule_after = [
- (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS),
- (mentat.const.DFLT_INTERVAL_RUNLOG, mentat.const.DFLT_EVENT_SAVE_RUNLOG)
- ],
-
#
# Define required daemon components.
#
diff --git a/lib/mentat/script/base.py b/lib/mentat/script/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..3b3ec01b9c64f7eb0d56006d2bbaf20e0cd3cf35
--- /dev/null
+++ b/lib/mentat/script/base.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This module provides base implementation of all Mentat scripts.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+#
+# Custom libraries.
+#
+import pyzenkit.baseapp
+import pyzenkit.zenscript
+import mentat.const
+
+
+class MentatBaseScript(pyzenkit.zenscript.ZenScript): # pylint: disable=locally-disabled,abstract-method
+ """
+ Base implementation of all Mentat scripts.
+ """
+
+ def __init__(self, **kwargs):
+ #
+ # Configure required script paths.
+ #
+ kwargs.setdefault('path_cfg', mentat.const.PATH_CFG)
+ kwargs.setdefault('path_var', mentat.const.PATH_VAR)
+ kwargs.setdefault('path_log', mentat.const.PATH_LOG)
+ kwargs.setdefault('path_run', mentat.const.PATH_RUN)
+ kwargs.setdefault('path_tmp', mentat.const.PATH_TMP)
+
+ #
+ # Override default configurations.
+ #
+ kwargs.setdefault(
+ 'default_config_dir',
+ self.get_resource_path(mentat.const.PATH_CFG_CORE)
+ )
+
+ super().__init__(**kwargs)
diff --git a/lib/mentat/script/fetcher.py b/lib/mentat/script/fetcher.py
index ebb644b2db226b6aa89d9d1660c671566be0c9c6..5cdc50289d566923f61ca07f69a86cf8af384eb1 100644
--- a/lib/mentat/script/fetcher.py
+++ b/lib/mentat/script/fetcher.py
@@ -28,11 +28,12 @@ import time
import pyzenkit.baseapp
import pyzenkit.zenscript
import mentat.const
+import mentat.script.base
import mentat.plugin.app.eventstorage
import mentat.plugin.app.sqlstorage
-class FetcherScript(pyzenkit.zenscript.ZenScript): # pylint: disable=locally-disabled,abstract-method
+class FetcherScript(mentat.script.base.MentatBaseScript): # pylint: disable=locally-disabled,abstract-method
"""
Base implementation of generic message fetching and processing script.
"""
@@ -150,11 +151,12 @@ class DemoFetcherScript(FetcherScript):
#
# Configure required script paths.
#
- path_bin = '/tmp',
- path_cfg = '/tmp',
- path_log = '/tmp',
- path_run = '/tmp',
- path_tmp = '/tmp',
+ path_bin = 'tmp',
+ path_cfg = 'tmp',
+ path_var = 'tmp',
+ path_log = 'tmp',
+ path_run = 'tmp',
+ path_tmp = 'tmp',
#
# Override default configurations.
diff --git a/lib/mentat/system.py b/lib/mentat/system.py
index a34a8a6ae308ad4795db57600537ea7bbcb97697..d0fd42f797676af9c447d5e9a0be5e3622be8232 100644
--- a/lib/mentat/system.py
+++ b/lib/mentat/system.py
@@ -729,7 +729,7 @@ def module_status(mod_data, pidf_data, proc_data, overall_status):
STATUS_RT_RUNNING_FEWER,
"There are fewer instances running than required ({}:{})".format(mod_data.count, len(proc_data.keys()))
)
- elif mod_data.count < len(proc_data.keys()):
+ if mod_data.count < len(proc_data.keys()):
return (
STATUS_RT_RUNNING_MORE,
"There are more instances running than required ({}:{})".format(mod_data.count, len(proc_data.keys()))
diff --git a/setup.py b/setup.py
index 4e2cf2c4559639926da24efc5fdf502576fac36f..2a790b998a2750606e248fa47f30813333a54517 100644
--- a/setup.py
+++ b/setup.py
@@ -104,7 +104,6 @@ setup(
'bin/mentat-controller.py',
'bin/mentat-dbmngr.py',
'bin/mentat-enricher.py',
- 'bin/mentat-hawat.py',
'bin/mentat-hawat.wsgi',
'bin/mentat-ideagen.py',
'bin/mentat-informant.py',