diff --git a/.pylintrc-lib b/.pylintrc-lib
index c4f3f2b9362eb862f2871985e2639022fefd420c..e626fe5de5a9eb1cfa61a7572ca6de2860cde884 100644
--- a/.pylintrc-lib
+++ b/.pylintrc-lib
@@ -136,7 +136,7 @@ function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
# Good variable names which should always be accepted, separated by a comma
-good-names=i,j,k,ex,Run,_,maxDiff
+good-names=i,j,k,ex,Run,_,maxDiff,rc
# Include a hint for the correct naming format with invalid-name
include-naming-hint=no
diff --git a/lib/mentat/module/controller.py b/lib/mentat/module/controller.py
index 087825398494da3cf5dd7690853a5c45aae642e8..0af487b2a3196400b473d05482b60ab588b25a56 100644
--- a/lib/mentat/module/controller.py
+++ b/lib/mentat/module/controller.py
@@ -3,6 +3,7 @@
#-------------------------------------------------------------------------------
# This file is part of Mentat system (https://mentat.cesnet.cz/).
#
+
# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
@@ -15,6 +16,12 @@ This script is implemented using the :py:mod:`pyzenkit.zenscript` framework and
so it provides all of its core features. See the documentation for more in-depth
details.
+The Mentat system is a collection of many real-time processing and post-processing
+modules. Launching and managing so many processes would be really tedious work. And
+that is exactly the use-case for this module. Its purpose is to start/stop/restart
+all preconfigured real-time message processing modules and enable/disable all
+preconfigured message post-processing modules (cronjobs).
+
Usage examples
--------------
@@ -30,6 +37,27 @@ Usage examples
# Run with increased logging level.
mentat-controller.py --log-level debug
+ # Determine the current status of Mentat system and all of its modules.
+ mentat-controller.py
+ mentat-controller.py --command status
+
+ # Start/stop/restart all configured real-time message processing modules.
+ mentat-controller.py --command start
+ mentat-controller.py --command stop
+ mentat-controller.py --command restart
+
+ # Enable/Disable all configured message post-processing modules (cronjobs).
+ mentat-controller.py --command enable
+ mentat-controller.py --command disable
+
+ # Send signal to all configured real-time message processing modules.
+ mentat-controller.py --command signal-usr1
+
+ # Work with particular modules.
+ mentat-controller.py --command start --target mentat-storage.py
+ mentat-controller.py --command stop --target mentat-enricher.py mentat-inspector.py
+ mentat-controller.py --command signal-usr1 --target mentat-inspector-b.py
+
Available script commands
-------------------------
@@ -104,6 +132,20 @@ Custom command line options
*Type:* ``string``
+
+Custom configuration file options
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``modules``
+ List of real-time message processing modules that should be managed.
+
+ *Type:* ``list of dicts``
+
+``cronjobs``
+ List of message post-processing modules that should be managed.
+
+ *Type:* ``list of dicts``
+
"""
@@ -132,6 +174,7 @@ CRON_SCRIPT_DIR = '/etc/cron.d'
class MentatControllerScript(pyzenkit.zenscript.ZenScript):
+
"""
Implementation of Mentat module (script) providing Mentat system control
functions and features.
@@ -180,7 +223,9 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
and it must return valid :py:class:`argparse.ArgumentParser` object. It
appends additional command line options custom for this script object.
+
This method is called from the main constructor in :py:func:`pyzenkit.baseapp.BaseApp.__init__`
+
as a part of the **__init__** stage of application`s life cycle.
:param kwargs: Various additional parameters passed down from object constructor.
@@ -213,7 +258,9 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
:return: Default configuration structure.
:rtype: dict
"""
+
cfgs = (
+
(self.CONFIG_TARGET, None),
) + cfgs
return super()._init_config(cfgs, **kwargs)
@@ -225,8 +272,8 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
This method is called from the main setup method :py:func:`pyzenkit.baseapp.BaseApp._stage_setup`
as a part of the **setup** stage of application`s life cycle.
"""
- self.modules = mentat.system.make_module_list(self.c(self.CONFIG_MODULES))
- self.cronjobs = mentat.system.make_cronjob_list(self.c(self.CONFIG_CRONJOBS))
+ self.modules = mentat.system.make_module_list(self.c(self.CONFIG_MODULES)) # pylint: disable=locally-disabled,attribute-defined-outside-init
+ self.cronjobs = mentat.system.make_cronjob_list(self.c(self.CONFIG_CRONJOBS)) # pylint: disable=locally-disabled,attribute-defined-outside-init
#---------------------------------------------------------------------------
@@ -238,6 +285,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
in case it is not explicitly selected either by command line option, or
by configuration file directive.
+
:return: Name of the default command.
:rtype: str
"""
@@ -258,8 +306,6 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.paths.get(self.PATH_LOG),
self.paths.get(self.PATH_RUN)
)
- # TODO: This proved to be really expensive, another solution is necessary.
- #self.dbgout("System status: >>>\n{}".format(pprint.pformat(status)))
return status
def cbk_command_status(self):
@@ -287,13 +333,15 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Cronjob module '%s': '%s'", mname, stat[1])
self.logger.info("Overall cronjob module status: '%s'", status['resultc'][1])
+
+
for msgcat in ['info', 'notice', 'warning', 'error']:
if status['messages'][msgcat]:
for prob in status['messages'][msgcat]:
getattr(self.logger, msgcat)(prob[0])
self.logger.info("Overall Mentat system status: '%s'", status['result'][1])
- self.rc = status['resultc'][0]
+ self.rc = status['resultc'][0] # pylint: disable=locally-disabled,attribute-defined-outside-init
return self.RESULT_SUCCESS
def cbk_command_start(self):
@@ -311,14 +359,13 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
modulelist = self._get_target()
for mname in modulelist:
stat = status['modules'][mname]
- pidf = status['pid_files'].get(mname, None)
proc = status['processes'].get(mname, None)
if stat[0] == mentat.system.STATUS_RT_RUNNING_OK:
self.logger.info("Module '%s': Module is already running properly, nothing to do", mname)
elif stat[0] == mentat.system.STATUS_RT_NOT_RUNNING:
self.logger.info("Module '%s': Launching module", mname)
- self._module_start(self.modules[mname], pidf, proc)
+ self._module_start(self.modules[mname], proc)
elif stat[0] == mentat.system.STATUS_RT_RUNNING_FEWER:
self.logger.info("Module '%s': Module is running in fewer than required instances", mname)
elif stat[0] == mentat.system.STATUS_RT_RUNNING_MORE:
@@ -329,7 +376,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Waiting for modules to fully start up")
time.sleep(1)
status = self.get_system_status()
- self.rc = status['resultm'][0]
+ self.rc = status['resultm'][0] # pylint: disable=locally-disabled,attribute-defined-outside-init
if self.rc == mentat.system.STATUS_RT_RUNNING_OK:
self.logger.info("System startup seems successfull")
@@ -364,7 +411,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.error("Cron module '%s': Module is in weird state, unable to perform automatic startup", mname)
status = self.get_system_status()
- self.rc = status['resultc'][0]
+ self.rc = status['resultc'][0] # pylint: disable=locally-disabled,attribute-defined-outside-init
if self.rc == mentat.system.STATUS_CJ_ENABLED:
self.logger.info("System startup seems successfull")
@@ -396,7 +443,6 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
# Process all modules from the list.
for mname in modulelist:
stat = status['modules'][mname]
- pidf = status['pid_files'].get(mname, None)
proc = status['processes'].get(mname, None)
if stat[0] == mentat.system.STATUS_RT_NOT_RUNNING:
@@ -416,7 +462,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
mentat.system.STATUS_RT_RUNNING_MORE):
# Perform attemt to stop the module and mark it for another check in next iteration.
self.logger.info("Module '%s': Stopping module, attempt #%s", mname, i)
- self._module_signal(self.modules[mname], pidf, proc, signal.SIGINT)
+ self._module_signal(self.modules[mname], proc, signal.SIGINT)
nextmodulelist.append(mname)
continue
@@ -432,7 +478,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
# Review the overall Mentat system status.
status = self.get_system_status()
- self.rc = status['resultm'][0]
+ self.rc = status['resultm'][0] # pylint: disable=locally-disabled,attribute-defined-outside-init
if self.rc == mentat.system.STATUS_RT_NOT_RUNNING:
self.logger.info("System shutdown seems successfull")
@@ -467,7 +513,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.error("Cron module '%s': Module is in weird state, unable to perform automatic shutdown", mname)
status = self.get_system_status()
- self.rc = status['result'][0]
+ self.rc = status['result'][0] # pylint: disable=locally-disabled,attribute-defined-outside-init
if self.rc == mentat.system.STATUS_CJ_ENABLED:
self.logger.info("System shutdown seems successfull")
@@ -505,7 +551,6 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
modulelist = self._get_target()
for mname in modulelist:
stat = status['modules'][mname]
- pidf = status['pid_files'].get(mname, None)
proc = status['processes'].get(mname, None)
if stat[0] == mentat.system.STATUS_RT_NOT_RUNNING:
@@ -517,7 +562,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
mentat.system.STATUS_RT_RUNNING_FEWER,
mentat.system.STATUS_RT_RUNNING_MORE):
self.logger.info("Module '%s': Killing module", mname)
- self._module_signal(self.modules[mname], pidf, proc, signal.SIGKILL)
+ self._module_signal(self.modules[mname], proc, signal.SIGKILL)
else:
self.logger.error("Module '%s': Module is in weird state, unable to kill it", mname)
@@ -538,11 +583,10 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
modulelist = self._get_target()
for mname in modulelist:
stat = status['modules'][mname]
- pidf = status['pid_files'].get(mname, None)
proc = status['processes'].get(mname, None)
if stat[0] == mentat.system.STATUS_RT_RUNNING_OK:
- self._module_signal(self.modules[mname], pidf, proc, signal.SIGHUP)
+ self._module_signal(self.modules[mname], proc, signal.SIGHUP)
else:
self.logger.error("Module '%s': Unable to send signal", mname)
@@ -563,11 +607,10 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
modulelist = self._get_target()
for mname in modulelist:
stat = status['modules'][mname]
- pidf = status['pid_files'].get(mname, None)
proc = status['processes'].get(mname, None)
if stat[0] == mentat.system.STATUS_RT_RUNNING_OK:
- self._module_signal(self.modules[mname], pidf, proc, signal.SIGUSR1)
+ self._module_signal(self.modules[mname], proc, signal.SIGUSR1)
else:
self.logger.error("Module '%s': Unable to send signal", mname)
@@ -588,11 +631,10 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
modulelist = self._get_target()
for mname in modulelist:
stat = status['modules'][mname]
- pidf = status['pid_files'].get(mname, None)
proc = status['processes'].get(mname, None)
if stat[0] == mentat.system.STATUS_RT_RUNNING_OK:
- self._module_signal(self.modules[mname], pidf, proc, signal.SIGUSR2)
+ self._module_signal(self.modules[mname], proc, signal.SIGUSR2)
else:
self.logger.error("Module '%s': Unable to send signal", mname)
@@ -676,12 +718,12 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
os.execv(executable, arguments)
self.logger.critical("Module '%s': Worker process '%d' was unable to execute module", mod_data.name, os.getpid())
- os._exit(0)
+ os._exit(0) # pylint: disable=locally-disabled,protected-access
else:
self.logger.info("Module '%s': Waiting for worker process '%d'", mod_data.name, newpid)
time.sleep(1)
- def _module_start(self, mod_data, pid_files, processes):
+ def _module_start(self, mod_data, processes):
"""
Start required number of instances of given module.
"""
@@ -692,23 +734,31 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
instances = instances - len(processes.keys())
for i in range(0, instances):
+ self.logger.info("Module '%s': Launching instance '%d'", mod_data.name, i)
self._execute(mod_data)
- @staticmethod
- def _module_enable(mod_data, meta):
+ def _module_enable(self, mod_data, meta):
"""
Enable given cron module.
"""
- os.symlink(
+ link_path = os.path.join(CRON_SCRIPT_DIR, meta['name'])
+ self.logger.debug(
+ "Module '%s': Creating symlink from '%s' to '%s'",
+ mod_data.name,
meta['path'],
- os.path.join(CRON_SCRIPT_DIR, meta['name'])
+ link_path
)
+ os.symlink(meta['path'], link_path)
- @staticmethod
- def _module_disable(mod_data, meta):
+ def _module_disable(self, mod_data, meta):
"""
Disable given cron module.
"""
+ self.logger.debug(
+ "Module '%s': Removing symlink '%s'",
+ mod_data.name,
+ meta['link'],
+ )
os.remove(meta['link'])
@@ -722,7 +772,7 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Module '%s': Sending signal '%s' to process '%d'", mod_data.name, SIGNALS_TO_NAMES_DICT.get(sig, sig), pid)
os.kill(pid, sig)
- def _module_signal(self, mod_data, pid_files, processes, sig):
+ def _module_signal(self, mod_data, processes, sig):
"""
Stop given module.
"""
@@ -730,7 +780,6 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self._signal(mod_data, pid, sig)
#
-# TODO: fix (remove dangling PID files)
-# stats (display statistics for modules: last write to log, ...)
+# TODO: stats (display statistics for modules: last write to log, ...)
# status (return rc according to the overall status)
#