diff --git a/LICENSE.txt b/LICENSE.txt
index 224316ffe29b1313a73b1f8676df6a3104b5395a..32725ef3f1647490b59e466d4c69644e4cec288a 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,6 +1,11 @@
The MIT License (MIT)
-Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
+Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+Use of this package is governed by the MIT license, see LICENSE file.
+
+This project was initially written for personal use of the original author. Later
+it was developed much further and used for project of author`s employer.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/Makefile b/Makefile
index 750041b7dd70cad820f9b267f7c729601575f0f9..c3a61157a92f8eacd600aa6e82b4f4965cfe13f4 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
-SPHINXPROJ = ZenKit-PythonScriptAndDaemonToolkit
+SPHINXPROJ = PyZenKit-Python3ScriptAndDaemonToolkit
SOURCEDIR = .
BUILDDIR = doc/_build
diff --git a/README.rst b/README.rst
index 4200298441be5dc8c461d73827187aad6daff961..ecb1e040cd34b61f4dc228b8eb20e6dfd0167ea0 100644
--- a/README.rst
+++ b/README.rst
@@ -1,19 +1,59 @@
-PyZenKit
+PyZenKit - Python script and daemon toolkit's documentation!
================================================================================
-Collection of usefull tools and utilities for Python 3.
.. warning::
- This library is still work in progress.
+ Although production code is based on this library, it should still be considered
+ as work in progress.
-.. note::
- For usage and examples please see the source code, for demonstration execute
- the appropriate module with Python3 interpreter.
+Introduction
+--------------------------------------------------------------------------------
+
+This package contains collection of usefull tools and utilities for creating
+console applications, scripts and system services (daemons) in Python 3. It
+provides easily extendable and customizable base implementations of generic
+application, script or daemon and which take care of many common issues and
+tasks like configuration loading and merging, command line argument parsing,
+logging setup, etc.
+
+The extensive documentation and tutorials is still under development, however
+usage examples and demonstration applications are provided right in the source
+code of appropriate module. Just execute the module with Python3 interpretter
+to see the demonstration::
+
+ python3 path/to/application.py --help
+
+
+Features
+--------------------------------------------------------------------------------
+
+Currently the package contains following features:
+
+:py:mod:`pyzenkit.jsonconf`
+ Module for handling JSON based configuration files and directories.
+
+:py:mod:`pyzenkit.daemonizer`
+ Module for taking care of all process daemonization tasks.
+
+:py:mod:`pyzenkit.baseapp`
+ Module for writing generic console applications.
+
+:py:mod:`pyzenkit.zenscript`
+ Module for writing generic console scripts with built-in support for repeated
+ executions (for example by cron-like service).
+
+:py:mod:`pyzenkit.zendaemon`
+ Module for writing generic system services (daemons).
+
Copyright
--------------------------------------------------------------------------------
-Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
+Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
Use of this package is governed by the MIT license, see LICENSE file.
+
+This project was initially written for personal use of the original author. Later
+it was developed much further and used for project of author`s employer.
diff --git a/conf.py b/conf.py
index 0436043284a06aa79a009373edd79d6b4785162c..b2a3803489c2510b110d2e4362bf95ada4c3ff31 100644
--- a/conf.py
+++ b/conf.py
@@ -67,9 +67,9 @@ author = u'Jan Mach'
# built documents.
#
# The short X.Y version.
-version = u'1.0'
+version = u'0.32'
# The full version, including alpha/beta/rc tags.
-release = u'1.0'
+release = u'0.32'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
@@ -168,4 +168,4 @@ texinfo_documents = [
# Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {'python': ('https://docs.python.org/3.4', None)}
diff --git a/doc/_pages/_inc.bin.app-opt.rst b/doc/_pages/_inc.bin.app-opt.rst
new file mode 100644
index 0000000000000000000000000000000000000000..da4228a13bc9a5c481383c8f6f77ea3c4849ba5a
--- /dev/null
+++ b/doc/_pages/_inc.bin.app-opt.rst
@@ -0,0 +1,75 @@
+``--help``
+ Display help and usage description and exit (*flag*)
+
+``--name alternative-name``
+ Alternative name for application instead of default ``$0``, using which names
+ for log, runlog, pid, status and other files will be generated.
+
+ *Type:* ``string``, *default:* ``$0``
+
+``--quiet``
+ Run in quiet mode (*flag*).
+
+ Do not write anything to ``stdout`` or ``stderr``.
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--verbose``
+ Increase application output verbosity (*flag*, *repeatable*).
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--log-file file-name``
+ Name of the log file.
+
+ *Type:* ``string``, *default:* autodetected
+
+``--log-level level``
+ Logging level [``debug``, ``info``, ``warning``, ``error``, ``critical``].
+
+ *Type:* ``string``, *default:* ``info``
+
+``--runlog-dir dir-name``
+ Name of the runlog directory.
+
+ *Type:* ``string``, *default:* autodetected
+
+``--runlog-dump``
+ Dump runlog to stdout when done processing (*flag*).
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--runlog_log``
+ Write runlog to logging service when done processing (*flag*)
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--pstate_file file-name``
+ Name of the persistent state file.
+
+ *Type:* ``string``, *default:* autodetected
+
+``--pstate_dump``
+ Dump persistent state to stdout when done processing (*flag*).
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--pstate_log``
+ Write persistent state to logging service when done processing (*flag*).
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--action action``
+ Execute given quick action and exit. List of available actions can be displayed with ``--help`` option.
+
+ *Type:* ``string``, *default:* ``None``
+
+``--user name-or-id``
+ Name/gid of the system user for process permissions.
+
+ *Type:* ``string``, *default:* ``None``
+
+``--group name-or-id``
+ Name/gid of the system group for process permissions.
+
+ *Type:* ``string``, *default:* ``None``
diff --git a/doc/_pages/_inc.bin.daemon-opt.rst b/doc/_pages/_inc.bin.daemon-opt.rst
new file mode 100644
index 0000000000000000000000000000000000000000..cad8aa9230cd3aaf8ab74c6c9c0e51c517e034c0
--- /dev/null
+++ b/doc/_pages/_inc.bin.daemon-opt.rst
@@ -0,0 +1,39 @@
+``--no-daemon``
+ Do not daemonize and stay in foreground (*flag*).
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--chroot-dir dir-name``
+ Name of the chroot directory.
+
+ *Type:* ``string``, *default:* ``None``
+
+``--work_dir dir-name``
+ Name of the process work directory.
+
+ *Type:* ``string``, *default:* ``/``
+
+``--pid_file file-name``
+ Name of the pid file.
+
+ *Type:* ``string``, *default:* autodetected
+
+``--state_file file-name``
+ Name of the state file.
+
+ *Type:* ``string``, *default:* autodetected
+
+``--umask mask``
+ Default file umask.
+
+ *Type:* ``string``, *default:* ``0o002``
+
+``--stats_interval interval``
+ Processing statistics display interval in seconds.
+
+ *Type:* ``integer``, *default:* ``300``
+
+``--paralel``
+ Run in paralel mode (*flag*).
+
+ *Type:* ``boolean``, *default:* ``False``
diff --git a/doc/_pages/_inc.bin.script-opt.rst b/doc/_pages/_inc.bin.script-opt.rst
new file mode 100644
index 0000000000000000000000000000000000000000..257384f3d2ac8795b3093ab06e4329d4a7f48df0
--- /dev/null
+++ b/doc/_pages/_inc.bin.script-opt.rst
@@ -0,0 +1,29 @@
+``--regular``
+ Operational mode: regular script execution (*flag*). Conflicts with ``--shell`` option.
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--shell``
+ Operational mode: manual script execution from shell (*flag*). Conflicts with ``--regular`` option.
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--command name``
+ Name of the script command to be executed.
+
+ *Type:* ``string``, *default:* autodetected
+
+``--interval interval``
+ Execution interval. This value should correspond with related cron script.
+
+ *Type:* ``string``, *default:* ``daily``
+
+``--adjust_thresholds``
+ Round-up time interval threshols to interval size (*flag*).
+
+ *Type:* ``boolean``, *default:* ``False``
+
+``--time_high time``
+ Upper time interval threshold.
+
+ *Type:* ``float``, *default:* time.time
diff --git a/doc/_pages/api.rst b/doc/_pages/api.rst
index 41a2b81335ba16dff9f6dca1edb5387693ac0a44..047e32c79d5ecaedaef12f0f21b832c50c1bea2b 100644
--- a/doc/_pages/api.rst
+++ b/doc/_pages/api.rst
@@ -5,11 +5,17 @@ API
.. warning::
- Although production code is based on this library, it should still be considered work in progress.
+ Although a working production code is based on this library, it should still
+ be considered to be work in progress.
.. toctree::
:maxdepth: 1
:caption: API Contents:
:glob:
- api_*
+ api_pyzenkit.jsonconf
+ api_pyzenkit.daemonizer
+ api_pyzenkit.baseapp
+ api_pyzenkit.zenscript
+ api_pyzenkit.zendaemon
+ api_pyzenkit.zencli
diff --git a/doc/_pages/api_pyzenkit.baseapp.rst b/doc/_pages/api_pyzenkit.baseapp.rst
index b4c441fa8439337c017f8b30d60876eabd7d07f5..a1921a0c7a40d289abdce2af2ab8ad1dfbad25ea 100644
--- a/doc/_pages/api_pyzenkit.baseapp.rst
+++ b/doc/_pages/api_pyzenkit.baseapp.rst
@@ -5,3 +5,5 @@ pyzenkit.baseapp
.. automodule:: pyzenkit.baseapp
:members:
+ :private-members:
+ :special-members:
diff --git a/doc/_pages/api_pyzenkit.zendaemon.rst b/doc/_pages/api_pyzenkit.zendaemon.rst
index 2cbb885e11caeb0a0bbdc6e036d7549ebe9272af..5b6a8a4dac208d54e680f0e340446fec62c303c6 100644
--- a/doc/_pages/api_pyzenkit.zendaemon.rst
+++ b/doc/_pages/api_pyzenkit.zendaemon.rst
@@ -5,3 +5,5 @@ pyzenkit.zendaemon
.. automodule:: pyzenkit.zendaemon
:members:
+ :private-members:
+ :special-members:
diff --git a/doc/_pages/api_pyzenkit.zenscript.rst b/doc/_pages/api_pyzenkit.zenscript.rst
index 0f378dce0c9c1f7935e59ee853cfeab702f344c1..5fbc68d07e31dfa562a5abc4bf2ba82761ae39ff 100644
--- a/doc/_pages/api_pyzenkit.zenscript.rst
+++ b/doc/_pages/api_pyzenkit.zenscript.rst
@@ -5,3 +5,5 @@ pyzenkit.zenscript
.. automodule:: pyzenkit.zenscript
:members:
+ :private-members:
+ :special-members:
diff --git a/doc/_pages/architecture.rst b/doc/_pages/architecture.rst
new file mode 100644
index 0000000000000000000000000000000000000000..d8aea34dead58a02b93574b0a6f67c8a45c7cdb8
--- /dev/null
+++ b/doc/_pages/architecture.rst
@@ -0,0 +1,63 @@
+Application architecture
+================================================================================
+
+
+Configuration
+--------------------------------------------------------------------------------
+
+Every application supports multiple means for adjusting the internal configurations.
+When appropriate the default values for each configuration is hardcoded in module
+source code. However there are several options to change the value:
+
+* Override the internal default value when instantinating the application object
+ by passing different value to object constructor.
+* Pass the different value by configuration file.
+* Pass the different value by command line option.
+
+The configuration values are assigned from the sources mentioned above in that
+particular order, so the value given by command line option overwrites the value
+written in configuration file.
+
+
+Command line options
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Configuration can be passed down to application by command line options. These options
+have the highest priority and will overwrite any other configuration values. Depending
+on the base object of the application different set of options is available.
+
+Common application options
+````````````````````````````````````````````````````````````````````````````````
+
+Following configuration options are available for all applications based on
+:py:mod:`pyzenkit.baseapp`:
+
+.. include:: _inc.bin.app-opt.rst
+
+Common script options
+````````````````````````````````````````````````````````````````````````````````
+
+Following configuration options are available on top of common applicationsoptions
+for all applications based on :py:mod:`pyzenkit.zenscript`:
+
+.. include:: _inc.bin.script-opt.rst
+
+Common daemon options
+````````````````````````````````````````````````````````````````````````````````
+
+Following configuration options are available on top of common applicationsoptions
+for all applications based on :py:mod:`pyzenkit.zendaemon`:
+
+.. include:: _inc.bin.daemon-opt.rst
+
+
+Configuration files and directories
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Configuration can be passed down to application using a combination of configuration
+file or configuration directory. The configuration file
+
+The available configuration keys are very similar to command line options and the
+names differ only in the use of ``_`` character instead of ``-``. However there is
+a certain set of configuration keys that is available only through command line
+options and not through configuration file and vice versa.
diff --git a/manual.rst b/manual.rst
index 1c660bc2e523c5fa83dd1b251beee6f9070bcfc0..ab7892a6507b9e9018b25c71d81d7787acc58f22 100644
--- a/manual.rst
+++ b/manual.rst
@@ -1,19 +1,20 @@
.. PyZenKit - Python script and daemon toolkit documentation master file, created by
sphinx-quickstart on Wed Feb 15 10:49:01 2017.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
Welcome to PyZenKit - Python script and daemon toolkit's documentation!
================================================================================
.. warning::
- Although production code is based on this library, it should still be considered work in progress.
+ Although production code is based on this library, it should still be considered
+ as work in progress.
.. toctree::
:maxdepth: 2
:caption: Contents:
+ README
+ doc/_pages/architecture
doc/_pages/api
Indices and tables
diff --git a/pyzenkit/__init__.py b/pyzenkit/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..16bbe090b036cfd6e408221d7c530f923d45bf31 100644
--- a/pyzenkit/__init__.py
+++ b/pyzenkit/__init__.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
+#-------------------------------------------------------------------------------
+
+__version__ = "0.32"
diff --git a/pyzenkit/baseapp.py b/pyzenkit/baseapp.py
index b3d0d4241c2b8321cf8f29cfe308b52048a81f09..08b21a8d3e0260d028c2d0263fb6ee7851880d15 100644
--- a/pyzenkit/baseapp.py
+++ b/pyzenkit/baseapp.py
@@ -1,37 +1,245 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
+
"""
-Implementation of generic processing script/daemon/application.
+This module provides base implementation of generic processing application with
+many usefull features including (but not limited to) following:
+
+Application configuration service
+ The base application provides tools for loading configurations from multiple
+ sources and merging it into single dictionary. Currently the following
+ configuration sources are available:
+
+ * Optional configuration via configuration directory.
+ * Optional configuration via external JSON configuration file.
+ * Optional configuration via command line arguments and options.
+
+Logging service
+ The base application is capable of automated setup of logging service based
+ on configuration values into followin destinations:
+
+ * Optional logging to console.
+ * Optional logging to text file.
+
+Persistent state service
+ The base application contains optinal persistent state feature, which is capable
+ of storing data between multiple executions. Simple JSON files are used for the
+ data storage.
+
+Application runlog service
+ The base application provides optional runlog service, which is a intended to
+ provide storage for relevant data and results during the processing and enable
+ further analysis later. Simple JSON files are used for the data storage.
+
+Plugin system
+ The application provides tools for writing and using plugins, that can be used
+ to further enhance the functionality of application and improve code reusability
+ by composing the application from smaller building blocks.
+
+Application actions
+ The application provides tools for quick actions. These are intended to be used
+ for application management tasks such as vieving or validating configuration
+ without executing the application itself, listing and evaluating runlogs and
+ so on. There is a number of built-in actions and more can be implemented very
+ easily.
+
+
+Application usage modes
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Applications created based on this framework can be utilized in two work modes:
+
+* **run**
+* **plugin**
+
+In a **run** mode all application features are configured and desired action or
+application code is immediatelly executed.
+
+In a **plugin** mode the application is only configured and any other interactions
+must be performed manually. This approach enables users to plug the apllication into
+another one on a wider scope. One example use case may be the implementation of an
+user command line interface that controls multiple applications (much like *git*).
+
+
+Application life-cycle
+^^^^^^^^^^^^^^^^^^^^^^
+
+Depending on the mode of operation (**run** or **plugin**) the application code goes
+through a different set of stages during its life span.
+
+In a **plugin** mode the following stages are performed:
+
+* **__init__**
+* **setup**
+
+In a **run** mode the following stages are performed in case some *action* is being
+handled:
+
+* **__init__**
+* **setup**
+* **action**
+
+In a **run** mode the following stages are performed in case of normal processing:
+
+* **__init__**
+* **setup**
+* **process**
+* **evaluate**
+* **teardown**
+
+Stage __init__
+``````````````
+
+The **__init__** stage is responsible for creating and basic initialization of
+application object. No exception should occur or be raised during the initialization
+stage and the code should always work regardles of environment setup, so no files
+should be opened, etc. Any exception during this stage will intentionally not get
+handled in any way and result in full traceback dump and immediate application
+termination.
+
+These more advanced setup tasks should be performed during the
+**setup** stage, which is capable of intelligent catching and displaying/logging
+of any exceptions. There are following substages in this stage:
+
+* init command line argument parser: :py:func:`BaseApp._init_argparser`
+* parse command line arguments: :py:func:`BaseApp._parse_cli_arguments`
+* initialize application name: :py:func:`BaseApp._init_name`
+* initialize filesystem paths: :py:func:`BaseApp._init_paths`
+* initialize application runlog: :py:func:`BaseApp._init_runlog`
+* initialize default configurations: :py:func:`BaseApp._init_config`
+* subclass hook for additional initializations: :py:func:`BaseApp._sub_stage_init`
+
+Any of the previous substages may be overriden in a subclass to enhance or alter
+the functionality, but always be sure of what you are doing.
+
+Stage *setup*
+`````````````
+
+The **setup** stage is responsible for bootstrapping the whole application. Any failure
+It
+consists of couple of substages:
+
+* setup configuration: :py:func:`BaseApp._stage_setup_configuration`
+* setup user and group privileges: :py:func:`BaseApp._stage_setup_privileges`
+* setup logging: :py:func:`BaseApp._stage_setup_logging`
+* setup persistent state: :py:func:`BaseApp._stage_setup_pstate`
+* setup plugins: :py:func:`BaseApp._stage_setup_plugins`
+* subclass hook for additional setup: :py:func:`BaseApp._sub_stage_setup`
+* setup dump: :py:func:`BaseApp._stage_setup_dump`
+
+Stage *action*
+``````````````
+
+The **action** stage takes care of executing built-in actions.
+
+Stage *process*
+```````````````
+
+The **process** stage is supposed to perform any required task and process runlog.
+
+Stage *evaluate*
+````````````````
+
+The **evaluate** stage is supposed to perform any evaluation of current runlog.
+
+Stage *teardown*
+````````````````
+
+The **teardown** stage is supposed to perform any cleanup tasks before the application
+exits. It consists of couple of substages:
+
+* :py:func:`BaseApp._sub_stage_teardown`
+* :py:func:`BaseApp._stage_teardown_pstate`
+* :py:func:`BaseApp._stage_teardown_runlog`
+
+
+Programming API
+^^^^^^^^^^^^^^^
+
+* public attributes:
+
+ * ``self.name``
+ * ``self.paths``
+ * ``self.runlog``
+ * ``self.config``
+ * ``self.logger``
+ * ``self.pstate``
+ * ``self.retc``
+
+* public methods:
+
+ * ``self.c``
+ * ``self.cc``
+ * ``self.p``
+ * ``self.dbgout``
+ * ``self.excout``
+ * ``self.error``
+ * ``self.json_dump``
+ * ``self.json_load``
+ * ``self.json_save``
+
+
+Application actions
+^^^^^^^^^^^^^^^^^^^
+
+* *config-view*
+* *runlog-dump*
+* *runlog-view*
+* *runlogs-dump*
+* *runlogs-list*
+* *runlogs-evaluate*
+
-This class provides base implementation of generic processing application with many
-usefull features including (but not limited to) following:
+Subclass extension hooks
+^^^^^^^^^^^^^^^^^^^^^^^^
-* Optional configuration via external JSON configuration file
-* Optional configuration via configuration directory
-* Optional configuration via command line arguments and options
-* Optional logging to console
-* Optional logging to text file
-* Optional persistent state storage between script executions
-* Optional runlog saving after each script execution
-* Integrated runlog analysis tools
+* :py:func:`BaseApp._sub_stage_init`
+* :py:func:`BaseApp._sub_stage_setup`
+* :py:func:`BaseApp._sub_stage_process`
+* :py:func:`BaseApp._sub_stage_teardown`
+* :py:func:`BaseApp._sub_runlog_analyze`
+* :py:func:`BaseApp._sub_runlog_format_analysis`
+* :py:func:`BaseApp._sub_runlogs_evaluate`
+* :py:func:`BaseApp._sub_runlogs_format_evaluation`
+
+Module contents
+^^^^^^^^^^^^^^^
+
+* :py:class:`ZenAppException`
+
+ * :py:class:`ZenAppSetupException`
+ * :py:class:`ZenAppProcessException`
+ * :py:class:`ZenAppEvaluateException`
+ * :py:class:`ZenAppTeardownException`
+
+* :py:class:`ZenAppPlugin`
+* :py:class:`BaseApp`
+* :py:class:`DemoBaseApp`
"""
+
+__author__ = "Jan Mach <honza.mach.ml@gmail.com>"
+
+
import os
import sys
import pwd
import grp
import re
-import shutil
import glob
-import math
import time
-import json
import argparse
import logging
import logging.handlers
@@ -40,76 +248,140 @@ import subprocess
import datetime
import traceback
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
-sys.path.insert(0, lib)
-
#
# Custom libraries.
#
import pyzenkit.jsonconf
import pydgets.widgets
-#
-# Global variables.
-#
-# Global flag, that turns on additional debugging messages.
-FLAG_DEBUG = False
+#-------------------------------------------------------------------------------
-def _json_default(o):
- return repr(o)
class ZenAppException(Exception):
"""
- Base class for all ZenApp specific exceptions.
-
- These exceptions will be catched, error will be displayed to the user and
- script will attempt to gracefully terminate without dumping the traceback
- to the user. These exceptions should be used for anticipated errors, which
- can occur during normal script execution and do not mean there is anything
- wrong with the script itself, for example missing configuration file, etc...
+ Base class for all ZenApp custom exceptions.
+
+ When appropriate, these exceptions will be catched, error will be displayed
+ to the user and the application will attempt to gracefully terminate without
+ dumping the traceback visibly to the user. These exceptions should be used
+ for anticipated errors, which can occur during normal application execution and
+ do not mean there is anything wrong with the code itself, for example missing
+ configuration file, etc...
"""
- def __init__(self, description):
- self._description = description
+ def __init__(self, description, **params):
+ """
+ Initialize new exception with given description and optional additional
+ parameters.
+
+ :param str description: Description of the problem.
+ :param params: Optional additional parameters.
+ """
+ super().__init__()
+
+ self.description = description
+ self.params = params
+
def __str__(self):
- return repr(self._description)
+ """
+ Operator override for automatic string output.
+ """
+ return repr(self.description)
class ZenAppSetupException(ZenAppException):
"""
- Describes problems or errors during the script 'setup' phase.
+ Describes problems or errors that occur during the **setup** phase.
"""
pass
class ZenAppProcessException(ZenAppException):
"""
- Describes problems or errors during the script 'process' phase.
+ Describes problems or errors that occur during the **process** phase.
"""
pass
class ZenAppEvaluateException(ZenAppException):
"""
- Describes problems or errors during the script 'evaluate' phase.
+ Describes problems or errors that occur during the **evaluate** phase.
"""
pass
class ZenAppTeardownException(ZenAppException):
"""
- Describes problems or errors during the script 'teardown' phase.
+ Describes problems or errors that occur during the **teardown** phase.
"""
pass
-class BaseApp:
+
+#-------------------------------------------------------------------------------
+
+
+class ZenAppPlugin:
+ """
+ Base class for all ZenApp application plugins. Plugins can be used to further
+ enhance the code reusability by composing the application from smaller building
+ blocks.
"""
- Base implementation of generic executable script.
- This class attempts to provide robust and stable framework, which can be used
- to writing all kinds of scripts or daemons. This is however low level framework
- and should not be used directly, use the zenscript.py or zendaemon.py
- modules for writing custom scripts or daemons respectively.
+ def __str__(self):
+ """
+ Operator override for automatic string output.
+ """
+ return self.__class__.__name__
+
+ def init_argparser(self, app, argparser, **kwargs):
+ """
+ Callback to be called during argparser initialization phase.
+ """
+ return argparser
+
+ def init_config(self, app, config, **kwargs):
+ """
+ Callback to be called during default configuration initialization phase.
+ """
+ return config
+
+ def init_runlog(self, app, runlog, **kwargs):
+ """
+ Callback to be called during runlog initialization phase.
+ """
+ return runlog
+
+ def configure(self, app):
+ """
+ Callback to be called during configuration phase (after initialization).
+ """
+ raise NotImplementedError('This method must be implemented in subclass')
+
+ def setup(self, app):
+ """
+ Callback to be called during setup phase (after setup).
+ """
+ raise NotImplementedError('This method must be implemented in subclass')
+
+
+#-------------------------------------------------------------------------------
+
+
+class BaseApp:
+ """
+ Base implementation of generic executable application. This class attempts to
+ provide robust and stable framework, which can be used to writing all kinds
+ of scripts or daemons. Although is is usable, this is however a low level framework
+ and should not be used directly, use the :py:mod:`pyzenkit.zenscript` or :py:mod:`pyzenkit.zendaemon`
+ modules for writing custom scripts or daemons respectively. That being said,
+ the :py:class:`pyzenkit.baseapp.DemoBaseApp` class is an example implementation
+ of using this class directly without any additional overhead.
"""
- # List of all possible return codes
+ #
+ # Class constants.
+ #
+
+ # Global flag, that turns on additional debugging messages.
+ FLAG_DEBUG = False
+
+ # List of all possible return codes.
RC_SUCCESS = os.EX_OK
RC_FAILURE = 1
@@ -117,10 +389,11 @@ class BaseApp:
RESULT_SUCCESS = 'success'
RESULT_FAILURE = 'failure'
- # String patterns
+ # String patterns.
PTRN_ACTION_CBK = 'cbk_action_'
+ PTRN_APP_NAME = '^[_a-zA-Z][-_a-zA-Z0-9.]*$'
- # Paths
+ # Paths.
PATH_BIN = 'bin'
PATH_CFG = 'cfg'
PATH_LOG = 'log'
@@ -130,8 +403,8 @@ class BaseApp:
# List of core configuration keys.
CORE = '__core__'
CORE_LOGGING = 'logging'
- CORE_LOGGING_TOFILE = 'tofile'
- CORE_LOGGING_TOCONS = 'toconsole'
+ CORE_LOGGING_TOFILE = 'to_file'
+ CORE_LOGGING_TOCONS = 'to_console'
CORE_LOGGING_LEVEL = 'level'
CORE_LOGGING_LEVELF = 'level_file'
CORE_LOGGING_LEVELC = 'level_console'
@@ -140,12 +413,15 @@ class BaseApp:
CORE_RUNLOG = 'runlog'
CORE_RUNLOG_SAVE = 'save'
- # List of possible configuration keys.
+ # List of configuration keys.
+ CONFIG_PLUGINS = 'plugins'
CONFIG_DEBUG = 'debug'
CONFIG_QUIET = 'quiet'
CONFIG_VERBOSITY = 'verbosity'
CONFIG_RUNLOG_DUMP = 'runlog_dump'
CONFIG_PSTATE_DUMP = 'pstate_dump'
+ CONFIG_RUNLOG_LOG = 'runlog_log'
+ CONFIG_PSTATE_LOG = 'pstate_log'
CONFIG_NAME = 'name'
CONFIG_ACTION = 'action'
CONFIG_INPUT = 'input'
@@ -159,20 +435,19 @@ class BaseApp:
CONFIG_PSTATE_FILE = 'pstate_file'
CONFIG_RUNLOG_DIR = 'runlog_dir'
- # Runlog keys
- RLKEY_NAME = 'name'
- RLKEY_PID = 'pid'
- RLKEY_ARGV = 'argv'
- RLKEY_COMMAND = 'command'
- RLKEY_TS = 'ts'
- RLKEY_TSFSF = 'ts_fsf'
- RLKEY_TSSTR = 'ts_str'
- RLKEY_RESULT = 'result'
- RLKEY_RC = 'rc'
- RLKEY_ERRORS = 'errors'
- RLKEY_TMARKS = 'time_marks'
-
- # Runlog analysis keys
+ # List of runlog keys.
+ RLKEY_NAME = 'name' # Application name.
+ RLKEY_PID = 'pid' # Application process PID.
+ RLKEY_ARGV = 'argv' # Application command line arguments.
+ RLKEY_TS = 'ts' # Timestamp as float.
+ RLKEY_TSFSF = 'ts_fsf' # Timestamp as sortable string (usefull for generating sortable file names).
+ RLKEY_TSSTR = 'ts_str' # Timestamp as readable string.
+ RLKEY_RESULT = 'result' # Result as a string.
+ RLKEY_RC = 'rc' # Result as numeric return code.
+ RLKEY_ERRORS = 'errors' # List of arrors during execution.
+ RLKEY_TMARKS = 'time_marks' # Time measuring marks.
+
+ # List of runlog analysis keys.
RLANKEY_LABEL = 'label'
RLANKEY_COMMAND = 'command'
RLANKEY_AGE = 'age'
@@ -185,177 +460,206 @@ class BaseApp:
RLANKEY_DURATIONS = 'durations'
RLANKEY_EFFECTIVITY = 'effectivity'
- # Runlog evaluation keys
+ # List of runlog evaluation keys.
RLEVKEY_ANALYSES = 'analyses'
+
+ #---------------------------------------------------------------------------
+ # "__INIT__" STAGE METHODS.
+ #---------------------------------------------------------------------------
+
+
def __init__(self, **kwargs):
"""
- Default script object constructor.
+ Base application object constructor. Only defines core internal variables.
+ The actual object initialization, during which command line arguments and
+ configuration files are parsed, is done during the configure() stage of
+ the run() sequence.
- Only defines core internal variables. The actual object initialization,
- during which command line arguments and configuration files are parsed,
- is done during the configure() stage of the run() sequence.
+ :param kwargs: Various additional parameters.
"""
- # [PUBLIC] Default script help description.
- self.description = kwargs.get('description', 'BaseApp - Simple generic script')
+ # Initialize list of desired plugins.
+ self._plugins = kwargs.get(self.CONFIG_PLUGINS, [])
+
+ # [PUBLIC] Default application help description.
+ self.description = kwargs.get('description', 'BaseApp - Generic application')
# [PUBLIC] Initialize command line argument parser.
self.argparser = self._init_argparser(**kwargs)
# Parse CLI arguments immediatelly, we need to check for a few priority
- # flags and switches
- self._config_cli = self._parse_cli_arguments()
+ # flags and switches.
+ self._config_cli = self._parse_cli_arguments(self.argparser)
+ self._config_file = None
+ self._config_dir = None
- # [PUBLIC] Detect name of the script.
+ # [PUBLIC] Detect name of the application.
self.name = self._init_name(**kwargs)
- # [PUBLIC] Script paths.
+ # [PUBLIC] Script paths, will be used to construct various absolute file paths.
self.paths = self._init_paths(**kwargs)
# [PUBLIC] Script processing runlog.
self.runlog = self._init_runlog(**kwargs)
- # [PUBLIC] Storage for script configurations.
- self.config = self._init_config(**kwargs)
- # [PUBLIC] Logger object.
+ # [PUBLIC] Storage for application configurations.
+ self.config = self._init_config((), **kwargs)
+ # [PUBLIC] Internal logger object.
self.logger = None
# [PUBLIC] Persistent state object.
self.pstate = None
# [PUBLIC] Final return code.
- self.rc = self.RC_SUCCESS
+ self.retc = self.RC_SUCCESS
# Perform subinitializations on default configurations and argument parser.
- self._init_custom(self.config, self.argparser, **kwargs)
+ self._sub_stage_init(**kwargs)
- def __del__(self):
- """
- Default script object destructor. Perform generic cleanup.
+ def _init_argparser(self, **kwargs):
"""
- pass
+ Initialize application command line argument parser. This method may be overriden
+ in subclasses, however it must return valid :py:class:`argparse.ArgumentParser`
+ object.
- #---------------------------------------------------------------------------
- # Object initialization helper methods
- #---------------------------------------------------------------------------
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
- def _init_argparser(self, **kwargs):
- """
- Initialize script command line argument parser.
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Initialized argument parser object.
+ :rtype: argparse.ArgumentParser
"""
argparser = argparse.ArgumentParser(description = self.description)
- # Option flag indicating that script is running in debug mode. This option
+ # Option flag indicating that application is running in debug mode. This option
# will enable displaying of additional helpful debugging messages. The
# messages will be printed directly to terminal, without the use of
# logging framework.
argparser.add_argument('--debug', help = 'run in debug mode (flag)', action = 'store_true', default = None)
- # Option flag indicating that script is running in quiet mode. This option
- # will prevent script from displaying information to console.
- argparser.add_argument('--quiet', help = 'run in quiet mode (flag)', action = 'store_true', default = None)
-
- # Option for setting the output verbosity level.
- argparser.add_argument('--verbosity', help = 'increase output verbosity', action = 'count', default = None)
-
- # Option flag indicating that the script should dump the runlog to logger,
- # when the processing is done.
- argparser.add_argument('--runlog-dump', help = 'dump runlog when done processing (flag)', action = 'store_true', default = None)
-
- # Option flag indicating that the script should dump the persistent state to logger,
- # when the processing is done.
- argparser.add_argument('--pstate-dump', help = 'dump persistent state when done processing (flag)', action = 'store_true', default = None)
-
- # Option for overriding the name of the component.
- argparser.add_argument('--name', help = 'name of the component')
-
- # Option for setting the desired action.
- argparser.add_argument('--action', help = 'choose which action should be performed', choices = self._utils_detect_actions())
+ # Setup mutually exclusive group for quiet x verbose mode option.
+ group_a = argparser.add_mutually_exclusive_group()
- # Option for setting the desired operation.
- argparser.add_argument('--input', help = 'file to be used as source file in action')
+ # Option flag indicating that application is running in quiet mode. This option
+ # will prevent application from displaying any information to console.
+ group_a.add_argument('--quiet', help = 'run in quiet mode (flag)', action = 'store_true', default = None)
- # Option for setting the result limit.
- argparser.add_argument('--limit', help = 'apply given limit to the result', type = int)
-
- # Option for overriding the process UID.
- argparser.add_argument('--user', help = 'process UID or user name')
-
- # Option for overriding the process GID.
- argparser.add_argument('--group', help = 'process GID or group name')
+ # Option for setting the output verbosity level.
+ group_a.add_argument('--verbose', help = 'increase output verbosity (flag, repeatable)', action = 'count', default = None, dest = 'verbosity')
- # Option for overriding the name of the configuration file.
- argparser.add_argument('--config-file', help = 'name of the config file')
+ #
+ # Create and populate options group for common application arguments.
+ #
+ arggroup_common = argparser.add_argument_group('common application arguments')
- # Option for overriding the name of the configuration directory.
- argparser.add_argument('--config-dir', help = 'name of the config directory')
+ arggroup_common.add_argument('--name', help = 'name of the application', type = str, default = None)
+ arggroup_common.add_argument('--user', help = 'process UID or user name', default = None)
+ arggroup_common.add_argument('--group', help = 'process GID or group name', default = None)
- # Option for overriding the name of the log file.
- argparser.add_argument('--log-file', help = 'name of the log file')
+ arggroup_common.add_argument('--config-file', help = 'name of the configuration file', type = str, default = None)
+ arggroup_common.add_argument('--config-dir', help = 'name of the configuration directory', type = str, default = None)
+ arggroup_common.add_argument('--log-file', help = 'name of the log file', type = str, default = None)
+ arggroup_common.add_argument('--log-level', help = 'set logging level', choices = ['debug', 'info', 'warning', 'error', 'critical'], type = str, default = None)
- # Option for setting the level of logging information.
- argparser.add_argument('--log-level', help = 'set logging level', choices = ['debug', 'info', 'warning', 'error', 'critical'])
+ arggroup_common.add_argument('--action', help = 'name of the quick action to be performed', choices = self._utils_detect_actions(), type = str, default = None)
+ arggroup_common.add_argument('--input', help = 'file to be used as source file in action', type = str, default = None)
+ arggroup_common.add_argument('--limit', help = 'apply given limit to the result', type = int, default = None)
- # Option for overriding the name of the persistent state file.
- argparser.add_argument('--pstate-file', help = 'name of the persistent state file')
+ arggroup_common.add_argument('--pstate-file', help = 'name of the persistent state file', type = str, default = None)
+ arggroup_common.add_argument('--pstate-dump', help = 'dump persistent state to stdout when done processing (flag)', action = 'store_true', default = None)
+ arggroup_common.add_argument('--pstate-log', help = 'write persistent state to logging service when done processing (flag)', action = 'store_true', default = None)
- # Option for overriding the name of the runlog directory.
- argparser.add_argument('--runlog-dir', help = 'name of the runlog directory')
+ arggroup_common.add_argument('--runlog-dir', help = 'name of the runlog directory', type = str, default = None)
+ arggroup_common.add_argument('--runlog-dump', help = 'dump runlog to stdout when done processing (flag)', action = 'store_true', default = None)
+ arggroup_common.add_argument('--runlog-log', help = 'write runlog to logging service when done processing (flag)', action = 'store_true', default = None)
#argparser.add_argument('args', help = 'optional additional arguments', nargs='*')
+ for plugin in self._plugins:
+ argparser = plugin.init_argparser(self, argparser, **kwargs)
+
return argparser
- def _parse_cli_arguments(self):
+ def _parse_cli_arguments(self, argparser):
"""
- Load and initialize script configuration received from command line.
+ Load and initialize application configuration received as command line arguments.
+ Use the previously configured ;py:class:`argparse.ArgumentParser` object
+ for parsing CLI arguments. Immediatelly perform dirty check for ``--debug``
+ flag to turn on debug output as soon as possible.
+
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
- Use the configured ArgumentParser object for parsing CLI arguments.
+ :param argparse.ArgumentParser argparser: Argument parser object to use.
+ :return: Parsed command line arguments.
+ :rtype: dict
"""
- # Finally actually process command line arguments.
- cli_args = vars(self.argparser.parse_args())
+ # Actually process command line arguments.
+ cli_cfgs = vars(argparser.parse_args())
- # Check for debug flag
- if cli_args.get(self.CONFIG_DEBUG, False):
- global FLAG_DEBUG
- FLAG_DEBUG = True
- self.dbgout("[STATUS] FLAG_DEBUG set to 'True' via command line argument")
+ # Immediatelly check for debug flag.
+ if cli_cfgs.get(self.CONFIG_DEBUG, False):
+ BaseApp.FLAG_DEBUG = True
+ self.dbgout("FLAG_DEBUG set to 'True' via command line option")
- self.dbgout("[STATUS] Parsed command line arguments: '{}'".format(' '.join(sys.argv)))
- return cli_args
+ self.dbgout("Received command line arguments: '{}'".format(' '.join(sys.argv)))
+ return cli_cfgs
def _init_name(self, **kwargs):
"""
- Initialize script name.
+ Initialize application name. The application name will then be used to
+ autogenerate default paths to various files and directories, like log
+ file, persistent state file etc. The default value for application name
+ is automagically detected from command line, or it may be explicitly set
+ either using command line option ``--name``, or by using parameter ``name``
+ of application object constructor.
+
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
+
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Name of the application.
+ :rtype: str
"""
cli_name = self._config_cli.get(self.CONFIG_NAME)
if cli_name:
- if re.fullmatch('^[_a-zA-Z][-_a-zA-Z0-9.]*$', cli_name):
- self.dbgout("[STATUS] Using custom script name '{}".format(cli_name))
+ if re.fullmatch(self.PTRN_APP_NAME, cli_name):
+ self.dbgout("Using custom application name '{}' received as command line option".format(cli_name))
return cli_name
- else:
- raise ZenAppException("Invalid script name '{}'. Valid pattern is '^[a-zA-Z][-_a-zA-Z0-9]*$'".format(cli_name))
- elif 'name' in kwargs:
- if re.fullmatch('^[_a-zA-Z][-_a-zA-Z0-9.]*$', kwargs['name']):
- self.dbgout("[STATUS] Using custom script name '{}".format(kwargs['name']))
- return kwargs['name']
- else:
- raise ZenAppException("Invalid script name '{}'. Valid pattern is '^[a-zA-Z][-_a-zA-Z0-9]*$'".format(cli_name))
- else:
- scr_name = os.path.basename(sys.argv[0])
- self.dbgout("[STATUS] Using default script name '{}".format(scr_name))
- return scr_name
+ raise ZenAppException("Invalid application name '{}'. Valid pattern is '{}'".format(cli_name, self.PTRN_APP_NAME))
+
+ if self.CONFIG_NAME in kwargs:
+ if re.fullmatch(self.PTRN_APP_NAME, kwargs[self.CONFIG_NAME]):
+ self.dbgout("Using custom application name '{}' received as constructor option".format(kwargs[self.CONFIG_NAME]))
+ return kwargs[self.CONFIG_NAME]
+ raise ZenAppException("Invalid application name '{}'. Valid pattern is '{}'".format(cli_name, self.PTRN_APP_NAME))
+
+ app_name = os.path.basename(sys.argv[0])
+ self.dbgout("Using default application name '{}".format(app_name))
+ return app_name
def _init_paths(self, **kwargs):
"""
- Initialize various script paths.
+ Initialize various application filesystem paths like temp directory, log
+ directory etc. These values will when be used to autogenerate default paths
+ to various files and directories, like log file, persistent state file etc.
+
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
+
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Configurations for various filesystem paths.
+ :rtype: dict
"""
return {
- self.PATH_BIN: kwargs.get('path_bin', "/usr/local/bin"), # Application executable directory.
- self.PATH_CFG: kwargs.get('path_cfg', "/etc"), # Configuration directory.
- self.PATH_LOG: kwargs.get('path_log', "/var/log"), # Log directory.
- self.PATH_RUN: kwargs.get('path_run', "/var/run"), # Runlog directory.
- self.PATH_TMP: kwargs.get('path_tmp', "/var/tmp"), # Temporary file directory.
+ self.PATH_BIN: kwargs.get('path_{}'.format(self.PATH_BIN), "/usr/local/bin"), # Application executable directory.
+ self.PATH_CFG: kwargs.get('path_{}'.format(self.PATH_CFG), "/etc"), # Configuration directory.
+ self.PATH_LOG: kwargs.get('path_{}'.format(self.PATH_LOG), "/var/log"), # Log directory.
+ self.PATH_RUN: kwargs.get('path_{}'.format(self.PATH_RUN), "/var/run"), # Runlog directory.
+ self.PATH_TMP: kwargs.get('path_{}'.format(self.PATH_TMP), "/var/tmp"), # Temporary file directory.
}
def _init_runlog(self, **kwargs):
"""
- Initialize script runlog.
+ Initialize application runlog. Runlog should contain vital information about
+ application progress and it will be stored into file upon exit.
+
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
+
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Runlog structure.
+ :rtype: dict
"""
runlog = {
self.RLKEY_NAME: self.name,
@@ -371,11 +675,23 @@ class BaseApp:
# Timestamp as readable string.
runlog[self.RLKEY_TSSTR] = time.strftime('%Y-%m-%d %X', time.localtime(runlog[self.RLKEY_TS]))
+ for plugin in self._plugins:
+ runlog = plugin.init_runlog(self, runlog, **kwargs)
+
return runlog
- def _init_config(self, **kwargs):
+ def _init_config(self, cfgs, **kwargs):
"""
- Initialize script configurations to default values.
+ Initialize default application configurations. This method may be used
+ from subclasses by passing any additional configurations in ``cfgs``
+ parameter.
+
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
+
+ :param list cfgs: Additional set of configurations.
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Default configuration structure.
+ :rtype: dict
"""
cfgs = (
(self.CONFIG_DEBUG, False),
@@ -383,6 +699,8 @@ class BaseApp:
(self.CONFIG_VERBOSITY, 0),
(self.CONFIG_RUNLOG_DUMP, False),
(self.CONFIG_PSTATE_DUMP, False),
+ (self.CONFIG_RUNLOG_LOG, False),
+ (self.CONFIG_PSTATE_LOG, False),
(self.CONFIG_ACTION, None),
(self.CONFIG_INPUT, None),
(self.CONFIG_LIMIT, None),
@@ -394,286 +712,276 @@ class BaseApp:
(self.CONFIG_LOG_LEVEL, 'info'),
(self.CONFIG_PSTATE_FILE, os.path.join(self.paths.get(self.PATH_RUN), "{}.pstate".format(self.name))),
(self.CONFIG_RUNLOG_DIR, os.path.join(self.paths.get(self.PATH_RUN), "{}".format(self.name))),
- )
+ ) + cfgs
config = {}
- for c in cfgs:
- config[c[0]] = kwargs.get('default_' + c[0], c[1])
+ for cfg in cfgs:
+ config[cfg[0]] = kwargs.get('default_{}'.format(cfg[0]), cfg[1])
+
+ for plugin in self._plugins:
+ config = plugin.init_config(self, config, **kwargs)
+
return config
- def _init_custom(self, config, argparser, **kwargs):
- """
- Perform subinitializations on default configurations and argument parser.
- """
- pass
#---------------------------------------------------------------------------
- # Template method hooks (intended to be used by subclassess)
+ # TEMPLATE METHOD HOOKS (INTENDED TO BE USED BY SUBCLASSESS).
#---------------------------------------------------------------------------
- def _sub_runlog_analyze(self, runlog, analysis):
- """
- Analyze given runlog (hook for subclasses).
- """
- return analysis
- def _sub_runlog_format_analysis(self, analysis):
+ def _sub_stage_init(self, **kwargs):
"""
- Format given runlog analysis (hook for subclasses).
+ **SUBCLASS HOOK**: Perform additional custom initialization actions in **__init__** stage.
+
+ :param kwargs: Various additional parameters passed down from constructor.
"""
pass
- def _sub_runlogs_evaluate(self, runlogs, evaluation):
+ def _sub_stage_setup(self):
"""
- Evaluate given runlog analyses (hook for subclasses).
- """
- return evaluation
+ **SUBCLASS HOOK**: Perform additional custom setup actions in **setup** stage.
- def _sub_runlogs_format_evaluation(self, evaluation):
- """
- Format given runlogs evaluation (hook for subclasses).
+ Gets called from :py:func:`BaseApp._stage_setup` and it is a **SETUP SUBSTAGE 06**.
"""
pass
- #---------------------------------------------------------------------------
- # Helpers and shortcut methods
- #---------------------------------------------------------------------------
-
- def get_fn_runlog(self):
+ def _sub_stage_process(self):
"""
- Return the name of the runlog file for current process.
+ **SUBCLASS HOOK**: Perform some actual processing in **process** stage.
"""
- return os.path.join(self.c(self.CONFIG_RUNLOG_DIR), "{}.runlog".format(self.runlog[self.RLKEY_TSFSF]))
+ raise NotImplementedError("Method must be implemented in subclass")
- def get_fn_pstate(self):
- """
- Return the name of the persistent state file for current process.
+ def _sub_stage_teardown(self):
"""
- return self.c(self.CONFIG_PSTATE_FILE)
+ **SUBCLASS HOOK**: Perform additional teardown actions in **teardown** stage.
- def c(self, key, default = None):
+ Gets called from :py:func:`BaseApp._stage_teardown` and it is a **TEARDOWN SUBSTAGE 01**.
"""
- Shortcut method: Get given configuration value.
- """
- if default is None:
- return self.config.get(key)
- else:
- return self.config.get(key, default)
+ pass
- def cc(self, key, default = None):
+ def _sub_runlog_analyze(self, runlog, analysis):
"""
- Shortcut method: Get given CORE configuration value.
+ **SUBCLASS HOOK**: Analyze given runlog.
"""
- if default is None:
- return self.config[self.CORE].get(key)
- else:
- return self.config[self.CORE].get(key, default)
+ return analysis
- def p(self, string, level = 0):
+ def _sub_runlog_format_analysis(self, analysis):
"""
- Shortcut method: Print given string.
+ **SUBCLASS HOOK**: Format given runlog analysis.
"""
- if not self.c(self.CONFIG_QUIET) and self.c(self.CONFIG_VERBOSITY) >= level:
- print(string)
+ pass
- def error(self, error, rc = None, tb = None):
+ def _sub_runlogs_evaluate(self, runlogs, evaluation):
"""
- Method for registering error, that occured during script run.
+ **SUBCLASS HOOK**: Evaluate given runlog analyses.
"""
- self.rc = rc if rc is not None else self.RC_FAILURE
-
- errstr = "{}".format(error)
- self.logger.error(errstr)
-
- if tb:
- tbexc = traceback.format_tb(tb)
- self.logger.error("\n" + "".join(tbexc))
-
- self.runlog[self.RLKEY_ERRORS].append(errstr)
- self.runlog[self.RLKEY_RESULT] = self.RESULT_FAILURE
- self.runlog[self.RLKEY_RC] = self.rc
+ return evaluation
- def dbgout(self, message):
+ def _sub_runlogs_format_evaluation(self, evaluation):
"""
- Routine for printing additional debug messages.
-
- The given message is printed only in case the global 'FLAG_DEBUG' flag is
- set to True.
+ **SUBCLASS HOOK**: Format given runlogs evaluation.
"""
- if FLAG_DEBUG:
- print("*DBG* {} {}".format(time.strftime('%Y-%m-%d %X', time.localtime()), message), file=sys.stderr)
+ pass
- def errout(self, exception):
- """
- Routine for printing error messages.
- """
- print("*ERR* {} {}".format(time.strftime('%Y-%m-%d %X', time.localtime()), exception), file=sys.stderr)
- sys.exit(self.RC_FAILURE)
#---------------------------------------------------------------------------
- # Internal utilities
+ # "SETUP:CONFIGURATION" SUBSTAGE METHODS.
#---------------------------------------------------------------------------
- def _utils_detect_actions(self):
- """
- Returns the sorted list of all available actions current script is capable
- of performing. The detection algorithm is based on string analysis of all
- available methods. Any method starting with string 'cbk_action_' will
- be appended to the list, lowercased and with '_' characters replaced with '-'.
- """
- ptrn = re.compile(self.PTRN_ACTION_CBK)
- attrs = sorted(dir(self))
- result = []
- for a in attrs:
- if not callable(getattr(self, a)):
- continue
- match = ptrn.match(a)
- if match:
- result.append(a.replace(self.PTRN_ACTION_CBK,'').replace('_','-').lower())
- return result
- #---------------------------------------------------------------------------
- # CONFIGURATION RELATED METHODS (SETUP CONFIGURATION SUBSTAGE)
- #---------------------------------------------------------------------------
-
- def _configure_cli(self):
+ def _configure_from_cli(self):
"""
- Load and initialize script configuration received from command line.
+ Process application configurations received from command line. It would be
+ much cleaner to do the parsing in this method. However the arguments had
+ to be already parsed and loaded, because we needed to hack the process to
+ check for ``--debug`` option to turn the debug flag on and enable output
+ of debug messages. So only task this method has is to perform some additional
+ processing. The presence of ``--config-file`` or ``--config-dir`` options
+ will cause the appropriate default values to be overwritten. This way an
+ alternative configuration file or directory will be loaded in next step.
- Use the configured ArgumentParser object for parsing CLI arguments.
+ Gets called from :py:func:`BaseApp._stage_setup_configuration` and is
+ therefore part of **SETUP** stage.
"""
# IMPORTANT! Immediatelly rewrite the default value for configuration file
- # and configuration directory names, if the value was received as command
- # line argument.
+ # names, if the value was received as command line argument.
if self._config_cli[self.CONFIG_CFG_FILE] is not None:
- self.dbgout("[STATUS] Config file option override from '{}' to '{}'".format(self.config[self.CONFIG_CFG_FILE], self._config_cli[self.CONFIG_CFG_FILE]))
self.config[self.CONFIG_CFG_FILE] = self._config_cli[self.CONFIG_CFG_FILE]
+ self.dbgout("Config file name overridden from '{}' to '{}' by command line option".format(self.config[self.CONFIG_CFG_FILE], self._config_cli[self.CONFIG_CFG_FILE]))
+
+ # IMPORTANT! Immediatelly rewrite the default value for configuration file
+ # names, if the value was received as command line argument.
if self._config_cli[self.CONFIG_CFG_DIR] is not None:
- self.dbgout("[STATUS] Config directory option override from '{}' to '{}'".format(self.config[self.CONFIG_CFG_DIR], self._config_cli[self.CONFIG_CFG_DIR]))
self.config[self.CONFIG_CFG_DIR] = self._config_cli[self.CONFIG_CFG_DIR]
+ self.dbgout("Config directory name overridden from '{}' to '{}' by command line option".format(self.config[self.CONFIG_CFG_DIR], self._config_cli[self.CONFIG_CFG_DIR]))
- return self._config_cli
-
- def _configure_file(self):
+ def _configure_from_file(self):
"""
- Load and initialize script configuration received from configuration file.
+ Load and initialize application configurations from single configuration file.
+
+ Gets called from :py:func:`BaseApp._stage_setup_configuration` and is
+ therefore part of **SETUP** stage.
"""
try:
self._config_file = pyzenkit.jsonconf.config_load(self.c(self.CONFIG_CFG_FILE))
- self.dbgout("[STATUS] Loaded configuration file '{}'".format(self.c(self.CONFIG_CFG_FILE)))
+ self.dbgout("Loaded contents of configuration file '{}'".format(self.c(self.CONFIG_CFG_FILE)))
+
except FileNotFoundError as exc:
raise ZenAppSetupException("Configuration file '{}' does not exist".format(self.c(self.CONFIG_CFG_FILE)))
- def _configure_dir(self):
+ def _configure_from_dir(self):
"""
- Load and initialize script configuration received from configuration directory.
+ Load and initialize application configurations from multiple files in
+ configuration directory.
+
+ Gets called from :py:func:`BaseApp._stage_setup_configuration` and is
+ therefore part of **SETUP** stage.
"""
try:
self._config_dir = pyzenkit.jsonconf.config_load_dir(self.c(self.CONFIG_CFG_DIR))
- self.dbgout("[STATUS] Loaded configuration directory '{}'".format(self.c(self.CONFIG_CFG_DIR)))
+ self.dbgout("Loaded contents of configuration directory '{}'".format(self.c(self.CONFIG_CFG_DIR)))
+
except FileNotFoundError as exc:
raise ZenAppSetupException("Configuration directory '{}' does not exist".format(self.c(self.CONFIG_CFG_DIR)))
def _configure_merge(self):
"""
- Configure script and produce final configuration by merging all available
+ Configure application and produce final configuration by merging all available
configuration values in appropriate order ('default' <= 'dir' <= 'file' <= 'cli').
+
+ Gets called from :py:func:`BaseApp._stage_setup_configuration` and is
+ therefore part of **SETUP** stage.
"""
+ exceptions = (self.CONFIG_CFG_FILE, self.CONFIG_CFG_DIR)
+
# Merge configuration directory values with current config, if possible.
if self.c(self.CONFIG_CFG_DIR, False):
- self.dbgout("[STATUS] Merging global config with DIRECTORY configurations")
- self.config.update((k, v) for k, v in self._config_dir.items() if v is not None)
+ self.config.update((key, val) for key, val in self._config_dir.items() if val is not None and key not in exceptions)
+ self.dbgout("Merged directory configurations into global configurations")
# Merge configuration file values with current config, if possible.
if self.c(self.CONFIG_CFG_FILE, False):
- self.dbgout("[STATUS] Merging global config with FILE configurations")
- self.config.update((k, v) for k, v in self._config_file.items() if v is not None)
+ self.config.update((key, val) for key, val in self._config_file.items() if val is not None and key not in exceptions)
+ self.dbgout("Merged file configurations into global configurations")
# Merge command line values with current config, if possible.
- self.dbgout("[STATUS] Merging global config with CLI configurations")
- self.config.update((k, v) for k, v in self._config_cli.items() if v is not None)
+ self.config.update((key, val) for key, val in self._config_cli.items() if val is not None)
+ self.dbgout("Merged command line configurations into global configurations")
def _configure_postprocess(self):
"""
- Perform configuration postprocessing.
+ Perform configuration postprocessing and calculate core configurations.
+
+ Gets called from :py:func:`BaseApp._stage_setup_configuration` and is
+ therefore part of **SETUP** stage.
"""
+ # Always mstart with a clean slate.
self.config[self.CORE] = {}
- cc = {}
- cc[self.CORE_LOGGING_TOCONS] = True
- cc[self.CORE_LOGGING_TOFILE] = True
- cc[self.CORE_LOGGING_LEVEL] = self.c(self.CONFIG_LOG_LEVEL).upper()
- cc[self.CORE_LOGGING_LEVELF] = cc[self.CORE_LOGGING_LEVEL]
- cc[self.CORE_LOGGING_LEVELC] = cc[self.CORE_LOGGING_LEVEL]
- self.config[self.CORE][self.CORE_LOGGING] = cc
-
- cc = {}
- cc[self.CORE_PSTATE_SAVE] = True
- self.config[self.CORE][self.CORE_PSTATE] = cc
-
- cc = {}
- cc[self.CORE_RUNLOG_SAVE] = True
- self.config[self.CORE][self.CORE_RUNLOG] = cc
-
+ # Initial logging configurations.
+ ccfg = {}
+ ccfg[self.CORE_LOGGING_TOCONS] = True
+ ccfg[self.CORE_LOGGING_TOFILE] = True
+ ccfg[self.CORE_LOGGING_LEVEL] = self.c(self.CONFIG_LOG_LEVEL).upper()
+ ccfg[self.CORE_LOGGING_LEVELF] = ccfg[self.CORE_LOGGING_LEVEL]
+ ccfg[self.CORE_LOGGING_LEVELC] = ccfg[self.CORE_LOGGING_LEVEL]
+ self.config[self.CORE][self.CORE_LOGGING] = ccfg
+
+ # Initial persistent configurations.
+ ccfg = {}
+ ccfg[self.CORE_PSTATE_SAVE] = True
+ self.config[self.CORE][self.CORE_PSTATE] = ccfg
+
+ # Initial runlog configurations.
+ ccfg = {}
+ ccfg[self.CORE_RUNLOG_SAVE] = True
+ self.config[self.CORE][self.CORE_RUNLOG] = ccfg
+
+ # Postprocess user account configurations, when necessary.
if self.config[self.CONFIG_USER]:
- u = self.config[self.CONFIG_USER]
+ usa = self.config[self.CONFIG_USER]
res = None
if not res:
try:
- res = pwd.getpwnam(u)
+ res = pwd.getpwnam(usa)
self.config[self.CONFIG_USER] = [res[0], res[2]]
except:
pass
if not res:
try:
- res = pwd.getpwuid(int(u))
+ res = pwd.getpwuid(int(usa))
self.config[self.CONFIG_USER] = [res[0], res[2]]
except:
pass
if not res:
- raise ZenAppSetupException("Unknown user account '{}'".format(u))
+ raise ZenAppSetupException("Requested unknown user account '{}'".format(usa))
+ self.dbgout("System user account will be set to '{}':'{}'".format(res[0], res[2]))
+
+ # Postprocess group account configurations, when necessary.
if self.config[self.CONFIG_GROUP]:
- g = self.config[self.CONFIG_GROUP]
+ gra = self.config[self.CONFIG_GROUP]
res = None
if not res:
try:
- res = grp.getgrnam(g)
+ res = grp.getgrnam(gra)
self.config[self.CONFIG_GROUP] = [res[0], res[2]]
except:
pass
if not res:
try:
- res = grp.getgrgid(int(g))
+ res = grp.getgrgid(int(gra))
self.config[self.CONFIG_GROUP] = [res[0], res[2]]
except:
pass
if not res:
- raise ZenAppSetupException("Unknown group account '{}'".format(g))
+ raise ZenAppSetupException("Requested unknown group account '{}'".format(gra))
+
+ self.dbgout("System group account will be set to '{}':'{}'".format(res[0], res[2]))
+
+ def _configure_plugins(self):
+ """
+ Perform configurations of all application plugins.
+
+ Gets called from :py:func:`BaseApp._stage_setup_configuration` and is
+ therefore part of **SETUP** stage.
+ """
+ for plugin in self._plugins:
+ self.dbgout("Configuring application plugin '{}'".format(plugin))
+ plugin.configure(self)
def _configure_check(self):
"""
- TODO: Implement config checking mechanism.
+ Perform configuration validation and checking.
+
+ Gets called from :py:func:`BaseApp._stage_setup_configuration` and is
+ therefore part of **SETUP** stage.
+
+ .. todo::
+
+ Missing implementation, work in progress.
"""
pass
#---------------------------------------------------------------------------
- # "SETUP" STAGE RELATED METHODS
+ # "SETUP" STAGE METHODS.
#---------------------------------------------------------------------------
def _stage_setup_configuration(self):
"""
- Setup script configurations.
+ **SETUP SUBSTAGE 01:** Setup application configurations.
+
+ Gets called from :py:func:`BaseApp._stage_setup`.
"""
# Load configurations from command line.
- self._configure_cli()
+ self._configure_from_cli()
# Load configurations from config file, if the appropriate feature is enabled.
if self.c(self.CONFIG_CFG_FILE, False):
- self._configure_file()
+ self._configure_from_file()
# Load configurations from config directory, if the appropriate feature is enabled.
if self.c(self.CONFIG_CFG_DIR, False):
- self._configure_dir()
+ self._configure_from_dir()
# Merge all available configurations together with default.
self._configure_merge()
@@ -681,39 +989,47 @@ class BaseApp:
# Postprocess loaded configurations
self._configure_postprocess()
+ # Postprocess loaded configurations
+ self._configure_plugins()
+
# Check all loaded configurations.
self._configure_check()
def _stage_setup_privileges(self):
"""
- Adjust the script privileges according to the configration.
+ **SETUP SUBSTAGE 02:** Setup application privileges (user and group account).
+
+ Gets called from :py:func:`BaseApp._stage_setup`.
"""
- g = self.c(self.CONFIG_GROUP, None)
- if g and g[1] != os.getgid():
- cg = grp.getgrgid(os.getgid())
- self.dbgout("[STATUS] Dropping group privileges from '{}':'{}' to '{}':'{}'".format(cg[0], cg[2], g[0], g[1]))
- os.setgid(g[1])
- u = self.c(self.CONFIG_USER, None)
- if u and u[1] != os.getuid():
- cu = pwd.getpwuid(os.getuid())
- self.dbgout("[STATUS] Dropping user privileges from '{}':'{}' to '{}':'{}'".format(cu[0], cu[2], u[0], u[1]))
- os.setuid(u[1])
+ gra = self.c(self.CONFIG_GROUP, None)
+ if gra and gra[1] != os.getgid():
+ cga = grp.getgrgid(os.getgid())
+ self.dbgout("Dropping group privileges from '{}':'{}' to '{}':'{}'".format(cga[0], cga[2], gra[0], gra[1]))
+ os.setgid(gra[1])
+
+ usa = self.c(self.CONFIG_USER, None)
+ if usa and usa[1] != os.getuid():
+ cua = pwd.getpwuid(os.getuid())
+ self.dbgout("Dropping user privileges from '{}':'{}' to '{}':'{}'".format(cua[0], cua[2], usa[0], usa[1]))
+ os.setuid(usa[1])
def _stage_setup_logging(self):
"""
- Setup terminal and file logging facilities.
+ **SETUP SUBSTAGE 03:** Setup terminal and file logging facilities.
+
+ Gets called from :py:func:`BaseApp._stage_setup`.
"""
cc = self.cc(self.CORE_LOGGING, {})
if cc[self.CORE_LOGGING_TOCONS] or cc[self.CORE_LOGGING_TOFILE]:
# [PUBLIC] Register the logger object as internal attribute.
- self.logger = logging.getLogger('zenlogger')
+ self.logger = logging.getLogger('zenapplogger')
self.logger.setLevel(cc[self.CORE_LOGGING_LEVEL])
- # Setup console logging
+ # Setup console logging.
if cc[self.CORE_LOGGING_TOCONS]:
logging_level = getattr(logging, cc[self.CORE_LOGGING_LEVELC], None)
if not isinstance(logging_level, int):
- raise ValueError("Invalid log level: '{}'".format(cc[self.CORE_LOGGING_LEVELC]))
+ raise ValueError("Invalid log severity level '{}'".format(cc[self.CORE_LOGGING_LEVELC]))
# Initialize console logging handler.
fm1 = logging.Formatter('%(asctime)s %(levelname)s: %(message)s')
@@ -721,13 +1037,13 @@ class BaseApp:
ch1.setLevel(logging_level)
ch1.setFormatter(fm1)
self.logger.addHandler(ch1)
- self.dbgout("[STATUS] Logging to console with level threshold '{}'".format(cc[self.CORE_LOGGING_LEVELC]))
+ self.dbgout("Logging to console with severity threshold '{}'".format(cc[self.CORE_LOGGING_LEVELC]))
# Setup file logging
if cc[self.CORE_LOGGING_TOFILE]:
logging_level = getattr(logging, cc[self.CORE_LOGGING_LEVELF], None)
if not isinstance(logging_level, int):
- raise ValueError("Invalid log level: '{}'".format(cc[self.CORE_LOGGING_LEVELF]))
+ raise ValueError("Invalid log severity level '{}'".format(cc[self.CORE_LOGGING_LEVELF]))
lfn = self.c(self.CONFIG_LOG_FILE)
fm2 = logging.Formatter('%(asctime)s {} [%(process)d] %(levelname)s: %(message)s'.format(self.name))
@@ -736,83 +1052,97 @@ class BaseApp:
ch2.setLevel(logging_level)
ch2.setFormatter(fm2)
self.logger.addHandler(ch2)
- self.dbgout("[STATUS] Logging to log file '{}' with level threshold '{}'".format(lfn, cc[self.CORE_LOGGING_LEVELF]))
+ self.dbgout("Logging to log file '{}' with severity threshold '{}'".format(lfn, cc[self.CORE_LOGGING_LEVELF]))
def _stage_setup_pstate(self):
"""
- Setup persistent state.
+ **SETUP SUBSTAGE 04:** Setup persistent state from external JSON file.
- Load persistent script state from external file (JSON).
+ Gets called from :py:func:`BaseApp._stage_setup`.
"""
if os.path.isfile(self.c(self.CONFIG_PSTATE_FILE)):
- self.dbgout("[STATUS] Loading persistent state from file '{}'".format(self.c(self.CONFIG_PSTATE_FILE)))
+ self.dbgout("Loading persistent state from file '{}'".format(self.c(self.CONFIG_PSTATE_FILE)))
self.pstate = self.json_load(self.c(self.CONFIG_PSTATE_FILE))
else:
- self.dbgout("[STATUS] Setting default persistent state".format(self.c(self.CONFIG_PSTATE_FILE)))
+ self.dbgout("Setting default empty persistent state")
self.pstate = {}
- def _stage_setup_custom(self):
+ def _stage_setup_plugins(self):
"""
- Custom setup.
+ **SETUP SUBSTAGE 05:** Perform setup of all application plugins.
+
+ Gets called from :py:func:`BaseApp._stage_setup`.
"""
- pass
+ for plugin in self._plugins:
+ self.dbgout("Setting-up application plugin '{}'".format(plugin))
+ plugin.setup(self)
def _stage_setup_dump(self):
"""
- Dump script setup information.
+ **SETUP SUBSTAGE 07:** Dump application setup information. This method will
+ display all vital information about application setup like filesystem paths,
+ configurations etc.
- This method will display information about script system paths, configuration
- loaded from CLI arguments or config file, final merged configuration.
+ Gets called from :py:func:`BaseApp._stage_setup`.
"""
- self.logger.debug("Script name detected as '{}'".format(self.name))
- self.logger.debug("System paths >>>\n{}".format(self.json_dump(self.paths, default=_json_default)))
+ self.logger.debug("Application name is '%s'", self.name)
+ self.logger.debug("System paths >>>\n%s", self.json_dump(self.paths))
if self.c(self.CONFIG_CFG_DIR, False):
- self.logger.debug("Loaded DIRECTORY configurations >>>\n{}".format(self.json_dump(self._config_dir, default=_json_default)))
+ self.logger.debug("Loaded directory configurations >>>\n%s", self.json_dump(self._config_dir))
if self.c(self.CONFIG_CFG_FILE, False):
- self.logger.debug("Loaded FILE configurations >>>\n{}".format(self.json_dump(self._config_file, default=_json_default)))
- self.logger.debug("Loaded CLI configurations >>>\n{}".format(self.json_dump(self._config_cli, default=_json_default)))
- self.logger.debug("Script configurations >>>\n{}".format(self.json_dump(self.config, default=_json_default)))
- self.logger.debug("Loaded persistent state >>>\n{}".format(self.json_dump(self.pstate, default=_json_default)))
+ self.logger.debug("Loaded file configurations >>>\n%s", self.json_dump(self._config_file))
+ self.logger.debug("Loaded command line configurations >>>\n%s", self.json_dump(self._config_cli))
+ self.logger.debug("Final application configurations >>>\n%s", self.json_dump(self.config))
+ self.logger.debug("Loaded persistent state >>>\n%s", self.json_dump(self.pstate))
+ self.logger.debug("Application plugins >>>\n%s", self.json_dump(self._plugins))
+
#---------------------------------------------------------------------------
- # "TEARDOWN" STAGE RELATED METHODS
+ # "TEARDOWN" STAGE METHODS.
#---------------------------------------------------------------------------
- def _stage_teardown_custom(self):
- """
- Custom teardown.
- """
- pass
def _stage_teardown_pstate(self):
"""
- Teardown state.
+ **TEARDOWN SUBSTAGE 02:** Save application persistent state into JSON file, dump
+ it to ``stdout`` or write it to logging service.
- Save persistent script state to external file (JSON).
+ Gets called from :py:func:`BaseApp._stage_teardown`.
"""
if self.cc(self.CORE_PSTATE, {}).get(self.CORE_PSTATE_SAVE):
- self.pstate_save(self.pstate)
+ self._utils_pstate_save(self.pstate)
if self.c(self.CONFIG_PSTATE_DUMP):
- self.pstate_dump(self.pstate)
+ self._utils_pstate_dump(self.pstate)
+ if self.c(self.CONFIG_PSTATE_LOG):
+ self._utils_pstate_log(self.pstate)
def _stage_teardown_runlog(self):
"""
- Teardown runlog.
+ **TEARDOWN SUBSTAGE 03:** Save application runlog into JSON file, dump it to
+ ``stdout`` or write it to logging service.
- Save runlog to external file (JSON) and dump runlog to log.
+ Gets called from :py:func:`BaseApp._stage_teardown`.
"""
if self.cc(self.CORE_RUNLOG, {}).get(self.CORE_RUNLOG_SAVE):
- self.runlog_save(self.runlog)
+ self._utils_runlog_save(self.runlog)
if self.c(self.CONFIG_RUNLOG_DUMP):
- self.runlog_dump(self.runlog)
+ self._utils_runlog_dump(self.runlog)
+ if self.c(self.CONFIG_RUNLOG_LOG):
+ self._utils_runlog_log(self.runlog)
+
#---------------------------------------------------------------------------
- # MAIN STAGE METHODS
+ # MAIN STAGE METHODS.
#---------------------------------------------------------------------------
- def stage_setup(self):
+
+ def _stage_setup(self):
"""
- Script lifecycle stage: SETUP
+ **STAGE:** *SETUP*.
+
+ Perform full application bootstrap. Any exception during this stage will
+ get caught, logged using :py:func:`~BaseApp.excout` method and the application
+ will immediatelly terminate.
"""
self.time_mark('stage_setup_start', 'Start of the setup stage')
@@ -820,7 +1150,7 @@ class BaseApp:
# Setup configurations.
self._stage_setup_configuration()
- # Setup script privileges
+ # Setup application privileges
self._stage_setup_privileges()
# Setup logging, if the appropriate feature is enabled.
@@ -831,70 +1161,89 @@ class BaseApp:
if self.c(self.CONFIG_PSTATE_FILE):
self._stage_setup_pstate()
- # Perform custom setup operations.
- self._stage_setup_custom()
+ # Perform plugin setup actions.
+ self._stage_setup_plugins()
- # Finally dump information about the script setup.
+ # Perform custom setup actions.
+ self._sub_stage_setup()
+
+ # Finally dump information about the application setup.
self._stage_setup_dump()
except ZenAppSetupException as exc:
- # At this point the logging facilities are not yet configured, so we must
- # use other means of diplaying the error to the user. Use custom function
- # to suppres the backtrace print for known issues and errors.
- self.errout(exc)
+ # At this point the logging facilities may not yet be configured, so
+ # we must use other means of diplaying the error to the user. For that
+ # reason use custom function to supress the traceback print for known
+ # issues and errors.
+ self.excout(exc)
self.time_mark('stage_setup_stop', 'End of the setup stage')
- def stage_action(self):
+ def _stage_action(self):
"""
- Script lifecycle stage: ACTION
+ **STAGE:** *ACTION*.
+
+ Perform a quick action. Following method will call appropriate callback
+ method to service the requested action. The application will immediatelly
+ terminate afterwards.
- Perform some quick action. Following method will call appropriate
- callback method to service the selected action.
+ Name of the callback method is calculated from the name of the action by
+ prepending string ``cbk_action_`` and replacing all ``-`` with ``_``.
"""
self.time_mark('stage_action_start', 'Start of the action stage')
try:
- # Determine, which operation to execute.
+ # Determine, which action to execute.
self.runlog[self.CONFIG_ACTION] = self.c(self.CONFIG_ACTION)
- opname = self.c(self.CONFIG_ACTION)
- opcbkname = self.PTRN_ACTION_CBK + opname.lower().replace('-','_')
+ actname = self.c(self.CONFIG_ACTION)
+ actcbkname = self.PTRN_ACTION_CBK + actname.lower().replace('-','_')
- cbk = getattr(self, opcbkname, None)
+ cbk = getattr(self, actcbkname, None)
if cbk:
+ self.dbgout("Executing callback '{}' for action '{}'".format(actcbkname, actname))
cbk()
else:
- raise ZenAppProcessException("Invalid action '{}', callback '{}' does not exist".format(opname, opcbkname))
+ raise ZenAppProcessException("Invalid action '{}', callback '{}' does not exist".format(actname, actcbkname))
except subprocess.CalledProcessError as err:
self.error("System command error: {}".format(err))
except ZenAppProcessException as exc:
- self.error("ZenAppProcessException: {}".format(exc))
+ self.error("Action exception: {}".format(exc))
except ZenAppException as exc:
- self.error("ZenAppException: {}".format(exc))
+ self.error("Application exception: {}".format(exc))
self.time_mark('stage_action_stop', 'End of the action stage')
- def stage_process(self):
+ def _stage_process(self):
"""
- Script lifecycle stage: PROCESSING
+ **STAGE:** *PROCESS*.
- Perform some real work (finally). Following method will call appropriate
- callback method operation to service the selected operation.
+ Finally perform some real work. This method will call :py:func:`_sub_stage_process`
+ hook, which must be implemented in subclass.
"""
- #self.time_mark('stage_process_start', 'Start of the processing stage')
+ self.time_mark('stage_process_start', 'Start of the processing stage')
- raise Exception("stage_process() method must be implemented in subclass")
+ try:
+ self._sub_stage_process()
- #self.time_mark('stage_process_stop', 'End of the processing stage')
+ except subprocess.CalledProcessError as err:
+ self.error("System command error: {}".format(err))
+
+ except ZenAppProcessException as exc:
+ self.error("Processing exception: {}".format(exc))
- def stage_evaluate(self):
+ except ZenAppException as exc:
+ self.error("Application exception: {}".format(exc))
+
+ self.time_mark('stage_process_stop', 'End of the processing stage')
+
+ def _stage_evaluate(self):
"""
- Script lifecycle stage: EVALUATE
+ **STAGE:** *EVALUATE*.
- Perform script runlog evaluation.
+ Perform application runlog postprocessing evaluation.
"""
self.time_mark('stage_evaluate_start', 'Start of the evaluation stage')
@@ -902,23 +1251,25 @@ class BaseApp:
pass
except ZenAppEvaluateException as exc:
- self.error("ZenAppEvaluateException: {}".format(exc))
+ self.error("Evaluation exception: {}".format(exc))
+
+ except ZenAppException as exc:
+ self.error("Application exception: {}".format(exc))
self.time_mark('stage_evaluate_stop', 'End of the evaluation stage')
- def stage_teardown(self):
+ def _stage_teardown(self):
"""
- Script lifecycle stage: TEARDOWN
+ **STAGE:** *TEARDOWN*
Main teardown routine. This method will call the sequence of all configured
teardown routines.
"""
try:
- # Perform custom teardown operations.
- self._stage_teardown_custom()
+ # Perform custom teardown actions.
+ self._sub_stage_teardown()
- # Teardown persistent state, if the appropriate feature is enabled and
- # also we are running in regular mode.
+ # Teardown persistent state.
if self.c(self.CONFIG_PSTATE_FILE):
self._stage_teardown_pstate()
@@ -927,102 +1278,109 @@ class BaseApp:
self._stage_teardown_runlog()
except ZenAppTeardownException as exc:
- self.error("ZenAppTeardownException: {}".format(exc))
+ self.error("Teardown exception: {}".format(exc))
+
+ except ZenAppException as exc:
+ self.error("Application exception: {}".format(exc))
+
#---------------------------------------------------------------------------
- # MAIN RUN METHODS
+ # APPLICATION MODE METHODS (MAIN RUN METHODS).
#---------------------------------------------------------------------------
+
def run(self):
"""
- Standalone script mode - Main processing method.
+ **APPLICATION MODE:** *Standalone application mode* - Main processing method.
- Run as standalone script, performs all stages of script object life cycle:
+ Run as standalone application, performs all stages of object life cycle:
1. setup stage
- 2.1 action processing stage
- 2.2.1 script processing stage
- 2.2.2 script evaluation stage
- 2.2.3 script teardown stage
+ 2.1 action stage
+ 2.2.1 processing stage
+ 2.2.2 evaluation stage
+ 2.2.3 teardown stage
"""
- self.stage_setup()
+ self._stage_setup()
if self.c(self.CONFIG_ACTION):
- self.stage_action()
+ self._stage_action()
else:
- self.stage_process()
- self.stage_evaluate()
- self.stage_teardown()
+ self._stage_process()
+ self._stage_evaluate()
+ self._stage_teardown()
- self.dbgout("[STATUS] Exiting with return code '{}'".format(self.rc))
- sys.exit(self.rc)
+ self.dbgout("Exiting with return code '{}'".format(self.retc))
+ sys.exit(self.retc)
def plugin(self):
"""
- Plugin mode - Main processing method.
+ **APPLICATION MODE:** *Plugin mode* - Main processing method.
This method allows the object to be used as plugin within larger framework.
Only the necessary setup is performed.
"""
- self.stage_setup()
+ self._stage_setup()
+
#---------------------------------------------------------------------------
- # BUILT-IN ACTION CALLBACK METHODS
+ # BUILT-IN ACTION CALLBACK METHODS.
#---------------------------------------------------------------------------
+
def cbk_action_config_view(self):
"""
- Parse and view script configurations.
+ **ACTION:** Parse and view application configurations.
"""
- print("Script configurations:")
+ self.p("Script configurations:")
tree = pydgets.widgets.TreeWidget()
- tree.display(self.config)
+ self.p(tree.render(self.config))
def cbk_action_runlog_dump(self):
"""
- Dump given script runlog.
+ **ACTION:** Dump given application runlog.
"""
rld = self.c(self.CONFIG_RUNLOG_DIR)
input_file = self.c(self.CONFIG_INPUT, False)
if not input_file:
rlfn = os.path.join(rld, '*.runlog')
runlog_files = sorted(glob.glob(rlfn), reverse = True)
- if len(runlog_files):
+ if runlog_files:
input_file = runlog_files.pop(0)
else:
- print("There are no runlog files")
+ self.p("There are no runlog files")
return
- print("Viewing script runlog '{}':".format(input_file))
+ self.p("Viewing application runlog '{}':".format(input_file))
runlog = self.json_load(input_file)
- print("")
+ self.p("")
tree = pydgets.widgets.TreeWidget()
- tree.display(runlog)
+ self.p(tree.render(runlog))
def cbk_action_runlog_view(self):
"""
- View details of given script runlog.
+ **ACTION:** View details of given application runlog.
"""
rld = self.c(self.CONFIG_RUNLOG_DIR)
input_file = self.c(self.CONFIG_INPUT, False)
if not input_file:
rlfn = os.path.join(rld, '*.runlog')
runlog_files = sorted(glob.glob(rlfn), reverse = True)
- if len(runlog_files):
+ if runlog_files:
input_file = runlog_files.pop(0)
else:
- print("There are no runlog files")
+ self.p("There are no runlog files")
return
- print("Viewing script runlog '{}':".format(input_file))
+ self.p("Viewing application runlog '{}':".format(input_file))
runlog = self.json_load(input_file)
- print("")
+ self.p("")
analysis = self.runlog_analyze(runlog)
self.runlog_format_analysis(analysis)
def cbk_action_runlogs_list(self):
"""
- View the list of all available script runlogs.
+ **ACTION:** View list of all available application runlogs.
"""
rld = self.c(self.CONFIG_RUNLOG_DIR)
limit = self.c(self.CONFIG_LIMIT)
@@ -1032,18 +1390,18 @@ class BaseApp:
for rlf in runlog_files:
runlogtree[rld].append(rlf)
- print("Listing script runlogs in directory '{}':".format(rld))
- print(" Runlog(s) found: {:,d}".format(rlcount))
+ self.p("Listing application runlogs in directory '{}':".format(rld))
+ self.p(" Runlog(s) found: {:,d}".format(rlcount))
if limit:
- print(" Result limit: {:,d}".format(limit))
- if len(runlogtree[rld]):
- print("")
+ self.p(" Result limit: {:,d}".format(limit))
+ if runlogtree[rld]:
+ self.p("")
tree = pydgets.widgets.TreeWidget()
- tree.display(runlogtree)
+ self.p(tree.render(runlogtree))
def cbk_action_runlogs_dump(self):
"""
- View the list of all available script runlogs.
+ **ACTION:** View list of all available application runlogs.
"""
rld = self.c(self.CONFIG_RUNLOG_DIR)
limit = self.c(self.CONFIG_LIMIT)
@@ -1052,20 +1410,20 @@ class BaseApp:
for rlf in runlog_files:
runlogs.append((rlf, self.json_load(rlf)))
- print("Dumping script runlog(s) in directory '{}':".format(rld))
- print(" Runlog(s) found: {:,d}".format(rlcount))
+ self.p("Dumping application runlog(s) in directory '{}':".format(rld))
+ self.p(" Runlog(s) found: {:,d}".format(rlcount))
if limit:
- print(" Result limit: {:,d}".format(limit))
- if len(runlogs):
- print("")
+ self.p(" Result limit: {:,d}".format(limit))
+ if runlogs:
+ self.p("")
tree = pydgets.widgets.TreeWidget()
- for rl in runlogs:
- print("Runlog '{}':".format(rl[0]))
- tree.display(rl[1])
+ for runl in runlogs:
+ self.p("Runlog '{}':".format(runl[0]))
+ self.p(tree.render(runl[1]))
def cbk_action_runlogs_evaluate(self):
"""
- Evaluate previous script runlogs.
+ **ACTION:** Evaluate previous application runlogs.
"""
rld = self.c(self.CONFIG_RUNLOG_DIR)
limit = self.c(self.CONFIG_LIMIT)
@@ -1074,53 +1432,55 @@ class BaseApp:
for rlf in runlog_files:
runlogs.append(self.json_load(rlf))
- print("Evaluating script runlogs in directory '{}':".format(rld))
- print(" Runlog(s) found: {:,d}".format(rlcount))
+ self.p("Evaluating application runlogs in directory '{}':".format(rld))
+ self.p(" Runlog(s) found: {:,d}".format(rlcount))
if limit:
- print(" Result limit: {:,d}".format(limit))
- if len(runlogs):
- print("")
+ self.p(" Result limit: {:,d}".format(limit))
+ if runlogs:
+ self.p("")
evaluation = self.runlogs_evaluate(runlogs)
self.runlogs_format_evaluation(evaluation)
+
#---------------------------------------------------------------------------
# ACTION HELPERS
#---------------------------------------------------------------------------
+
def runlog_analyze(self, runlog):
"""
Analyze given runlog.
"""
- ct = int(time.time())
+ curt = int(time.time())
tm_tmp = {}
analysis = {self.RLANKEY_DURPRE: 0, self.RLANKEY_DURPROC: 0, self.RLANKEY_DURPOST: 0, self.RLANKEY_DURATIONS: {}}
analysis[self.RLANKEY_RUNLOG] = runlog
analysis[self.RLANKEY_LABEL] = runlog[self.RLKEY_TSSTR]
- analysis[self.RLANKEY_AGE] = ct - runlog[self.RLKEY_TS]
+ analysis[self.RLANKEY_AGE] = curt - runlog[self.RLKEY_TS]
analysis[self.RLANKEY_RESULT] = runlog[self.RLKEY_RESULT]
analysis[self.RLANKEY_COMMAND] = runlog.get(self.RLANKEY_COMMAND, runlog.get('operation', 'unknown'))
- # Calculate script processing duration
+ # Calculate application processing duration
analysis[self.RLANKEY_DURRUN] = runlog[self.RLKEY_TMARKS][-1]['time'] - runlog[self.RLKEY_TMARKS][0]['time']
# Calculate separate durations for all stages
- for tm in runlog[self.RLKEY_TMARKS]:
+ for tmark in runlog[self.RLKEY_TMARKS]:
ptrna = re.compile('^(.*)_start$')
ptrnb = re.compile('^(.*)_stop$')
- m = ptrna.match(tm['ident'])
- if m:
- mg = m.group(1)
- tm_tmp[mg] = tm['time']
+ match = ptrna.match(tmark['ident'])
+ if match:
+ matchg = match.group(1)
+ tm_tmp[matchg] = tmark['time']
continue
- m = ptrnb.match(tm['ident'])
- if m:
- mg = m.group(1)
- analysis[self.RLANKEY_DURATIONS][mg] = tm['time'] - tm_tmp[mg]
- if mg in ('stage_configure', 'stage_check', 'stage_setup'):
- analysis[self.RLANKEY_DURPRE] += analysis[self.RLANKEY_DURATIONS][mg]
- elif mg in ('stage_process'):
- analysis[self.RLANKEY_DURPROC] += analysis[self.RLANKEY_DURATIONS][mg]
- elif mg in ('stage_evaluate', 'stage_teardown'):
- analysis[self.RLANKEY_DURPOST] += analysis[self.RLANKEY_DURATIONS][mg]
+ match = ptrnb.match(tmark['ident'])
+ if match:
+ matchg = match.group(1)
+ analysis[self.RLANKEY_DURATIONS][matchg] = tmark['time'] - tm_tmp[matchg]
+ if matchg in ('stage_configure', 'stage_check', 'stage_setup'):
+ analysis[self.RLANKEY_DURPRE] += analysis[self.RLANKEY_DURATIONS][matchg]
+ elif matchg in ('stage_process'):
+ analysis[self.RLANKEY_DURPROC] += analysis[self.RLANKEY_DURATIONS][matchg]
+ elif matchg in ('stage_evaluate', 'stage_teardown'):
+ analysis[self.RLANKEY_DURPOST] += analysis[self.RLANKEY_DURATIONS][matchg]
continue
analysis[self.RLANKEY_EFFECTIVITY] = ((analysis[self.RLANKEY_DURPROC]/analysis[self.RLANKEY_DURRUN])*100)
@@ -1136,12 +1496,12 @@ class BaseApp:
{ 'label': 'Value', 'data_formating': '{:s}', 'align': '>' },
]
tbody = [
- ['Label:', analysis[self.RLANKEY_LABEL]],
- ['Age:', str(datetime.timedelta(seconds=int(analysis[self.RLANKEY_AGE])))],
- ['Command:', analysis[self.RLANKEY_COMMAND]],
- ['Result:', analysis[self.RLANKEY_RESULT]],
- ]
- tablew.display(tbody, columns = tcols, enumerate = False, header = False)
+ ['Label:', analysis[self.RLANKEY_LABEL]],
+ ['Age:', str(datetime.timedelta(seconds=int(analysis[self.RLANKEY_AGE])))],
+ ['Command:', analysis[self.RLANKEY_COMMAND]],
+ ['Result:', analysis[self.RLANKEY_RESULT]],
+ ]
+ self.p(tablew.render(tbody, columns = tcols, enumerate = False, header = False))
#treew = pydgets.widgets.TreeWidget()
#treew.display(analysis)
@@ -1153,8 +1513,8 @@ class BaseApp:
Evaluate given runlogs.
"""
evaluation = {self.RLEVKEY_ANALYSES: []}
- for rl in runlogs:
- rslt = self.runlog_analyze(rl)
+ for runl in runlogs:
+ rslt = self.runlog_analyze(runl)
evaluation[self.RLEVKEY_ANALYSES].append(rslt)
return self._sub_runlogs_evaluate(runlogs, evaluation)
@@ -1163,122 +1523,287 @@ class BaseApp:
Format runlog evaluation.
"""
table_columns = [
- { 'label': 'Date' },
- { 'label': 'Age', 'data_formating': '{}', 'align': '>' },
- { 'label': 'Runtime', 'data_formating': '{}', 'align': '>' },
- { 'label': 'Process', 'data_formating': '{}', 'align': '>' },
- { 'label': 'E [%]', 'data_formating': '{:6.2f}', 'align': '>' },
- { 'label': 'Errors', 'data_formating': '{:,d}', 'align': '>' },
- { 'label': 'Command', 'data_formating': '{}', 'align': '>' },
- { 'label': 'Result', 'data_formating': '{}', 'align': '>' },
- ]
+ { 'label': 'Date' },
+ { 'label': 'Age', 'data_formating': '{}', 'align': '>' },
+ { 'label': 'Runtime', 'data_formating': '{}', 'align': '>' },
+ { 'label': 'Process', 'data_formating': '{}', 'align': '>' },
+ { 'label': 'E [%]', 'data_formating': '{:6.2f}', 'align': '>' },
+ { 'label': 'Errors', 'data_formating': '{:,d}', 'align': '>' },
+ { 'label': 'Command', 'data_formating': '{}', 'align': '>' },
+ { 'label': 'Result', 'data_formating': '{}', 'align': '>' },
+ ]
table_data = []
- for an in evaluation[self.RLEVKEY_ANALYSES]:
+ for anl in evaluation[self.RLEVKEY_ANALYSES]:
table_data.append(
[
- an[self.RLANKEY_LABEL],
- str(datetime.timedelta(seconds=int(an[self.RLANKEY_AGE]))),
- str(datetime.timedelta(seconds=int(an[self.RLANKEY_DURRUN]))),
- str(datetime.timedelta(seconds=int(an[self.RLANKEY_DURPROC]))),
- an[self.RLANKEY_EFFECTIVITY],
- len(an[self.RLANKEY_RUNLOG][self.RLKEY_ERRORS]),
- an[self.RLANKEY_COMMAND],
- an[self.RLANKEY_RESULT],
+ anl[self.RLANKEY_LABEL],
+ str(datetime.timedelta(seconds=int(anl[self.RLANKEY_AGE]))),
+ str(datetime.timedelta(seconds=int(anl[self.RLANKEY_DURRUN]))),
+ str(datetime.timedelta(seconds=int(anl[self.RLANKEY_DURPROC]))),
+ anl[self.RLANKEY_EFFECTIVITY],
+ len(anl[self.RLANKEY_RUNLOG][self.RLKEY_ERRORS]),
+ anl[self.RLANKEY_COMMAND],
+ anl[self.RLANKEY_RESULT],
]
)
- print("General script processing statistics:")
+ self.p("General application processing statistics:")
tablew = pydgets.widgets.TableWidget()
- tablew.display(table_data, columns = table_columns)
+ self.p(tablew.render(table_data, columns = table_columns))
self._sub_runlogs_format_evaluation(evaluation)
- def runlog_dump(self, runlog, **kwargs):
+ def runlogs_list(self, **kwargs):
+ """
+ List all available runlogs.
+ """
+ reverse = kwargs.get('reverse', False)
+ limit = kwargs.get('limit', None)
+ rlfn = os.path.join(self.c(self.CONFIG_RUNLOG_DIR), '*.runlog')
+ rllist = sorted(glob.glob(rlfn), reverse = reverse)
+ rlcount = len(rllist)
+
+ if limit:
+ return (rllist[:limit], rlcount)
+ return (rllist, rlcount)
+
+
+ #---------------------------------------------------------------------------
+ # INTERNAL UTILITIES.
+ #---------------------------------------------------------------------------
+
+
+ def _get_fn_runlog(self):
+ """
+ Return the name of the runlog file for current process.
+
+ :return: Name of the runlog file.
+ :rtype: str
+ """
+ return os.path.join(self.c(self.CONFIG_RUNLOG_DIR), "{}.runlog".format(self.runlog[self.RLKEY_TSFSF]))
+
+ def _get_fn_pstate(self):
+ """
+ Return the name of the persistent state file for current process.
+
+ :return: Name of the persistent state file.
+ :rtype: str
+ """
+ return self.c(self.CONFIG_PSTATE_FILE)
+
+ def _utils_detect_actions(self):
+ """
+ Returns the sorted list of all available actions current application is capable
+ of performing. The detection algorithm is based on string analysis of all
+ available methods. Any method starting with string ``cbk_action_`` will
+ be appended to the list, lowercased and with ``_`` characters replaced with ``-``.
+ """
+ ptrn = re.compile(self.PTRN_ACTION_CBK)
+ attrs = sorted(dir(self))
+ result = []
+ for atr in attrs:
+ if not callable(getattr(self, atr)):
+ continue
+ match = ptrn.match(atr)
+ if match:
+ result.append(atr.replace(self.PTRN_ACTION_CBK,'').replace('_','-').lower())
+ return result
+
+ def _utils_runlog_dump(self, runlog):
+ """
+ Write application runlog into ``stdout``.
+
+ :param dict runlog: Structure containing application runlog.
+ """
+ self.p("Application runlog >>>\n{}".format(self.json_dump(runlog)))
+
+ def _utils_runlog_log(self, runlog):
"""
- Dump runlog.
+ Write application runlog into logging service.
- Dump script runlog to terminal (JSON).
+ :param dict runlog: Structure containing application runlog.
"""
- # Dump current script runlog.
- #self.logger.debug("Script runlog >>>\n{}".format(json.dumps(runlog, sort_keys=True, indent=4)))
- print("Script runlog >>>\n{}".format(self.json_dump(runlog, default=_json_default)))
+ self.p("Application runlog >>>\n{}".format(self.json_dump(runlog)))
- def runlog_save(self, runlog, **kwargs):
+ def _utils_runlog_save(self, runlog):
"""
- Save runlog.
+ Write application runlog to external JSON file.
- Save script runlog to external file (JSON).
+ :param dict runlog: Structure containing application runlog.
"""
- # Attempt to create script runlog directory.
+ # Attempt to create application runlog directory.
if not os.path.isdir(self.c(self.CONFIG_RUNLOG_DIR)):
- self.logger.info("Creating runlog directory '{}'".format(self.c(self.CONFIG_RUNLOG_DIR)))
+ self.logger.info("Creating application runlog directory '%s'", self.c(self.CONFIG_RUNLOG_DIR))
os.makedirs(self.c(self.CONFIG_RUNLOG_DIR))
- rlfn = self.get_fn_runlog()
- self.dbgout("[STATUS] Saving script runlog to file '{}'".format(rlfn))
+
+ rlfn = self._get_fn_runlog()
+ self.dbgout("Saving application runlog to file '{}'".format(rlfn))
self.json_save(rlfn, runlog)
- self.logger.info("Script runlog saved to file '{}'".format(rlfn))
+ self.logger.info("Application runlog saved to file '%s'", rlfn)
- def runlogs_list(self, **kwargs):
+ def _utils_pstate_dump(self, state):
"""
- List all available runlogs.
+ Write persistent state into ``stdout``.
+
+ :param dict state: Structure containing application persistent state.
"""
- reverse = kwargs.get('reverse', False)
- limit = kwargs.get('limit', None)
- rlfn = os.path.join(self.c(self.CONFIG_RUNLOG_DIR), '*.runlog')
- rllist = sorted(glob.glob(rlfn), reverse = reverse)
- rlcount = len(rllist)
- if limit:
- return (rllist[:limit], rlcount)
- else:
- return (rllist, rlcount)
+ self.p("Application persistent state >>>\n{}".format(self.json_dump(state)))
- def pstate_dump(self, state, **kwargs):
+ def _utils_pstate_log(self, state):
"""
- Dump persistent state.
+ Write persistent state into logging service.
- Dump script persistent state to terminal (JSON).
+ :param dict state: Structure containing application persistent state.
"""
- # Dump current script state.
- #self.logger.debug("Script state >>>\n{}".format(json.dumps(state, sort_keys=True, indent=4)))
- print("Script state >>>\n{}".format(self.json_dump(state, default=_json_default)))
+ self.logger.info("Application persistent state >>>\n%s", self.json_dump(state))
- def pstate_save(self, state, **kwargs):
+ def _utils_pstate_save(self, state):
"""
- Save persistent state.
+ Write application persistent state to external JSON file.
- Save script persistent state to external file (JSON).
+ :param dict state: Structure containing application persistent state.
"""
- sfn = self.get_fn_pstate()
- self.dbgout("[STATUS] Saving script persistent state to file '{}'".format(sfn))
+ sfn = self._get_fn_pstate()
+ self.dbgout("Saving application persistent state to file '{}'".format(sfn))
self.json_save(sfn, state)
- self.logger.info("Script persistent state saved to file '{}'".format(sfn))
+ self.logger.info("Application persistent state saved to file '%s'", sfn)
+
#---------------------------------------------------------------------------
- # TOOLS
+ # SHORTCUT METHODS, HELPERS AND TOOLS.
#---------------------------------------------------------------------------
- def execute_command(self, command, can_fail=False):
+
+ def c(self, key, default = None):
"""
- Execute given shell command
+ Shortcut method: Get given configuration value, shortcut for:
+
+ self.config.get(key, default)
+
+ :param str key: Name of the configuration value.
+ :param default: Default value to be returned when key is not set.
+ :return: Configuration value fogr given key.
"""
- self.logger.info("Executing system command >>>\n{}".format(command))
- #result = subprocess.run(command)
+ if default is None:
+ return self.config.get(key)
+ return self.config.get(key, default)
+
+ def cc(self, key, default = None):
+ """
+ Shortcut method: Get given core configuration value, shortcut for:
+
+ self.config[self.CORE].get(key, default)
+
+ Core configurations are special configurations under configuration key
+ ``__CORE__``, which may only either be hardcoded, or calculated from other
+ configurations.
+
+ :param str key: Name of the core configuration value.
+ :param default: Default value to be returned when key is not set.
+ :return: Core configuration value fogr given key.
+ """
+ if default is None:
+ return self.config[self.CORE].get(key)
+ return self.config[self.CORE].get(key, default)
+
+ def p(self, string, level = 0):
+ """
+ Print given string to ``sys.stdout`` with respect to ``quiet`` and ``verbosity``
+ settings.
+
+ :param str string: String to print.
+ :param int level: Required minimal verbosity level to print the message.
+ """
+ if not self.c(self.CONFIG_QUIET) and self.c(self.CONFIG_VERBOSITY) >= level:
+ print(string)
+
+ def error(self, error, retc = None, trcb = None):
+ """
+ Register given error, that occured during application run. Registering in
+ the case of this method means printing the error message to logging facility,
+ storing the message within the appropriate runlog data structure, generating
+ the traceback when required and altering the runlog result and return code
+ attributes accordingly.
+
+ :param str error: Error message to be written.
+ :param int retc: Requested return code with which to terminate the application.
+ :param Exception trcb: Optional exception object.
+ """
+ self.retc = retc if retc is not None else self.RC_FAILURE
+
+ errstr = "{}".format(error)
+ self.logger.error(errstr)
+
+ if trcb:
+ tbexc = traceback.format_tb(trcb)
+ self.logger.error("\n" + "".join(tbexc))
+
+ self.runlog[self.RLKEY_ERRORS].append(errstr)
+ self.runlog[self.RLKEY_RESULT] = self.RESULT_FAILURE
+ self.runlog[self.RLKEY_RC] = self.retc
+
+ @staticmethod
+ def dbgout(message):
+ """
+ Routine for printing additional debug messages. The given message will be
+ printed only in case the static class variable ``FLAG_DEBUG`` flag is set
+ to ``True``. This can be done either by explicit assignment in code, or
+ using command line argument ``--debug``, which is evaluated ASAP and sets
+ the variable to ``True``. The message will be printed to ``sys.stderr``.
+
+ :param str message: Message do be written.
+ """
+ if BaseApp.FLAG_DEBUG:
+ print("* {} DBGOUT: {}".format(time.strftime('%Y-%m-%d %X', time.localtime()), message), file=sys.stderr)
+
+ @staticmethod
+ def excout(exception, retc = None):
+ """
+ Routine for displaying the exception message to the user without traceback
+ and terminating the application. This routine is intended to display information
+ about application errors, that are not caused by the application code itself
+ (like missing configuration file, non-writable directories, etc.) and that
+ can not be logged because of the fact, that the logging service was not yet
+ initialized. For that reason this method is used to handle exceptions during
+ the **__init__** and **setup** stages.
+
+ :param Exception exception: Exception object.
+ :param int retc: Requested return code with which to terminate the application.
+ """
+ retc = retc if retc is not None else BaseApp.RC_FAILURE
+
+ print("{} CRITICAL ERROR: {}".format(time.strftime('%Y-%m-%d %X', time.localtime()), exception), file=sys.stderr)
+ sys.exit(retc)
+
+ def execute_command(self, command, can_fail = False):
+ """
+ Execute given shell command.
+ """
+ self.logger.info("Executing system command >>>\n%s", command)
+
result = None
if can_fail:
- result = subprocess.call(command, shell=True)
+ result = subprocess.call(command, shell = True)
else:
- result = subprocess.check_output(command, shell=True)
- self.logger.debug("System command result >>>\n{}".format(pprint.pformat(result,indent=4)))
+ result = subprocess.check_output(command, shell = True)
+
+ self.logger.debug("System command result >>>\n%s", pprint.pformat(result, indent=4))
return result
def time_mark(self, ident, descr):
"""
- Mark current time with additional identifiers and descriptions
+ Mark current time with additional identifier and description to application
+ runlog.
+
+ :param str ident: Time mark identifier.
+ :param str descr: Time mark description.
+ :return: Time mark data structure.
+ :rtype: dict
"""
mark = {
- 'ident': ident,
- 'descr': descr,
- 'time': time.time()
- }
+ 'ident': ident,
+ 'descr': descr,
+ 'time': time.time()
+ }
self.runlog[self.RLKEY_TMARKS].append(mark)
return mark
@@ -1286,6 +1811,11 @@ class BaseApp:
def json_dump(data, **kwargs):
"""
Dump given data structure into JSON string.
+
+ :param dict data: Data to be dumped to JSON.
+ :param kwargs: Optional arguments to pass to :py:func:`pyzenkit.jsonconf.json_dump` method.
+ :return: Data structure as JSON string.
+ :rtype: str
"""
return pyzenkit.jsonconf.json_dump(data, **kwargs)
@@ -1293,78 +1823,125 @@ class BaseApp:
def json_save(json_file, data, **kwargs):
"""
Save given data structure into given JSON file.
+
+ :param str json_file: Name of the JSON file to write to.
+ :param dict data: Data to be dumped to JSON.
+ :param kwargs: Optional arguments to pass to :py:func:`pyzenkit.jsonconf.json_save` method.
+ :return: Always returns ``True``.
+ :rtype: bool
"""
return pyzenkit.jsonconf.json_save(json_file, data, **kwargs)
@staticmethod
- def json_load(json_file, **kwargs):
+ def json_load(json_file):
"""
- Load data structure from given json file.
+ Load data structure from given JSON file.
+
+ :param str json_file: Name of the JSON file to read from.
+ :return: Loaded data structure.
+ :rtype: dict
"""
- return pyzenkit.jsonconf.json_load(json_file, **kwargs)
+ return pyzenkit.jsonconf.json_load(json_file)
- def format_progress_bar(self, percent, done, barLen = 50):
+ @staticmethod
+ def format_progress_bar(percent, bar_len = 50):
"""
- Format progress bar from given values
+ Format progress bar from given values.
"""
progress = ""
- for i in range(barLen):
- if i < int(barLen * percent):
+ for i in range(bar_len):
+ if i < int(bar_len * percent):
progress += "="
else:
progress += " "
return " [%s] %.2f%%" % (progress, percent * 100)
- def draw_progress_bar(self, percent, done, barLen = 50):
+ @staticmethod
+ def draw_progress_bar(percent, bar_len = 50):
"""
- Draw progress bar on standard output terminal
+ Draw progress bar on standard output terminal.
"""
sys.stdout.write("\r")
- sys.stdout.write(self.format_progress_bar(percent, done, barLen))
+ sys.stdout.write(BaseApp.format_progress_bar(percent, bar_len))
sys.stdout.flush()
-class _DemoBaseApp(BaseApp):
+
+#-------------------------------------------------------------------------------
+
+
+class DemoBaseApp(BaseApp):
"""
- Minimalistic class for demonstration purposes.
+ Minimalistic class for demonstration purposes. Study implementation of this
+ class for tutorial on how to use this framework.
"""
- def stage_process(self):
+ def __init__(self, name = None, description = None):
"""
- Script lifecycle stage: PROCESSING
+ Initialize demonstration application. This method overrides the base
+ implementation in :py:func:`baseapp.BaseApp.__init__` and it aims to
+ even more simplify the application object creation.
- Perform some real work (finally). Following method will call appropriate
- callback method operation to service the selected operation.
+ :param str name: Optional application name.
+ :param str description: Optional application description.
"""
- self.time_mark('stage_process_start', 'Start of the processing stage')
+ name = 'demo-baseapp.py' if not name else name
+ description = 'DemoBaseApp - Demonstration application' if not description else description
- # Log something to show we have reached this point of execution.
- self.logger.info("Demo implementation for default command")
+ super().__init__(
+ name = name,
+ description = description,
- # Test direct console output with conjunction with verbosity
- self.p("Hello world")
- self.p("Hello world, verbosity level 1", 1)
- self.p("Hello world, verbosity level 2", 2)
- self.p("Hello world, verbosity level 3", 3)
+ #
+ # Configure required application paths to harmless locations.
+ #
+ path_bin = '/tmp',
+ path_cfg = '/tmp',
+ path_log = '/tmp',
+ path_tmp = '/tmp',
+ path_run = '/tmp'
+ )
+ def _sub_stage_process(self):
+ """
+ Script lifecycle stage **PROCESS**.
+ """
# Update the persistent state to view the changes.
self.pstate['counter'] = self.pstate.get('counter', 0) + 1
- self.time_mark('stage_process_stop', 'End of the processing stage')
+ # Log something to show we have reached this point of execution.
+ self.logger.info("Demonstration implementation for BaseApp demo application")
+ self.logger.info("Try executing this demo with following parameters:")
+ self.logger.info("* python3 pyzenkit/baseapp.py --help")
+ self.logger.info("* python3 pyzenkit/baseapp.py --verbose")
+ self.logger.info("* python3 pyzenkit/baseapp.py --verbose --verbose")
+ self.logger.info("* python3 pyzenkit/baseapp.py --verbose --verbose --verbose")
+ self.logger.info("* python3 pyzenkit/baseapp.py --debug")
+ self.logger.info("* python3 pyzenkit/baseapp.py --log-level debug")
+ self.logger.info("* python3 pyzenkit/baseapp.py --pstate-dump")
+ self.logger.info("* python3 pyzenkit/baseapp.py --runlog-dump")
+ self.logger.info("Number of BaseApp runs from persistent state: '%d'", self.pstate.get('counter'))
+
+ # Test direct console output with conjunction with verbosity levels.
+ self.p("Hello world from BaseApp")
+ self.p("Hello world from BaseApp, verbosity level 1", 1)
+ self.p("Hello world from BaseApp, verbosity level 2", 2)
+ self.p("Hello world from BaseApp, verbosity level 3", 3)
+
+#-------------------------------------------------------------------------------
+
+#
+# Perform the demonstration.
+#
if __name__ == "__main__":
- """
- Perform the demonstration.
- """
- # Prepare the environment
- if not os.path.isdir('/tmp/baseapp.py'):
- os.mkdir('/tmp/baseapp.py')
- BaseApp.json_save('/tmp/baseapp.py.conf', {'test_a':1})
- script = _DemoBaseApp(
- path_cfg = '/tmp',
- path_log = '/tmp',
- path_tmp = '/tmp',
- path_run = '/tmp',
- description = 'DemoBaseApp - generic base script (DEMO)'
- )
- script.run()
+ # Prepare demonstration environment.
+ APP_NAME = 'demo-baseapp.py'
+ BaseApp.json_save('/tmp/{}.conf'.format(APP_NAME), {'test_a':1})
+ try:
+ os.mkdir('/tmp/{}'.format(APP_NAME))
+ except FileExistsError:
+ pass
+
+ BASE_APP = DemoBaseApp(APP_NAME)
+ BASE_APP.run()
diff --git a/pyzenkit/daemonizer.py b/pyzenkit/daemonizer.py
index 2d4dbec9ea13eac99996ca03d50d4516667ee940..6e78a5d2da657547a4ebc2027313214bf6637729 100644
--- a/pyzenkit/daemonizer.py
+++ b/pyzenkit/daemonizer.py
@@ -1,9 +1,14 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Pavel Kacha <ph@rook.cz>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
diff --git a/pyzenkit/jsonconf.py b/pyzenkit/jsonconf.py
index 819aed06c4eb7dd990206941c0fe068eecbc8357..98c2a333ae17b554871785fae3e66ad1b1864038 100644
--- a/pyzenkit/jsonconf.py
+++ b/pyzenkit/jsonconf.py
@@ -1,20 +1,25 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
"""
-This module provides following tools for manipulation with JSON configuration
-files:
+This module provides tools for manipulating JSON configuration files:
* Simple writing of formated JSON configuration files
* Simple reading of any JSON configuration files
-* Reading and merging of multiple JSON configuration files/directories
-* Support for single line comments in JSON files (``#``,``//``)
-* JSON schema validation
+* Merging multiple JSON configuration files or configuration directories
+* Support for single line comments in JSON files (``#``, ``//``)
+* Support for semi-automated JSON schema validation
"""
@@ -232,7 +237,7 @@ def config_load_n(config_files, schema = None):
.. warning::
- The merge is done using :py:func:``dict.update`` method and occurs only
+ The merge is done using :py:func:`dict.update` method and occurs only
at highest level.
:param str config_files: List of names of the source JSON config files to be loaded.
@@ -264,7 +269,7 @@ def config_load_dir(config_dir, schema = None, extension = '.json.conf'):
.. warning::
- The merge is done using :py:func:``dict.update`` method and occurs only
+ The merge is done using :py:func:`dict.update` method and occurs only
at highest level.
:param str config_dir: Names of the configuration directory.
diff --git a/pyzenkit/tests/__init__.py b/pyzenkit/tests/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..df1f5e250196009111232bd65e0c33cd5465df25 100644
--- a/pyzenkit/tests/__init__.py
+++ b/pyzenkit/tests/__init__.py
@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
+#-------------------------------------------------------------------------------
diff --git a/pyzenkit/tests/test_baseapp.py b/pyzenkit/tests/test_baseapp.py
index d982b4f21dd4d7ba83be23f786630691231cb5c3..2cec19d1f9e7c1e7b31f61037d64bda1c60849dd 100644
--- a/pyzenkit/tests/test_baseapp.py
+++ b/pyzenkit/tests/test_baseapp.py
@@ -1,10 +1,17 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
+
import unittest
from unittest.mock import Mock, MagicMock, call
from pprint import pformat, pprint
@@ -13,21 +20,24 @@ import os
import sys
import shutil
+
# Generate the path to custom 'lib' directory
lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))
sys.path.insert(0, lib)
import pyzenkit.baseapp
+
#
# Global variables
#
-SCR_NAME = 'test_baseapp.py' # Name of the script process
+APP_NAME = 'test-baseapp.py' # Name of the application process
JSON_FILE_NAME = '/tmp/script-state.json' # Name of the test JSON file
-CFG_FILE_NAME = '/tmp/{}.conf'.format(SCR_NAME) # Name of the script configuration file
-CFG_DIR_NAME = '/tmp/{}'.format(SCR_NAME) # Name of the script configuration directory
+CFG_FILE_NAME = '/tmp/{}.conf'.format(APP_NAME) # Name of the application configuration file
+CFG_DIR_NAME = '/tmp/{}'.format(APP_NAME) # Name of the application configuration directory
+
-class TestPyzenkitScript(unittest.TestCase):
+class TestPyzenkitBaseApp(unittest.TestCase):
def setUp(self):
pyzenkit.baseapp.BaseApp.json_save(CFG_FILE_NAME, {'test': 'x'})
@@ -36,13 +46,9 @@ class TestPyzenkitScript(unittest.TestCase):
except FileExistsError:
pass
- self.obj = pyzenkit.baseapp._DemoBaseApp(
- name = SCR_NAME,
- path_cfg = '/tmp',
- path_log = '/tmp',
- path_tmp = '/tmp',
- path_run = '/tmp',
- description = 'DemoBaseApp - generic base script (DEMO)'
+ self.obj = pyzenkit.baseapp.DemoBaseApp(
+ name = APP_NAME,
+ description = 'TestBaseApp - Testing application'
)
def tearDown(self):
os.remove(CFG_FILE_NAME)
@@ -50,29 +56,61 @@ class TestPyzenkitScript(unittest.TestCase):
def test_01_utils(self):
"""
- Perform tests of generic script utils.
+ Perform tests of generic application utils.
"""
- # Test the saving of JSON files
- self.assertEqual(self.obj.name, SCR_NAME)
+ self.maxDiff = None
- # Test the saving of JSON files
+ # Test the name generation capabilities.
+ self.assertEqual(self.obj.name, APP_NAME)
+
+ # Test the saving of JSON files.
self.assertTrue(self.obj.json_save(JSON_FILE_NAME, { "test": 1 }))
- # Test that the JSON file was really created
+ # Test that the JSON file was really created.
self.assertTrue(os.path.isfile(JSON_FILE_NAME))
- # Test the loading of JSON files
+ # Test the loading of JSON files.
self.assertEqual(self.obj.json_load(JSON_FILE_NAME), { "test": 1 })
- # Remove the JSON file we are done with
+ # Remove the JSON file we are done with.
os.remove(JSON_FILE_NAME)
- def test_02_basic(self):
+ def test_02_argument_parsing(self):
+ """
+ Perform tests of argument parsing.
+ """
+ self.maxDiff = None
+
+ # Test argument parsing.
+ argp = self.obj._init_argparser()
+ self.assertEqual(vars(argp.parse_args(['--verbose'])), {'action': None,
+ 'config_dir': None,
+ 'config_file': None,
+ 'debug': None,
+ 'group': None,
+ 'input': None,
+ 'limit': None,
+ 'log_file': None,
+ 'log_level': None,
+ 'name': None,
+ 'pstate_dump': None,
+ 'pstate_file': None,
+ 'pstate_log': None,
+ 'quiet': None,
+ 'runlog_dir': None,
+ 'runlog_dump': None,
+ 'runlog_log': None,
+ 'user': None,
+ 'verbosity': 1
+ })
+
+ def test_03_plugin(self):
"""
- Perform the basic operativity tests.
+ Perform tests of plugin mode.
"""
+ self.maxDiff = None
+
self.obj.plugin()
if __name__ == "__main__":
unittest.main()
-
diff --git a/pyzenkit/tests/test_daemonizer.py b/pyzenkit/tests/test_daemonizer.py
index 604a8d1a937897048600a098ceb9aa92db30a8f2..b8e12b3ead1756712bac3e6a80c065461934bff9 100644
--- a/pyzenkit/tests/test_daemonizer.py
+++ b/pyzenkit/tests/test_daemonizer.py
@@ -1,12 +1,16 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Pavel Kacha <ph@rook.cz>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
-
import unittest
from unittest.mock import Mock, MagicMock, call
from pprint import pformat, pprint
diff --git a/pyzenkit/tests/test_jsonconf.py b/pyzenkit/tests/test_jsonconf.py
index e5759a7b93bd2fa7fce54e67c392b372b174804c..39eee7c735352c50bf61c64524b7feaf34da7eb9 100644
--- a/pyzenkit/tests/test_jsonconf.py
+++ b/pyzenkit/tests/test_jsonconf.py
@@ -1,8 +1,14 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
import unittest
@@ -188,4 +194,3 @@ class TestPyzenkitJsonconf(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
-
diff --git a/pyzenkit/tests/test_zendaemon.py b/pyzenkit/tests/test_zendaemon.py
index 90de2751c09274c3446e16f5dc26da152f8a7a1f..938774a6f19d50f662e6304edb6a0da10d2061e0 100644
--- a/pyzenkit/tests/test_zendaemon.py
+++ b/pyzenkit/tests/test_zendaemon.py
@@ -1,8 +1,14 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
import unittest
diff --git a/pyzenkit/tests/test_zenscript.py b/pyzenkit/tests/test_zenscript.py
index 582538b1b46979f00e739054b4aa0e00146f576b..13349a419797fbb06c88855de6b25ea96cdf39de 100644
--- a/pyzenkit/tests/test_zenscript.py
+++ b/pyzenkit/tests/test_zenscript.py
@@ -1,10 +1,17 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
+
import unittest
from unittest.mock import Mock, MagicMock, call
from pprint import pformat, pprint
@@ -12,6 +19,8 @@ from pprint import pformat, pprint
import os
import sys
import shutil
+import datetime
+
# Generate the path to custom 'lib' directory
lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))
@@ -20,13 +29,14 @@ sys.path.insert(0, lib)
import pyzenkit.baseapp
import pyzenkit.zenscript
+
#
# Global variables
#
-SCR_NAME = 'test_zenscript.py' # Name of the script process
-JSON_FILE_NAME = '/tmp/daemon-state.json' # Name of the test JSON file
-CFG_FILE_NAME = '/tmp/{}.conf'.format(SCR_NAME) # Name of the script configuration file
-CFG_DIR_NAME = '/tmp/{}'.format(SCR_NAME) # Name of the script configuration directory
+SCR_NAME = 'test-zenscript.py' # Name of the script process
+CFG_FILE_NAME = '/tmp/{}.conf'.format(SCR_NAME) # Name of the script configuration file
+CFG_DIR_NAME = '/tmp/{}'.format(SCR_NAME) # Name of the script configuration directory
+
class TestPyzenkitZenScript(unittest.TestCase):
@@ -37,13 +47,9 @@ class TestPyzenkitZenScript(unittest.TestCase):
except FileExistsError:
pass
- self.obj = pyzenkit.zenscript._DemoZenScript(
- name = SCR_NAME,
- path_cfg = '/tmp',
- path_log = '/tmp',
- path_tmp = '/tmp',
- path_run = '/tmp',
- description = 'DemoZenScript - generic script (DEMO)'
+ self.obj = pyzenkit.zenscript.DemoZenScript(
+ name = SCR_NAME,
+ description = 'TestZenScript - Testing script'
)
def tearDown(self):
os.remove(CFG_FILE_NAME)
@@ -53,16 +59,21 @@ class TestPyzenkitZenScript(unittest.TestCase):
"""
Perform the basic utility tests.
"""
+ self.maxDiff = None
+
+ self.obj.plugin()
# Test the interval threshold calculations
- self.assertEqual(self.obj.calculate_interval_thresholds('daily', time_cur = 1454934631), (1454848231, 1454934631))
- self.assertEqual(self.obj.calculate_interval_thresholds('daily', time_cur = 1454934631, flag_floor = True), (1454803200, 1454889600))
+ self.assertEqual(self.obj.calculate_interval_thresholds(time_high = 1454934631, interval = 'daily'), (datetime.datetime(2016, 2, 7, 13, 30, 31), datetime.datetime(2016, 2, 8, 13, 30, 31)))
+ self.assertEqual(self.obj.calculate_interval_thresholds(time_high = 1454934631, interval = 'daily', adjust = True), (datetime.datetime(2016, 2, 7, 1, 0), datetime.datetime(2016, 2, 8, 1, 0)))
- def test_02_basic(self):
+ def test_02_plugin(self):
"""
Perform the basic operativity tests.
"""
- self.assertTrue(True)
+ self.maxDiff = None
+
+ self.obj.plugin()
if __name__ == "__main__":
unittest.main()
diff --git a/pyzenkit/zencli.py b/pyzenkit/zencli.py
index c72cce05fd4e4373193c6dbd187fe0945096575b..103ff7018dfa8ddd55609e04b7fff815f490e518 100644
--- a/pyzenkit/zencli.py
+++ b/pyzenkit/zencli.py
@@ -1,9 +1,15 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
+#-------------------------------------------------------------------------------
# Notes:
# - The concept for dynamic module loading was taken from here
# [1] https://lextoumbourou.com/blog/posts/dynamically-loading-modules-and-classes-in-python/
diff --git a/pyzenkit/zendaemon.py b/pyzenkit/zendaemon.py
index b31c70fb828b9b6f966cdba3b2bc931c6cec151f..24e28132c5548e4db43e26058cfaa082c230a19e 100644
--- a/pyzenkit/zendaemon.py
+++ b/pyzenkit/zendaemon.py
@@ -1,14 +1,44 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
+
"""
-Base implementation of generic daemon.
+This module provides base implementation of generic daemon. It builds on top of
+:py:mod:`pyzenkit.baseapp` and adds following usefull features:
+
+* Event driven design.
+* Support for arbitrary signal handling.
+* Support for modularity with daemon components.
+* Fully automated daemonization process.
+
+Events and event queue
+^^^^^^^^^^^^^^^^^^^^^^
+
+Signal handling
+^^^^^^^^^^^^^^^
+
+Daemon components
+^^^^^^^^^^^^^^^^^
+
+Daemonization
+^^^^^^^^^^^^^
+
"""
+
+__author__ = "Jan Mach <honza.mach.ml@gmail.com>"
+
+
import os
import re
import sys
@@ -23,145 +53,196 @@ import math
import glob
import pprint
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
-sys.path.insert(0, lib)
-
#
# Custom libraries.
#
import pyzenkit.baseapp
import pyzenkit.daemonizer
+
# Translation table to translate signal numbers to their names.
SIGNALS_TO_NAMES_DICT = dict((getattr(signal, n), n) \
for n in dir(signal) if n.startswith('SIG') and '_' not in n )
-# Simple method for JSON serialization
-def _json_default(o):
- if isinstance(o, ZenDaemonComponent):
- return "COMPONENT({})".format(o.__class__.__name__)
- elif callable(o):
- return "CALLBACK({}:{})".format(o.__self__.__class__.__name__, o.__name__)
- else:
- return repr(o)
+
+def _json_default(obj):
+ """
+ Fallback method for serializing unknown objects into JSON.
+ """
+ if isinstance(obj, ZenDaemonComponent):
+ return "COMPONENT({})".format(obj.__class__.__name__)
+ if callable(obj):
+ return "CALLBACK({}:{})".format(obj.__self__.__class__.__name__, obj.__name__)
+ return repr(obj)
+
+
+#-------------------------------------------------------------------------------
+
class QueueEmptyException(Exception):
"""
- Exception representing empty event queue.
+ Exception representing empty event queue. This exception will be thrown by
+ :py:class:`zendaemon.EventQueueManager` in the event of empty event queue.
"""
+ def __init__(self, description, **params):
+ """
+ Initialize new exception with given description and optional additional
+ parameters.
+
+ :param str description: Description of the problem.
+ :param params: Optional additional parameters.
+ """
+ super().__init__()
+
+ self.description = description
+ self.params = params
- def __init__(self, description):
- self._description = description
def __str__(self):
- return repr(self._description)
+ """
+ Operator override for automatic string output.
+ """
+ return repr(self.description)
+
class EventQueueManager:
"""
- Implementation of event queue manager.
-
- This implementation supports scheduling of both generic sequential events and
- timed events.
+ Implementation of event queue manager. This implementation supports scheduling
+ of both sequential events and timed events (events scheduled for specific time).
+ The actual event object, that is added into the queue may be arbitrary object,
+ there are no restrictions for its type or interface, because the queue manager
+ does not interacts with the event itself. Internally two separate event queues
+ are used, one for sequentialy scheduled events and another for timed events.
+ For best performance the sequential queue is implemented using :py:class:`collections.dequeue`
+ object and the timed queue is implemented using :py:mod:`heapq` module.
"""
- def __init__(self, **kwargs):
+ def __init__(self):
"""
-
+ Base event queue manager constructor. Initialize internal event queues.
"""
self.events = collections.deque()
self.events_at = []
- def __del__(self):
- """
- Default script object destructor. Perform generic cleanup.
- """
- pass
-
def schedule(self, event, args = None):
"""
Schedule new event to the end of the event queue.
+
+ :param event: Event to be scheduled.
+ :param args: Optional event arguments to be stored alongside the event.
"""
self.events.append((event, args))
def schedule_next(self, event, args = None):
"""
Schedule new event to the beginning of the event queue.
+
+ :param event: Event to be scheduled.
+ :param args: Optional event arguments to be stored alongside the event.
"""
self.events.appendleft((event, args))
- def schedule_at(self, ts, event, args = None):
+ def schedule_at(self, tstamp, event, args = None):
"""
Schedule new event for a specific time.
+
+ :param float tstamp: Timestamp to which to schedule the event (compatible with :py:func:`time.time`).
+ :param event: Event to be scheduled.
+ :param args: Optional event arguments to be stored alongside the event.
"""
- heapq.heappush(self.events_at, (ts, event, args))
+ heapq.heappush(self.events_at, (tstamp, event, args))
def schedule_after(self, delay, event, args = None):
"""
- Schedule new event after a given.
+ Schedule new event after a given time delay.
+
+ :param float delay: Time delay after which to schedule the event.
+ :param event: Event to be scheduled.
+ :param args: Optional event arguments to be stored alongside the event.
"""
- ts = time.time() + delay
- heapq.heappush(self.events_at, (ts, event, args))
+ tstamp = time.time() + delay
+ heapq.heappush(self.events_at, (tstamp, event, args))
def next(self):
"""
- Fetch next event from event queue.
+ Fetch next event from queue.
+
+ :raises QueueEmptyException: If the queue is empty.
+ :return: Return next scheduled event from queue along with its optional arguments.
+ :rtype: tuple
"""
- l1 = len(self.events_at)
- if l1:
+ len1 = len(self.events_at)
+ if len1:
if self.events_at[0][0] <= time.time():
- (ts, event, args) = heapq.heappop(self.events_at)
+ (tstamp, event, args) = heapq.heappop(self.events_at)
return (event, args)
- l2 = len(self.events)
- if l2:
+ len2 = len(self.events)
+ if len2:
return self.events.popleft()
- if (l1 + l2) == 0:
+ if (len1 + len2) == 0:
raise QueueEmptyException("Event queue is empty")
return (None, None)
def when(self):
"""
- Determine the time when the next event is scheduled.
+ Determine the timestamp of the next scheduled event.
+
+ :return: Unix timestamp of next scheduled event.
+ :rtype: float
"""
+ if self.events:
+ return time.time()
return self.events_at[0][0]
def wait(self):
"""
- Calculate the waiting period until the next even is due.
+ Calculate the waiting period until the next event in queue is due.
+
+ :return: Time interval for which to wait until the next event is due.
+ :rtype: float
"""
+ if self.events:
+ return 0
return self.events_at[0][0] - time.time()
def count(self):
"""
Count the total number of scheduled events.
+
+ :return: Number of events.
+ :rtype: int
"""
return len(self.events_at) + len(self.events)
-class ZenDaemonComponentException(Exception):
- """
+#-------------------------------------------------------------------------------
+
+
+class ZenDaemonComponentException(pyzenkit.baseapp.ZenAppProcessException):
"""
- def __init__(self, description):
- self._description = description
- def __str__(self):
- return repr(self._description)
+ Describes problems specific to daemon components.
+ """
+ pass
+
class ZenDaemonComponent:
"""
- Base implementation of all daemon components.
+ Base implementation for all daemon components. Daemon components are building
+ blocks of each daemon and they are responsible for the actual work to be done.
+ This approach enables very easy reusability.
"""
def __init__(self, **kwargs):
"""
-
+ Base daemon component object constructor.
"""
self.statistics_cur = {}
self.statistics_prev = {}
self.statistics_ts = time.time()
- self.pattern_stats = "{}\n\t{:15s} {:12,d} (+{:8,d}, {:8,.2f} #/s)"
+ self.pattern_stats = "{}\n\t{:15s} {:12,d} (+{:8,d}, {:8,.2f} #/s)"
- def inc_statistics(self, key, increment = 1):
+ def inc_statistic(self, key, increment = 1):
"""
- Raise given statistics key with given increment.
+ Raise given statistic key with given increment.
"""
self.statistics_cur[key] = self.statistics_cur.get(key, 0) + increment
@@ -169,41 +250,46 @@ class ZenDaemonComponent:
"""
Get the list of event names and their appropriate callback handlers.
"""
- raise Exception("This method must be implemented in subclass")
+ raise NotImplementedError("This method must be implemented in subclass")
- def get_state(self, daemon):
+ def get_state(self):
"""
Get the current internal state of component (for debugging).
"""
- return {}
+ return {
+ 'statistics': self.statistics_cur
+ }
- def calc_statistics(self, daemon, stats_cur, stats_prev, tdiff):
+ @staticmethod
+ def calc_statistics(stats_cur, stats_prev, tdiff):
"""
-
+ Calculate daemon component statistics.
"""
result = {}
- for k in stats_cur:
- if isinstance(stats_cur[k], int):
- result[k] = {
- 'cnt': stats_cur[k],
- 'inc': stats_cur[k] - stats_prev.get(k, 0),
- 'spd': (stats_cur[k] - stats_prev.get(k, 0)) / tdiff
- }
- elif isinstance(stats_cur[k], dict):
- result[k] = self.calc_statistics(daemon, stats_cur[k], stats_prev.get(k, {}), tdiff)
+ for key in stats_cur:
+ result[key] = {
+ # Absolute count.
+ 'cnt': stats_cur[key],
+ # Increase from previous value.
+ 'inc': stats_cur[key] - stats_prev.get(key, 0),
+ # Processing speed (#/s)
+ 'spd': (stats_cur[key] - stats_prev.get(key, 0)) / tdiff,
+ # Percentage increase.
+ 'pct': (stats_cur[key] - stats_prev.get(key, 0)) / (stats_cur[key] / 100)
+ }
return result
- def get_statistics(self, daemon):
+ def get_statistics(self):
"""
Calculate processing statistics
"""
- ct = time.time()
- tdiff = ct - self.statistics_ts
+ curts = time.time()
+ tdiff = curts - self.statistics_ts
- stats = self.calc_statistics(daemon, self.statistics_cur, self.statistics_prev, tdiff)
+ stats = self.calc_statistics(self.statistics_cur, self.statistics_prev, tdiff)
self.statistics_prev = copy.copy(self.statistics_cur)
- self.statistics_ts = ct
+ self.statistics_ts = curts
return stats
def setup(self, daemon):
@@ -218,31 +304,41 @@ class ZenDaemonComponent:
"""
pass
-class ZenDaemonException(pyzenkit.baseapp.ZenAppException):
+
+#-------------------------------------------------------------------------------
+
+
+class ZenDaemonException(pyzenkit.baseapp.ZenAppProcessException):
"""
Describes problems specific to daemons.
"""
pass
+
class ZenDaemon(pyzenkit.baseapp.BaseApp):
"""
Base implementation of generic daemon.
"""
+ #
+ # Class constants.
+ #
+
# Event loop processing flags.
FLAG_CONTINUE = 1
FLAG_STOP = 0
+ # List of event names.
EVENT_SIGNAL_HUP = 'signal_hup'
EVENT_SIGNAL_USR1 = 'signal_usr1'
EVENT_SIGNAL_USR2 = 'signal_usr2'
EVENT_LOG_STATISTICS = 'log_statistics'
# List of core configuration keys.
- CORE_STATE = 'state'
- CORE_STATE_SAVE = 'save'
+ CORE_STATE = 'state'
+ CORE_STATE_SAVE = 'save'
- # List of possible configuration keys.
+ # List of configuration keys.
CONFIG_COMPONENTS = 'components'
CONFIG_NODAEMON = 'no_daemon'
CONFIG_CHROOT_DIR = 'chroot_dir'
@@ -253,12 +349,21 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
CONFIG_STATS_INTERVAL = 'stats_interval'
CONFIG_PARALEL = 'paralel'
+
def __init__(self, **kwargs):
"""
- Default script object constructor.
+ Default application object constructor.
+
+ Only defines core internal variables. The actual object initialization,
+ during which command line arguments and configuration files are parsed,
+ is done during the configure() stage of the run() sequence. This method
+ overrides the base implementation in :py:func:`baseapp.BaseApp.__init__`.
+
+ :param kwargs: Various additional parameters.
"""
super().__init__(**kwargs)
+ self.flag_done = False
self.queue = EventQueueManager()
self.components = []
self.callbacks = {}
@@ -267,56 +372,58 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
self._init_components(**kwargs)
self._init_schedule(**kwargs)
- def _init_config(self, **kwargs):
- """
- Initialize script configurations to default values.
+ def _init_config(self, cfgs, **kwargs):
"""
- config = super()._init_config(**kwargs)
+ Initialize default application configurations. This method overrides the
+ base implementation in :py:func:`baseapp.BaseApp._init_argparser` and it
+ adds additional configurations via ``cfgs`` parameter.
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
+
+ :param list cfgs: Additional set of configurations.
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Default configuration structure.
+ :rtype: dict
+ """
cfgs = (
(self.CONFIG_NODAEMON, False),
(self.CONFIG_CHROOT_DIR, None),
(self.CONFIG_WORK_DIR, '/'),
(self.CONFIG_PID_FILE, os.path.join(self.paths.get(self.PATH_RUN), "{}.pid".format(self.name))),
(self.CONFIG_STATE_FILE, os.path.join(self.paths.get(self.PATH_RUN), "{}.state".format(self.name))),
- (self.CONFIG_UMASK, None),
+ (self.CONFIG_UMASK, 0o002),
(self.CONFIG_STATS_INTERVAL, 300),
(self.CONFIG_PARALEL, False),
)
- for c in cfgs:
- config[c[0]] = kwargs.pop('default_' + c[0], c[1])
- return config
+ return super()._init_config(cfgs, **kwargs)
def _init_argparser(self, **kwargs):
"""
- Initialize script command line argument parser.
- """
- argparser = super()._init_argparser(**kwargs)
-
- # Option flag indicating that the script should not daemonize and stay
- # in foreground (usefull for debugging or testing).
- argparser.add_argument('--no-daemon', help = 'do not daemonize, stay in foreground (flag)', action='store_true', default = None)
-
- # Option for overriding the name of the chroot directory.
- argparser.add_argument('--chroot-dir', help = 'name of the chroot directory')
-
- # Option for overriding the name of the work directory.
- argparser.add_argument('--work-dir', help = 'name of the work directory')
-
- # Option for overriding the name of the PID file.
- argparser.add_argument('--pid-file', help = 'name of the pid file')
+ Initialize application command line argument parser. This method overrides
+ the base implementation in :py:func:`baseapp.BaseApp._init_argparser` and
+ it must return valid :py:class:`argparse.ArgumentParser` object.
- # Option for overriding the name of the state file.
- argparser.add_argument('--state-file', help = 'name of the state file')
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
- # Option for overriding the default umask.
- argparser.add_argument('--umask', help = 'default file umask')
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Initialized argument parser object.
+ :rtype: argparse.ArgumentParser
+ """
+ argparser = super()._init_argparser(**kwargs)
- # Option for defining processing statistics display interval.
- argparser.add_argument('--stats-interval', help = 'define processing statistics display interval')
+ #
+ # Create and populate options group for common daemon arguments.
+ #
+ arggroup_daemon = argparser.add_argument_group('common daemon arguments')
- # Option flag indicating that the script may run in paralel processes.
- argparser.add_argument('--paralel', help = 'run in paralel mode (flag)', action = 'store_true', default = None)
+ arggroup_daemon.add_argument('--no-daemon', help = 'do not fully daemonize and stay in foreground (flag)', action='store_true', default = None)
+ arggroup_daemon.add_argument('--chroot-dir', help = 'name of the chroot directory', type = str, default = None)
+ arggroup_daemon.add_argument('--work-dir', help = 'name of the process work directory', type = str, default = None)
+ arggroup_daemon.add_argument('--pid-file', help = 'name of the pid file', type = str, default = None)
+ arggroup_daemon.add_argument('--state-file', help = 'name of the state file', type = str, default = None)
+ arggroup_daemon.add_argument('--umask', help = 'default file umask', default = None)
+ arggroup_daemon.add_argument('--stats-interval', help = 'processing statistics display interval in seconds', type = int)
+ arggroup_daemon.add_argument('--paralel', help = 'run in paralel mode (flag)', action = 'store_true', default = None)
return argparser
@@ -336,6 +443,7 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
Initialize internal event callbacks.
"""
for event in self.get_events():
+ self.dbgout("Initializing event callback '{}':'{}'".format(str(event['event']), str(event['callback'])))
self._init_event_callback(event['event'], event['callback'], event['prepend'])
def _init_components(self, **kwargs):
@@ -366,21 +474,26 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
for event in initial_events:
self.queue.schedule_after(*event)
+
#---------------------------------------------------------------------------
+
def _configure_postprocess(self):
"""
- Setup internal script core mechanics. Config postprocessing routine.
+ Perform configuration postprocessing and calculate core configurations.
+ This method overrides the base implementation in :py:func:`baseapp.BaseApp._configure_postprocess`.
+
+ Gets called from :py:func:`BaseApp._stage_setup_configuration`.
"""
super()._configure_postprocess()
- cc = {}
- cc[self.CORE_STATE_SAVE] = True
- self.config[self.CORE][self.CORE_STATE] = cc
+ ccfg = {}
+ ccfg[self.CORE_STATE_SAVE] = True
+ self.config[self.CORE][self.CORE_STATE] = ccfg
if self.c(self.CONFIG_NODAEMON):
- self.dbgout("[STATUS] Console log output is enabled via '--no-daemon' configuration")
self.config[self.CORE][self.CORE_LOGGING][self.CORE_LOGGING_TOCONS] = True
+ self.dbgout("Console log output is enabled via '--no-daemon' configuration")
else:
self.config[self.CORE][self.CORE_LOGGING][self.CORE_LOGGING_TOCONS] = False
@@ -388,11 +501,14 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
self.config[self.CORE][self.CORE_RUNLOG][self.CORE_RUNLOG_SAVE] = True
self.config[self.CORE][self.CORE_PSTATE][self.CORE_PSTATE_SAVE] = True
- def _stage_setup_custom(self):
+ def _sub_stage_setup(self):
"""
- Perform custom daemon related setup.
+ **SUBCLASS HOOK**: Perform additional custom setup actions in **setup** stage.
+
+ Gets called from :py:func:`BaseApp._stage_setup` and it is a **SETUP SUBSTAGE 06**.
"""
for component in self.components:
+ self.dbgout("Configuring daemon component '{}'".format(component))
component.setup(self)
def _stage_setup_dump(self):
@@ -404,105 +520,121 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
"""
super()._stage_setup_dump()
- self.logger.debug("Daemon component list >>>\n{}".format(json.dumps(self.components, sort_keys=True, indent=4, default=_json_default)))
- self.logger.debug("Registered event callbacks >>>\n{}".format(json.dumps(self.callbacks, sort_keys=True, indent=4, default=_json_default)))
+ self.logger.debug("Daemon component list >>>\n%s", json.dumps(self.components, sort_keys=True, indent=4, default=_json_default))
+ self.logger.debug("Registered event callbacks >>>\n%s", json.dumps(self.callbacks, sort_keys=True, indent=4, default=_json_default))
self.logger.debug("Daemon component setup >>>\n")
for component in self.components:
- self.logger.debug(">>> {} >>>\n".format(component.__class__.__name__))
+ self.logger.debug(">>> %s >>>\n", component.__class__.__name__)
component.setup_dump(self)
+
#---------------------------------------------------------------------------
+
def _hnd_signal_wakeup(self, signum, frame):
"""
- Minimal signal handler - wakeup after sleep/pause.
+ Signal handler - wakeup after sleep/pause.
"""
- self.logger.info("Wakeup after pause")
+ self.logger.info("Received wakeup signal (%s)", signum)
def _hnd_signal_hup(self, signum, frame):
"""
- Minimal signal handler - SIGHUP
+ Signal handler - **SIGHUP**
Implementation of the handler is intentionally brief, actual signal
- handling is done via scheduling and handling event 'signal_hup'.
+ handling is done via scheduling and handling event ``signal_hup``.
"""
- self.logger.warning("Received signal 'SIGHUP'")
- self.queue.schedule_next('signal_hup')
+ self.logger.warning("Received signal 'SIGHUP' (%s)", signum)
+ self.queue.schedule_next(self.EVENT_SIGNAL_HUP)
def _hnd_signal_usr1(self, signum, frame):
"""
- Minimal signal handler - SIGUSR1
+ Signal handler - **SIGUSR1**
Implementation of the handler is intentionally brief, actual signal
- handling is done via scheduling and handling event 'signal_usr1'.
+ handling is done via scheduling and handling event ``signal_usr1``.
"""
- self.logger.info("Received signal 'SIGUSR1'")
- self.queue.schedule_next('signal_usr1')
+ self.logger.info("Received signal 'SIGUSR1' (%s)", signum)
+ self.queue.schedule_next(self.EVENT_SIGNAL_USR1)
def _hnd_signal_usr2(self, signum, frame):
"""
- Minimal signal handler - SIGUSR2
+ Signal handler - **SIGUSR2**
Implementation of the handler is intentionally brief, actual signal
- handling is done via scheduling and handling event 'signal_usr2'.
+ handling is done via scheduling and handling event ``signal_usr2``.
"""
- self.logger.info("Received signal 'SIGUSR2'")
- self.queue.schedule_next('signal_usr2')
+ self.logger.info("Received signal 'SIGUSR2' (%s)", signum)
+ self.queue.schedule_next(self.EVENT_SIGNAL_USR2)
+
#---------------------------------------------------------------------------
+
def cbk_event_signal_hup(self, daemon, args = None):
"""
- Event callback to handle signal - SIGHUP
+ Event callback for handling signal - **SIGHUP**
+
+ .. todo::
+
+ In the future this signal should be responsible for soft restart of
+ daemon process. Currently work in progress.
"""
self.logger.warning("Handling event for signal 'SIGHUP'")
- return (self.FLAG_CONTINUE, None)
+ return (self.FLAG_CONTINUE, args)
def cbk_event_signal_usr1(self, daemon, args = None):
"""
- Event callback to handle signal - SIGUSR1
+ Event callback for handling signal - **SIGUSR1**
+
+ This signal forces the daemon process to save the current runlog to JSON
+ file.
"""
self.logger.info("Handling event for signal 'SIGUSR1'")
- self.runlog_save(self.runlog)
- return (self.FLAG_CONTINUE, None)
+ self._utils_runlog_save(self.runlog)
+ return (self.FLAG_CONTINUE, args)
def cbk_event_signal_usr2(self, daemon, args = None):
"""
- Event callback to handle signal - SIGUSR2
+ Event callback for handling signal - **SIGUSR2**
+
+ This signal forces the daemon process to save the current state to JSON
+ file. State is more verbose than runlog and it contains almost all
+ internal data.
"""
self.logger.info("Handling event for signal 'SIGUSR2'")
if self.c(self.CONFIG_NODAEMON):
- self.state_dump(self._get_state())
+ self._utils_state_dump(self._get_state())
else:
- self.state_save(self._get_state())
- return (self.FLAG_CONTINUE, None)
+ self._utils_state_save(self._get_state())
+ return (self.FLAG_CONTINUE, args)
def cbk_event_log_statistics(self, daemon, args):
"""
Periodical processing statistics logging.
"""
self.queue.schedule_after(self.c(self.CONFIG_STATS_INTERVAL), self.EVENT_LOG_STATISTICS)
- return (self.FLAG_CONTINUE, None)
+ return (self.FLAG_CONTINUE, args)
#---------------------------------------------------------------------------
- def send_signal(self, s):
+ def send_signal(self, sign):
"""
- Send given signal to currently running daemon(s).
+ Send given signal to all currently running daemon(s).
"""
pid = None
try:
pidfl = None # PID file list
if not self.c(self.CONFIG_PARALEL):
- pidfl = [self.get_fn_pidfile()]
+ pidfl = [self._get_fn_pidfile()]
else:
- pidfl = self.pidfiles_list()
+ pidfl = self._pidfiles_list()
for pidfn in pidfl:
pid = pyzenkit.daemonizer.read_pid(pidfn)
if pid:
- print("Sending signal '{}' to process '{}' [{}]".format(SIGNALS_TO_NAMES_DICT.get(s, s), pid, pidfn))
- os.kill(pid, s)
+ print("Sending signal '{}' to process '{}' [{}]".format(SIGNALS_TO_NAMES_DICT.get(sign, sign), pid, pidfn))
+ os.kill(pid, sign)
except FileNotFoundError:
print("PID file '{}' does not exist".format(self.c(self.CONFIG_PID_FILE)))
@@ -514,7 +646,7 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
print("Process with PID '{}' does not exist".format(pid))
except PermissionError:
- print("Insufficient permissions to send signal '{}' to process '{}'".format(SIGNALS_TO_NAMES_DICT.get(s, s), pid))
+ print("Insufficient permissions to send signal '{}' to process '{}'".format(SIGNALS_TO_NAMES_DICT.get(sign, sign), pid))
def cbk_action_signal_check(self):
"""
@@ -552,15 +684,17 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
"""
self.send_signal(signal.SIGUSR2)
+
#---------------------------------------------------------------------------
+
def _get_state(self):
"""
-
+ Get current daemon state.
"""
state = {
'time': time.time(),
- 'rc': self.rc,
+ 'rc': self.retc,
'config': self.config,
'paths': self.paths,
'pstate': self.pstate,
@@ -575,7 +709,7 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
def _get_statistics(self):
"""
-
+ Get current daemon statistics.
"""
statistics = {
'time': time.time(),
@@ -585,7 +719,17 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
statistics['components'][component.__class__.__name__] = component.get_statistics(self)
return statistics
- def state_dump(self, state):
+ def _utils_state_dump(self, state):
+ """
+ Dump current daemon state.
+
+ Dump current daemon state to terminal (JSON).
+ """
+ # Dump current script state.
+ #self.logger.debug("Current daemon state >>>\n{}".format(json.dumps(state, sort_keys=True, indent=4)))
+ print("Current daemon state >>>\n{}".format(self.json_dump(state, default=_json_default)))
+
+ def _utils_state_log(self, state):
"""
Dump current daemon state.
@@ -595,20 +739,20 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
#self.logger.debug("Current daemon state >>>\n{}".format(json.dumps(state, sort_keys=True, indent=4)))
print("Current daemon state >>>\n{}".format(self.json_dump(state, default=_json_default)))
- def state_save(self, state):
+ def _utils_state_save(self, state):
"""
Save current daemon state.
Save current daemon state to external file (JSON).
"""
- sfn = self.get_fn_state()
- self.dbgout("[STATUS] Saving current daemon state to file '{}'".format(sfn))
+ sfn = self._get_fn_state()
+ self.dbgout("Saving current daemon state to file '{}'".format(sfn))
pprint.pprint(state)
- self.dbgout("[STATUS] Current daemon state:\n{}".format(self.json_dump(state, default=_json_default)))
+ self.dbgout("Current daemon state:\n{}".format(self.json_dump(state, default=_json_default)))
self.json_save(sfn, state, default=_json_default)
- self.logger.info("Current daemon state saved to file '{}'".format(sfn))
+ self.logger.info("Current daemon state saved to file '%s'", sfn)
- def pidfiles_list(self, **kwargs):
+ def _pidfiles_list(self, **kwargs):
"""
List all available pidfiles.
"""
@@ -616,38 +760,42 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
pfn = os.path.join(self.paths['run'], '{}*.pid'.format(self.name))
return sorted(glob.glob(pfn), reverse = reverse)
- def get_fn_state(self):
+ def _get_fn_state(self):
"""
Return the name of the state file for current process.
"""
if not self.c(self.CONFIG_PARALEL):
return self.c(self.CONFIG_STATE_FILE)
- else:
- fn = re.sub("\.state$",".{:05d}.state".format(os.getpid()), self.c(self.CONFIG_STATE_FILE))
- self.dbgout("[STATUS] Paralel mode: using '{}' as state file".format(fn))
- return fn
- def get_fn_pidfile(self):
+ sfn = re.sub(r'\.state$',".{:05d}.state".format(os.getpid()), self.c(self.CONFIG_STATE_FILE))
+ self.dbgout("Paralel mode: using '{}' as state file".format(sfn))
+ return sfn
+
+ def _get_fn_pidfile(self):
"""
Return the name of the pidfile for current process.
"""
if not self.c(self.CONFIG_PARALEL):
return self.c(self.CONFIG_PID_FILE)
- else:
- fn = re.sub("\.pid$",".{:05d}.pid".format(os.getpid()), self.c(self.CONFIG_PID_FILE))
- self.dbgout("[STATUS] Paralel mode: using '{}' as pid file".format(fn))
- return fn
- def get_fn_runlog(self):
+ pfn = re.sub(r'\.pid$',".{:05d}.pid".format(os.getpid()), self.c(self.CONFIG_PID_FILE))
+ self.dbgout("Paralel mode: using '{}' as pid file".format(pfn))
+ return pfn
+
+ def _get_fn_runlog(self):
"""
Return the name of the runlog file for current process.
"""
if not self.c(self.CONFIG_PARALEL):
return os.path.join(self.c(self.CONFIG_RUNLOG_DIR), "{}.runlog".format(self.runlog[self.RLKEY_TSFSF]))
- else:
- fn = os.path.join(self.c(self.CONFIG_RUNLOG_DIR), "{}.{:05d}.runlog".format(self.runlog[self.RLKEY_TSFSF], os.getpid()))
- self.dbgout("[STATUS] Paralel mode: using '{}' as runlog file".format(fn))
- return fn
+
+ rfn = os.path.join(self.c(self.CONFIG_RUNLOG_DIR), "{}.{:05d}.runlog".format(self.runlog[self.RLKEY_TSFSF], os.getpid()))
+ self.dbgout("Paralel mode: using '{}' as runlog file".format(rfn))
+ return rfn
+
+
+ #---------------------------------------------------------------------------
+
def get_events(self):
"""
@@ -666,7 +814,7 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
"""
period = math.ceil(period)
if period > 0:
- self.logger.info("Waiting for '{}' seconds until next scheduled event".format(period))
+ self.logger.info("Waiting for '%d' seconds until next scheduled event", period)
signal.signal(signal.SIGALRM, self._hnd_signal_wakeup)
signal.alarm(period)
signal.pause()
@@ -676,7 +824,7 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
"""
Set the DONE flag to True.
"""
- self.done = True
+ self.flag_done = True
def _daemonize(self):
"""
@@ -684,14 +832,14 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
"""
# Perform full daemonization
if not self.c(self.CONFIG_NODAEMON):
- self.dbgout("[STATUS] Performing full daemonization")
+ self.dbgout("Performing full daemonization")
self.logger.info("Performing full daemonization")
logs = pyzenkit.daemonizer.get_logger_files(self.logger)
pyzenkit.daemonizer.daemonize(
chroot_dir = self.c(self.CONFIG_CHROOT_DIR),
work_dir = self.c(self.CONFIG_WORK_DIR),
- pidfile = self.get_fn_pidfile(),
+ pid_file = self._get_fn_pidfile(),
umask = self.c(self.CONFIG_UMASK),
files_preserve = logs,
signals = {
@@ -706,13 +854,13 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
# Perform simple daemonization
else:
- self.dbgout("[STATUS] Performing simple daemonization")
+ self.dbgout("Performing simple daemonization")
self.logger.info("Performing simple daemonization")
pyzenkit.daemonizer.daemonize_lite(
chroot_dir = self.c(self.CONFIG_CHROOT_DIR),
work_dir = self.c(self.CONFIG_WORK_DIR),
- pidfile = self.get_fn_pidfile(),
+ pid_file = self._get_fn_pidfile(),
umask = self.c(self.CONFIG_UMASK),
signals = {
signal.SIGHUP: self._hnd_signal_hup,
@@ -724,13 +872,12 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
self.logger.info("Simple daemonization done")
self.runlog[self.RLKEY_PID] = os.getpid()
-
def _event_loop(self):
"""
Main event processing loop.
"""
- self.done = False
- while not self.done:
+ self.flag_done = False
+ while not self.flag_done:
try:
(event, args) = self.queue.next()
if event:
@@ -741,23 +888,18 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
if flag != self.FLAG_CONTINUE:
break
else:
- w = self.queue.wait()
- if w > 0:
- self.wait(w)
- pass
+ wait_time = self.queue.wait()
+ if wait_time > 0:
+ self.wait(wait_time)
+
except QueueEmptyException:
self.logger.info("Event queue is empty, terminating")
- self.done = True
- pass
+ self.flag_done = True
- def stage_process(self):
+ def _sub_stage_process(self):
"""
- Script lifecycle stage: PROCESSING
-
- Perform some real work (finally). Following method will call appropriate
- callback method operation to service the selected operation.
+ **SUBCLASS HOOK**: Perform some actual processing in **process** stage.
"""
- self.time_mark('stage_process_start', 'Start of the processing stage')
try:
self._daemonize()
@@ -776,12 +918,10 @@ class ZenDaemon(pyzenkit.baseapp.BaseApp):
self.error("ZenAppException: {}".format(exc))
except:
- (t, v, tb) = sys.exc_info()
- self.error("Exception: {}".format(v), tb = tb)
+ (exct, excv, exctb) = sys.exc_info()
+ self.error("Exception: {}".format(excv), trcb = exctb)
- self.time_mark('stage_process_stop', 'End of the processing stage')
-
-class _DemoDaemonComponent(ZenDaemonComponent):
+class DemoDaemonComponent(ZenDaemonComponent):
"""
Minimalistic class for demonstration purposes.
"""
@@ -803,31 +943,44 @@ class _DemoDaemonComponent(ZenDaemonComponent):
time.sleep(1)
return (daemon.FLAG_CONTINUE, None)
-class _DemoZenDaemon(ZenDaemon):
+class DemoZenDaemon(ZenDaemon):
"""
Minimalistic class for demonstration purposes.
"""
-
pass
+#-------------------------------------------------------------------------------
+
+#
+# Perform the demonstration.
+#
if __name__ == "__main__":
- """
- Perform the demonstration.
- """
- # Prepare the environment
- if not os.path.isdir('/tmp/zendaemon.py'):
- os.mkdir('/tmp/zendaemon.py')
- pyzenkit.baseapp.BaseApp.json_save('/tmp/zendaemon.py.conf', {'test_a':1})
-
- daemon = _DemoZenDaemon(
- path_cfg = '/tmp',
- path_log = '/tmp',
- path_tmp = '/tmp',
- path_run = '/tmp',
- description = 'DemoZenDaemon - generic daemon (DEMO)',
- schedule = [('default',)],
- components = [
- _DemoDaemonComponent()
- ]
- )
- daemon.run()
+
+ # Prepare demonstration environment.
+ pyzenkit.baseapp.BaseApp.json_save('/tmp/demo-zendaemon.py.conf', {'test_a':1})
+ try:
+ os.mkdir('/tmp/demo-zendaemon.py')
+ except FileExistsError:
+ pass
+
+ ZENDAEMON = DemoZenDaemon(
+ name = 'demo-zenscript.py',
+ description = 'DemoZenDaemon - Demonstration daemon',
+
+ #
+ # Configure required application paths to harmless locations.
+ #
+ path_bin = '/tmp',
+ path_cfg = '/tmp',
+ path_log = '/tmp',
+ path_tmp = '/tmp',
+ path_run = '/tmp',
+
+ default_no_daemon = True,
+
+ schedule = [('default',)],
+ components = [
+ DemoDaemonComponent()
+ ]
+ )
+ ZENDAEMON.run()
diff --git a/pyzenkit/zenscript.py b/pyzenkit/zenscript.py
index d30853ba0294140d2c24d6856d7e3217d8a844cc..4c1f7e9fc2f9b71842cf634cd1b80f37af0230a3 100644
--- a/pyzenkit/zenscript.py
+++ b/pyzenkit/zenscript.py
@@ -1,36 +1,93 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
-# Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
-# Use of this source is governed by the MIT license, see LICENSE file.
+# This file is part of PyZenKit package.
+#
+# Copyright (C) since 2016 CESNET, z.s.p.o (http://www.ces.net/)
+# Copyright (C) since 2015 Jan Mach <honza.mach.ml@gmail.com>
+# Use of this package is governed by the MIT license, see LICENSE file.
+#
+# This project was initially written for personal use of the original author. Later
+# it was developed much further and used for project of author`s employer.
#-------------------------------------------------------------------------------
+
"""
-Base implementation of generic one time execution script with cron support.
+This module provides base implementation of generic script with built-in support
+for regular executions. It builds on top of :py:mod:`pyzenkit.baseapp` module and
+adds couple of other usefull features:
+
+* Support for executing multiple different **commands**.
+* Support for regular executions.
+
+
+Script commands
+^^^^^^^^^^^^^^^
+
+Every script provides support for more, possibly similar, commands to be implemented
+within one script.
+
+
+Script execution modes
+^^^^^^^^^^^^^^^^^^^^^^
+
+Script execution supports following modes:
+
+* **regular**
+* **shell**
+* **default**
+
+In a **regular** mode the script is intended to be executed in regular time intervals
+from a *cron-like* service. The internal application configuration is forced into
+following state:
+
+* Console output is explicitly suppressed
+* Console logging level is explicitly forced to 'warning' level
+* Logging to log file is explicitly forced to be enabled
+* Runlog saving is explicitly forced to be enabled
+* Persistent state saving is explicitly forced to be enabled
+
+In a **shell** mode the script is intended to be executed by hand from interactive
+shell. It is intended to be used for debugging or experimental purposes and the
+internal application configuration is forced into following state:
+
+* Logging to log file is explicitly suppressed
+* Runlog saving is explicitly suppressed
+* Persistent state saving is explicitly suppressed
+
+
+Module contents
+^^^^^^^^^^^^^^^
+
+* :py:class:`ZenScriptException`
+* :py:class:`ZenScript`
+* :py:class:`DemoZenScript`
"""
+
+__author__ = "Jan Mach <honza.mach.ml@gmail.com>"
+
+
import os
import re
-import sys
-import json
import time
-import math
-import subprocess
-import pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../'))
-sys.path.insert(0, lib)
+import datetime
#
# Custom libraries.
#
import pyzenkit.baseapp
+
#
# Predefined constants for runtime intervals
#
RUN_INTERVALS = {
+ '5_minutes': 300,
+ '10_minutes': 600,
+ '15_minutes': 900,
+ '20_minutes': 1200,
+ '30_minutes': 1800,
'hourly': 3600,
'2_hourly': (2*3600),
'3_hourly': (3*3600),
@@ -43,185 +100,315 @@ RUN_INTERVALS = {
'4_weekly': (28*86400),
}
-class ZenScriptException(pyzenkit.baseapp.ZenAppException):
+RE_TIMESTAMP = re.compile(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})[Tt ]([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.([0-9]+))?([Zz]|(?:[+-][0-9]{2}:[0-9]{2}))$")
+
+
+#-------------------------------------------------------------------------------
+
+
+def t_datetime(val):
+ """
+ Convert/validate datetime. The value received by this conversion function may
+ be either ``datetime.datetime`` object (in that case no action will be done),
+ unix timestamp as ``float`` or datetime as RFC3339 string.
+
+ :param any val: Value to be converted/validated
+ :return: Datetime object
+ :rtype: datetime.datetime
+ :raises ValueError: if the value could not be converted to datetime.datetime object
+ """
+ # Maybe there is nothing to do
+ if isinstance(val, datetime.datetime):
+ return val
+
+ # Try numeric type
+ try:
+ return datetime.datetime.fromtimestamp(float(val))
+ except (TypeError, ValueError):
+ pass
+ # Try RFC3339 string
+ res = RE_TIMESTAMP.match(val)
+ if res is not None:
+ year, month, day, hour, minute, second = (int(n or 0) for n in res.group(*range(1, 7)))
+ us_str = (res.group(7) or '0')[:6].ljust(6, '0')
+ us_int = int(us_str)
+ zonestr = res.group(8)
+ zonespl = (0, 0) if zonestr in ['z', 'Z'] else [int(i) for i in zonestr.split(':')]
+ zonediff = datetime.timedelta(minutes = zonespl[0]*60+zonespl[1])
+ return datetime.datetime(year, month, day, hour, minute, second, us_int) - zonediff
+ else:
+ raise ValueError("Invalid datetime '{:s}'".format(val))
+
+
+#-------------------------------------------------------------------------------
+
+
+class ZenScriptException(pyzenkit.baseapp.ZenAppProcessException):
"""
Describes problems specific to scripts.
"""
pass
+
+#-------------------------------------------------------------------------------
+
+
class ZenScript(pyzenkit.baseapp.BaseApp):
"""
- Base implementation of generic one time execution script with cron support.
+ Base implementation of generic one-time execution script with built-in regular
+ execution interval support.
"""
#
# Class constants.
#
- # String patterns
+ # String patterns.
PTRN_COMMAND_CBK = 'cbk_command_'
- # List of possible configuration keys.
- CONFIG_REGULAR = 'regular'
- CONFIG_SHELL = 'shell'
- CONFIG_INTERVAL = 'interval'
- CONFIG_COMMAND = 'command'
+ # List of configuration keys.
+ CONFIG_REGULAR = 'regular'
+ CONFIG_SHELL = 'shell'
+ CONFIG_COMMAND = 'command'
+ CONFIG_INTERVAL = 'interval'
+ CONFIG_ADJUST_THRESHOLDS = 'adjust_thresholds'
+ CONFIG_TIME_HIGH = 'time_high'
+
+ # List of runlog keys.
+ RLKEY_COMMAND = 'command'
+
+
+ #---------------------------------------------------------------------------
+
def _init_argparser(self, **kwargs):
"""
- Initialize script command line argument parser.
+ Initialize application command line argument parser. This method overrides
+ the base implementation in :py:func:`baseapp.BaseApp._init_argparser` and
+ it must return valid :py:class:`argparse.ArgumentParser` object.
+
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
+
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Initialized argument parser object.
+ :rtype: argparse.ArgumentParser
"""
argparser = super()._init_argparser(**kwargs)
- # Setup mutually exclusive group.
- group_a = argparser.add_mutually_exclusive_group()
- # Option flag indicating that the script was executed via CRON tool.
- # This option will make sure, that no output will be produced to terminal.
- group_a.add_argument('--regular', help = 'regular script execution (flag)', action='store_true', default = None)
+ #
+ # Create and populate options group for common script arguments.
+ #
+ arggroup_script = argparser.add_argument_group('common script arguments')
- # Option flag indicating that the script was executed manually via terminal.
- # This optional will make sure, that no changes will be made in 'log',
- # 'runlog' or 'state' files.
- group_a.add_argument('--shell', help = 'manual script execution from shell (flag)', action = 'store_true', default = None)
+ # Setup mutually exclusive group for regular x shell mode option.
+ group_a = arggroup_script.add_mutually_exclusive_group()
- # Option for setting the interval for regular script runs.
- argparser.add_argument('--interval', help = 'define interval for regular executions', choices = RUN_INTERVALS.keys())
+ group_a.add_argument('--regular', help = 'operational mode: regular script execution (flag)', action='store_true', default = None)
+ group_a.add_argument('--shell', help = 'operational mode: manual script execution from shell (flag)', action = 'store_true', default = None)
- # Option for setting the desired command.
- argparser.add_argument('--command', help = 'choose which command should be performed', choices = self._utils_detect_commands())
+ arggroup_script.add_argument('--command', help = 'name of the script command to be executed', choices = self._utils_detect_commands(), type = str, default = None)
+ arggroup_script.add_argument('--interval', help = 'time interval for regular executions', choices = RUN_INTERVALS.keys(), type = str, default = None)
+ arggroup_script.add_argument('--adjust-thresholds', help = 'round-up time interval threshols to interval size (flag)', action = 'store_true', default = None)
+ arggroup_script.add_argument('--time-high', help = 'upper time interval threshold', type = float, default = None)
return argparser
- def _init_config(self, **kwargs):
- """
- Initialize script configurations to default values.
+ def _init_config(self, cfgs, **kwargs):
"""
- config = super()._init_config(**kwargs)
+ Initialize default application configurations. This method overrides the
+ base implementation in :py:func:`baseapp.BaseApp._init_argparser` and it
+ adds additional configurations via ``cfgs`` parameter.
+
+ Gets called from main constructor :py:func:`BaseApp.__init__`.
+ :param list cfgs: Additional set of configurations.
+ :param kwargs: Various additional parameters passed down from constructor.
+ :return: Default configuration structure.
+ :rtype: dict
+ """
cfgs = (
- (self.CONFIG_REGULAR, False),
- (self.CONFIG_SHELL, False),
- (self.CONFIG_INTERVAL, None),
- (self.CONFIG_COMMAND, self.get_default_command()),
- )
- for c in cfgs:
- config[c[0]] = kwargs.pop('default_' + c[0], c[1])
- return config
+ (self.CONFIG_REGULAR, False),
+ (self.CONFIG_SHELL, False),
+ (self.CONFIG_INTERVAL, None),
+ (self.CONFIG_COMMAND, self.get_default_command()),
+ (self.CONFIG_ADJUST_THRESHOLDS, False),
+ (self.CONFIG_TIME_HIGH, time.time()),
+ ) + cfgs
+ return super()._init_config(cfgs, **kwargs)
+
+ def _configure_postprocess(self):
+ """
+ Perform configuration postprocessing and calculate core configurations.
+ This method overrides the base implementation in :py:func:`baseapp.BaseApp._configure_postprocess`.
+
+ Gets called from :py:func:`BaseApp._stage_setup_configuration`.
+ """
+ super()._configure_postprocess()
+
+ if self.c(self.CONFIG_SHELL):
+ self.config[self.CORE][self.CORE_LOGGING][self.CORE_LOGGING_TOFILE] = False
+ self.dbgout("Logging to log file is explicitly suppressed by '--shell' configuration")
+
+ self.config[self.CORE][self.CORE_RUNLOG][self.CORE_RUNLOG_SAVE] = False
+ self.dbgout("Runlog saving is explicitly suppressed by '--shell' configuration")
+
+ self.config[self.CORE][self.CORE_PSTATE][self.CORE_PSTATE_SAVE] = False
+ self.dbgout("Persistent state saving is explicitly suppressed by '--shell' configuration")
+
+ elif self.c(self.CONFIG_REGULAR):
+ self.config[self.CONFIG_QUIET] = True
+ self.dbgout("Console output is explicitly suppressed by '--regular' configuration")
+
+ self.config[self.CORE][self.CORE_LOGGING][self.CORE_LOGGING_LEVELC] = 'WARNING'
+ self.dbgout("Console logging level is explicitly forced to 'warning' by '--regular' configuration")
+
+ self.config[self.CORE][self.CORE_LOGGING][self.CORE_LOGGING_TOFILE] = True
+ self.dbgout("Logging to log file is explicitly forced by '--regular' configuration")
+
+ self.config[self.CORE][self.CORE_RUNLOG][self.CORE_RUNLOG_SAVE] = True
+ self.dbgout("Runlog saving is explicitly forced by '--regular' configuration")
+
+ self.config[self.CORE][self.CORE_PSTATE][self.CORE_PSTATE_SAVE] = True
+ self.dbgout("Persistent state saving is explicitly forced by '--regular' configuration")
+
+ else:
+ self.config[self.CORE][self.CORE_LOGGING][self.CORE_LOGGING_TOFILE] = True
+ self.config[self.CORE][self.CORE_RUNLOG][self.CORE_RUNLOG_SAVE] = True
+ self.config[self.CORE][self.CORE_PSTATE][self.CORE_PSTATE_SAVE] = True
+
+ def _sub_stage_process(self):
+ """
+ **SUBCLASS HOOK**: Perform some actual processing in **process** stage.
+ """
+ # Determine, which command to execute.
+ cmdname = self.c(self.CONFIG_COMMAND)
+ self.runlog[self.RLKEY_COMMAND] = cmdname
+
+ # Execute.
+ self.execute_script_command(cmdname)
+
#---------------------------------------------------------------------------
+
def _utils_detect_commands(self):
"""
Returns the sorted list of all available commands current script is capable
of performing. The detection algorithm is based on string analysis of all
- available methods. Any method starting with string 'cbk_command_' will
- be appended to the list, lowercased and with '_' characters replaced with '-'.
+ available methods. Any method starting with string ``cbk_command_`` will
+ be appended to the list, lowercased and with ``_`` characters replaced with ``-``.
"""
ptrn = re.compile(self.PTRN_COMMAND_CBK)
attrs = sorted(dir(self))
result = []
- for a in attrs:
- if not callable(getattr(self, a)):
+ for atr in attrs:
+ if not callable(getattr(self, atr)):
continue
- match = ptrn.match(a)
+ match = ptrn.match(atr)
if match:
- result.append(a.replace(self.PTRN_COMMAND_CBK,'').replace('_','-').lower())
+ result.append(atr.replace(self.PTRN_COMMAND_CBK,'').replace('_','-').lower())
return result
+
+ #---------------------------------------------------------------------------
+
+
def get_default_command(self):
"""
- Return the name of the default operation. This method must be present and
- overriden in subclass and must return the name of desired default operation.
- Following code is just a reminder for programmer to not forget to implement
+ Return the name of the default command. This method must be present and
+ overriden in subclass and must return the name of desired default command.
+ Following code is just a reminder for developer to not forget to implement
this method.
- """
- raise Exception("get_default_command() method must be implemented in subclass")
- def calculate_interval_thresholds(self, thr_type = 'daily', time_cur = None, flag_floor = False, merge_count = 1, skip_count = 0, last_ts = None):
+ :return: Name of the default command.
+ :rtype: str
"""
- Calculate time thresholds based on following optional arguments:
- """
- if not thr_type in RUN_INTERVALS:
- raise Exception("Invalid threshold interval '{}'".format(thr_type))
- interval = RUN_INTERVALS[thr_type]
-
- time_l = 0 # Lower threshold.
- time_h = 0 # Upper threshold.
-
- # Define the upper interval threshold as current timestamp, or use the
- # one given as argument.
- time_h = time_cur
- if not time_h:
- time_h = math.floor(time.time());
-
- # Adjust the upper interval threshold.
- if flag_floor:
- time_h = time_h - (time_h % interval)
+ raise NotImplementedError("This method must be implemented in subclass")
- # Calculate the lower time threshold.
- time_l = time_h - interval
-
- return (time_l, time_h);
+ def execute_script_command(self, command_name):
+ """
+ Execute given script command and store the received results into script runlog.
- #---------------------------------------------------------------------------
+ Following method will call appropriate callback method to service the
+ requested script command.
- def _configure_postprocess(self):
- """
- Setup internal script core mechanics.
+ Name of the callback method is generated from the name of the command by
+ prepending string ``cbk_command_`` and replacing all ``-`` with ``_``.
"""
- super()._configure_postprocess()
-
- if self.c(self.CONFIG_SHELL):
- self.dbgout("[STATUS] Logging to log file is suppressed via '--shell' configuration")
- self.config[self.CORE][self.CORE_LOGGING][self.CORE_LOGGING_TOFILE] = False
- self.dbgout("[STATUS] Runlog saving is suppressed via '--shell' configuration")
- self.config[self.CORE][self.CORE_RUNLOG][self.CORE_RUNLOG_SAVE] = False
- self.dbgout("[STATUS] Persistent state saving is suppressed via '--shell' configuration")
- self.config[self.CORE][self.CORE_PSTATE][self.CORE_PSTATE_SAVE] = False
+ command_name = command_name.lower().replace('-','_')
+ command_cbkname = '{}{}'.format(self.PTRN_COMMAND_CBK, command_name)
+ self.dbgout("Executing callback '{}' for script command '{}'".format(command_cbkname, command_name))
+
+ cbk = getattr(self, command_cbkname, None)
+ if cbk:
+ self.logger.info("Executing script command '%s'", command_name)
+ self.runlog[command_name] = cbk()
else:
- self.config[self.CORE][self.CORE_LOGGING][self.CORE_LOGGING_TOFILE] = True
- self.config[self.CORE][self.CORE_RUNLOG][self.CORE_RUNLOG_SAVE] = True
- self.config[self.CORE][self.CORE_PSTATE][self.CORE_PSTATE_SAVE] = True
+ raise ZenScriptException("Invalid script command '{}', callback '{}' does not exist".format(command_name, command_cbkname))
- def stage_process(self):
+ def calculate_interval_thresholds(self, time_high = None, interval = 'daily', adjust = False):
"""
- Script lifecycle stage: PROCESSING
-
- Perform some real work (finally). Following method will call appropriate
- callback method operation to service the selected operation.
+ Calculate time interval thresholds based on given upper time interval boundary and
+ time interval size.
+
+ :param int time_high: Unix timestamp for upper time threshold.
+ :param str interval: Time interval, one of the interval defined in :py:mod:`pyzenkit.zenscript`.
+ :param bool adjust: Adjust time thresholds to round values (floor).
+ :return: Lower and upper time interval boundaries.
+ :rtype: tuple of datetime.datetime
"""
- self.time_mark('stage_process_start', 'Start of the processing stage')
+ if interval not in RUN_INTERVALS:
+ raise ValueError("Invalid time interval '{}', valid values are: '{}'".format(interval, ','.join(RUN_INTERVALS.keys())))
+ interval_delta = RUN_INTERVALS[interval]
- try:
- # Determine, which command to execute.
- self.runlog[self.RLKEY_COMMAND] = self.c(self.CONFIG_COMMAND)
- opname = self.c(self.CONFIG_COMMAND)
- opcbkname = self.PTRN_COMMAND_CBK + opname.lower().replace('-','_')
- self.logger.debug("Performing script command '{}' with method '{}'".format(opname, opcbkname))
+ if not time_high:
+ time_high = time.time()
- cbk = getattr(self, opcbkname, None)
- if cbk:
- self.logger.info("Executing command '{}'".format(opname))
- self.runlog[opname] = cbk()
- else:
- raise pyzenkit.baseapp.ZenAppProcessException("Invalid command '{}', callback '{}' does not exist".format(opname, opcbkname))
+ time_high = t_datetime(time_high)
+ time_low = time_high - datetime.timedelta(seconds=interval_delta)
+ self.logger.debug("Calculated time interval thresholds: '%s' -> '%s' (%s, %i -> %i)", str(time_low), str(time_high), interval, time_low.timestamp(), time_high.timestamp())
- except subprocess.CalledProcessError as err:
- self.error("System command error: {}".format(err))
+ if adjust:
+ ts_h = t_datetime(time_high.timestamp() - (time_high.timestamp() % interval_delta))
+ ts_l = ts_h - datetime.timedelta(seconds=interval_delta)
+ time_high = ts_h
+ time_low = ts_l
+ self.logger.debug("Adjusted time interval thresholds: '%s' -> '%s' (%s, %i -> %i)", str(time_low), str(time_high), interval, time_low.timestamp(), time_high.timestamp())
- except pyzenkit.baseapp.ZenAppProcessException as exc:
- self.error("ZenAppProcessException: {}".format(exc))
+ return (time_low, time_high)
- except pyzenkit.baseapp.ZenAppException as exc:
- self.error("ZenAppException: {}".format(exc))
- self.time_mark('stage_process_stop', 'End of the processing stage')
-
-class _DemoZenScript(ZenScript):
+class DemoZenScript(ZenScript):
"""
Minimalistic class for demonstration purposes.
"""
+ def __init__(self, name = None, description = None):
+ """
+ Initialize demonstration script. This method overrides the base
+ implementation in :py:func:`baseapp.BaseApp.__init__` and it aims to
+ even more simplify the script object creation.
+
+ :param str name: Optional script name.
+ :param str description: Optional script description.
+ """
+ name = 'demo-zenscript.py' if not name else name
+ description = 'DemoZenScript - Demonstration script' if not description else description
+
+ super().__init__(
+ name = name,
+ description = description,
+
+ #
+ # Configure required application paths to harmless locations.
+ #
+ path_bin = '/tmp',
+ path_cfg = '/tmp',
+ path_log = '/tmp',
+ path_tmp = '/tmp',
+ path_run = '/tmp'
+ )
+
def get_default_command(self):
"""
Return the name of a default script operation.
@@ -230,36 +417,67 @@ class _DemoZenScript(ZenScript):
def cbk_command_default(self):
"""
- Default script operation.
+ Default script command.
"""
- # Log something to show we have reached this point of execution.
- self.logger.info("Demo implementation for default command")
+ # Update the persistent state to view the changes.
+ self.pstate['counter'] = self.pstate.get('counter', 0) + 1
- # Test direct console output with conjunction with verbosity
+ # Log something to show we have reached this point of execution.
+ self.logger.info("Demonstration implementation for default script command")
+ self.logger.info("Try executing this demo with following parameters:")
+ self.logger.info("* python3 pyzenkit/zenscript.py --help")
+ self.logger.info("* python3 pyzenkit/zenscript.py --verbose")
+ self.logger.info("* python3 pyzenkit/zenscript.py --verbose --verbose")
+ self.logger.info("* python3 pyzenkit/zenscript.py --verbose --verbose --verbose")
+ self.logger.info("* python3 pyzenkit/zenscript.py --debug")
+ self.logger.info("* python3 pyzenkit/zenscript.py --log-level debug")
+ self.logger.info("* python3 pyzenkit/zenscript.py --pstate-dump")
+ self.logger.info("* python3 pyzenkit/zenscript.py --runlog-dump")
+ self.logger.info("* python3 pyzenkit/zenscript.py --command alternative")
+ self.logger.info("Number of runs from persistent state: '%d'", self.pstate.get('counter'))
+
+ # Test direct console output with conjunction with verbosity levels.
self.p("Hello world")
self.p("Hello world, verbosity level 1", 1)
self.p("Hello world, verbosity level 2", 2)
self.p("Hello world, verbosity level 3", 3)
+ return { 'result': self.RESULT_SUCCESS, 'data': 5 }
+
+ def cbk_command_alternative(self):
+ """
+ Alternative script command.
+ """
# Update the persistent state to view the changes.
self.pstate['counter'] = self.pstate.get('counter', 0) + 1
- return self.RESULT_SUCCESS
+ # Log something to show we have reached this point of execution.
+ self.logger.info("Demonstration implementation for alternative script command")
+ self.logger.info("Number of runs from persistent state: '%d'", self.pstate.get('counter'))
+ # Test direct console output with conjunction with verbosity levels.
+ self.p("Hello world")
+ self.p("Hello world, verbosity level 1", 1)
+ self.p("Hello world, verbosity level 2", 2)
+ self.p("Hello world, verbosity level 3", 3)
+
+ return { 'result': self.RESULT_SUCCESS, 'data': 100 }
+
+
+#-------------------------------------------------------------------------------
+
+#
+# Perform the demonstration.
+#
if __name__ == "__main__":
- """
- Perform the demonstration.
- """
- # Prepare the environment
- if not os.path.isdir('/tmp/zenscript.py'):
- os.mkdir('/tmp/zenscript.py')
- pyzenkit.baseapp.BaseApp.json_save('/tmp/zenscript.py.conf', {'test_a':1})
- script = _DemoZenScript(
- path_cfg = '/tmp',
- path_log = '/tmp',
- path_tmp = '/tmp',
- path_run = '/tmp',
- description = 'DemoZenScript - generic script (DEMO)'
- )
- script.run()
+ # Prepare demonstration environment.
+ SCR_NAME = 'demo-zenscript.py'
+ pyzenkit.baseapp.BaseApp.json_save('/tmp/{}.conf'.format(SCR_NAME), {'test_a':1})
+ try:
+ os.mkdir('/tmp/{}'.format(SCR_NAME))
+ except FileExistsError:
+ pass
+
+ ZENSCRIPT = DemoZenScript(SCR_NAME)
+ ZENSCRIPT.run()
diff --git a/setup.py b/setup.py
index 33d82ce223c4d4c74f7b4372fd1746ae2f9720d2..d0a5c1de22e94e414b34537186f7f5d5d3271965 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,7 @@ with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
setup(
name = 'pyzenkit',
- version = '0.20',
+ version = '0.32',
description = 'Python 3 script and daemon toolkit',
long_description = long_description,
classifiers = [