From 5fdfa23476eea552f322192b1794e86bfe9d4f30 Mon Sep 17 00:00:00 2001
From: Jan Mach <jan.mach@cesnet.cz>
Date: Wed, 31 May 2017 12:54:52 +0200
Subject: [PATCH] Code quality improvements in executable files in bin project
subdirectory.
Greatly improved code quality using pylint and pyflakes. All files now rank above 9 points with custom .pylintrc-bin file. Following exceptions were added to default pylint configuration:
* line-too-long - I do not want to have calls to logging service spanning over multiple lines and distracting from code flow. I want them to be only single line, no matter how long, because it is not important.
* bad-whitespace - I like to add additional whitespaces in places, where they should not be, mostly to align related items vertically, which in my opinion improves readability.
* logging-format-interpolation - In Python3 I prefer string.format() to string % (args).
Additionally the pattern for module name validation was altered, because of our executable file naming policy (added suppot for '-' and '.' characters in name).
(Redmine issue: #3443)
---
.pylintrc-bin | 425 +++++++++++++++++++++++++++++++++++++++
bin/mentat-backup.py | 116 ++++++-----
bin/mentat-cleanup.py | 117 ++++++-----
bin/mentat-controller.py | 229 +++++++++++----------
bin/mentat-dbmngr.py | 113 ++++++-----
bin/mentat-ideagen.py | 88 ++++----
bin/mentat-inspector.py | 48 ++---
bin/mentat-netmngr.py | 118 +++++------
bin/mentat-sampler.py | 67 +++---
bin/mentat-storage.py | 60 +++---
10 files changed, 938 insertions(+), 443 deletions(-)
create mode 100644 .pylintrc-bin
diff --git a/.pylintrc-bin b/.pylintrc-bin
new file mode 100644
index 000000000..93125393e
--- /dev/null
+++ b/.pylintrc-bin
@@ -0,0 +1,425 @@
+[MASTER]
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code
+extension-pkg-whitelist=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Add files or directories matching the regex patterns to the blacklist. The
+# regex matches against base names, not paths.
+ignore-patterns=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Use multiple processes to speed up Pylint.
+jobs=1
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# Specify a configuration file.
+#rcfile=
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=line-too-long,bad-whitespace,logging-format-interpolation,print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time (only on the command line, not in the configuration file where
+# it should appear only once). See also the "--disable" option for examples.
+enable=
+
+
+[REPORTS]
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+# Set the output format. Available formats are text, parseable, colorized, json
+# and msvs (visual studio).You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Tells whether to display a full report or only the messages
+reports=no
+
+# Activate the evaluation score.
+score=yes
+
+
+[REFACTORING]
+
+# Maximum number of nested blocks for function / method body
+max-nested-blocks=5
+
+
+[BASIC]
+
+# Naming hint for argument names
+argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct argument names
+argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Naming hint for attribute names
+attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct attribute names
+attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Naming hint for class attribute names
+class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Naming hint for class names
+class-name-hint=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression matching correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Naming hint for constant names
+const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression matching correct constant names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# Naming hint for function names
+function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct function names
+function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# Naming hint for inline iteration names
+inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Naming hint for method names
+method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct method names
+method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Naming hint for module names
+module-name-hint=(([a-z_][-.a-z0-9_]*)|([A-Z][-a-zA-Z0-9]+))$
+
+# Regular expression matching correct module names
+module-rgx=(([a-z_][-a-z0-9_]*)|([A-Z][-.a-zA-Z0-9]+))$
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=^_
+
+# List of decorators that produce properties, such as abc.abstractproperty. Add
+# to this list to register other decorators that produce valid properties.
+property-classes=abc.abstractproperty
+
+# Naming hint for variable names
+variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+# Regular expression matching correct variable names
+variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$
+
+
+[VARIABLES]
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+# Tells whether unused global variables should be treated as a violation.
+allow-global-unused-variables=yes
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,_cb
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*|^ignored_|^unused_
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# List of qualified module names which can have objects that can redefine
+# builtins.
+redefining-builtins-modules=six.moves,future.builtins
+
+
+[SIMILARITIES]
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[FORMAT]
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string=' '
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Maximum number of lines in a module
+max-module-lines=1000
+
+# List of optional constructs for which whitespace checking is disabled. `dict-
+# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
+# `trailing-comma` allows a space between comma and closing bracket: (a, ).
+# `empty-line` allows space-only lines.
+no-space-check=trailing-comma,dict-separator
+
+# Allow the body of a class to be on the same line as the declaration if body
+# contains single statement.
+single-line-class-stmt=no
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+
+[TYPECHECK]
+
+# List of decorators that produce context managers, such as
+# contextlib.contextmanager. Add to this list to register other decorators that
+# produce valid context managers.
+contextmanager-decorators=contextlib.contextmanager
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# This flag controls whether pylint should warn about no-member and similar
+# checks whenever an opaque object is returned when inferring. The inference
+# can return multiple potential results while evaluating a Python object, but
+# some branches might not be evaluated, which results in partial inference. In
+# that case, it might be useful to still emit no-member and other checks for
+# the rest of the inferred objects.
+ignore-on-opaque-inference=yes
+
+# List of class names for which member attributes should not be checked (useful
+# for classes with dynamically set attributes). This supports the use of
+# qualified names.
+ignored-classes=optparse.Values,thread._local,_thread._local
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis. It
+# supports qualified module names, as well as Unix pattern matching.
+ignored-modules=
+
+# Show a hint with possible names when a member name was not found. The aspect
+# of finding the hint is based on edit distance.
+missing-member-hint=yes
+
+# The minimum edit distance a name should have in order to be considered a
+# similar match for a missing member name.
+missing-member-hint-distance=1
+
+# The total number of similar names that should be taken in consideration when
+# showing a hint for a missing member.
+missing-member-max-choices=1
+
+
+[SPELLING]
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=5
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Maximum number of boolean expressions in a if statement
+max-bool-expr=5
+
+# Maximum number of branch for function / method body
+max-branches=12
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+
+[IMPORTS]
+
+# Allow wildcard imports from modules that define __all__.
+allow-wildcard-with-all=no
+
+# Analyse import fallback blocks. This can be used to support both Python 2 and
+# 3 compatible code, which means that the block might have code that exists
+# only in one or another interpreter, leading to false positives when analysed.
+analyse-fallback-blocks=no
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=optparse,tkinter.tix
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+# Force import order to recognize a module as part of the standard
+# compatibility libraries.
+known-standard-library=
+
+# Force import order to recognize a module as part of a third party library.
+known-third-party=enchant
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception
diff --git a/bin/mentat-backup.py b/bin/mentat-backup.py
index 14e18fe12..7a739d3af 100755
--- a/bin/mentat-backup.py
+++ b/bin/mentat-backup.py
@@ -7,12 +7,22 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
+
+"""
+Script providing Mentat system backup functions and features.
+"""
+
+
+__version__ = "0.1"
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
import os
import re
import time
import math
import glob
-import logging
import pprint
#
@@ -39,12 +49,9 @@ COLLECTIONS = (
class MentatBackupScript(pyzenkit.zenscript.ZenScript):
"""
- Script providing Mentat system backup functions and features
+ Script providing Mentat system backup functions and features.
"""
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
def get_default_command(self):
"""
Return the name of a default script operation.
@@ -79,7 +86,7 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
"""
# First calculate thresholds for collection backups.
flf = self.config.get('regular', False)
- (time_l, time_h) = self.calculate_interval_thresholds(self.config['interval'], flag_floor = flf);
+ (time_l, time_h) = self.calculate_interval_thresholds(self.config['interval'], flag_floor = flf)
self.logger.info("Lower backup period threshold: {} ({})".format(time.strftime('%Y-%m-%d %H:%M', time.localtime(time_l)), time_l))
self.logger.info("Upper backup period threshold: {} ({})".format(time.strftime('%Y-%m-%d %H:%M', time.localtime(time_h)), time_h))
@@ -96,7 +103,7 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
self.logger.debug("Backup file name: '{}'".format(backup_file))
# Perform actual backup.
- result = self._backup_database(dir_tmp, backup_file, time_l, time_h, self.pstate)
+ result = self._backup_database(dir_tmp, backup_file, time_l, time_h)
if not self.config['no_upload']:
# Mount the remote directory to local directory.
@@ -153,11 +160,11 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Latest backup is from '{}'".format(result['backups'][0]['ts_str']))
self.logger.info("Oldest backup is from '{}'".format(result['backups'][-1]['ts_str']))
previous = None
- for bi in result['backups']:
- if previous:
- if bi['step'] > 86400:
- self.logger.info("Missing '{:,d}' backup files between '{}' and '{}'".format(math.floor((bi['step'] / 86400) - 1), previous['ts_str'], bi['ts_str']))
- previous = bi
+ for bkp in result['backups']:
+ if previous is not None:
+ if bkp['step'] > 86400:
+ self.logger.info("Missing '{:,d}' backup files between '{}' and '{}'".format(math.floor((bkp['step'] / 86400) - 1), previous['ts_str'], bkp['ts_str']))
+ previous = bkp
else:
self.logger.warning("There are no backup files")
@@ -171,7 +178,7 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
# INTERNAL METHODS
#---------------------------------------------------------------------------
- def _backup_database(self, dir_tmp, backup_file, time_l, time_h, state):
+ def _backup_database(self, dir_tmp, backup_file, time_l, time_h):
"""
Perform backup of all configured database collections.
"""
@@ -190,26 +197,26 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
(coll, ts_attr, system) = rule
if system:
self.logger.info("Performing backup of system collection '{}'".format(coll))
- cmd = "mongodump -o {} -d '{}' -c '{}'".format(dir_tmp, 'mentat', coll)
+ cmd = r"mongodump -o {} -d '{}' -c '{}'".format(dir_tmp, 'mentat', coll)
result['dumpcommands'].append(cmd)
self.execute_command(cmd)
else:
self.logger.info("Performing backup of collection '{}'".format(coll))
- cmd = "mongodump -o {} -d '{}' -c '{}' -q '{{\"{}\": {{\"\$gte\": {}, \"\$lt\": {}}}}}'".format(dir_tmp, 'mentat', coll, ts_attr, time_l, time_h)
+ cmd = r"mongodump -o {} -d '{}' -c '{}' -q '{{\"{}\": {{\"\$gte\": {}, \"\$lt\": {}}}}}'".format(dir_tmp, 'mentat', coll, ts_attr, time_l, time_h)
result['dumpcommands'].append(cmd)
self.execute_command(cmd)
self.logger.info("Packing the database backup into archive file '{}'".format(backup_file))
- cmd = "tar -czPf {} {}".format(backup_file, dir_tmp)
+ cmd = r"tar -czPf {} {}".format(backup_file, dir_tmp)
self.execute_command(cmd)
self.logger.info("Removing local backup directory '{}'".format(dir_tmp))
- cmd = "rm -rf {}".format(dir_tmp)
+ cmd = r"rm -rf {}".format(dir_tmp)
self.execute_command(cmd)
- si = os.stat(backup_file)
- result['backup_file'] = backup_file,
- result['backup_size'] = si.st_size
+ sti = os.stat(backup_file)
+ result['backup_file'] = backup_file
+ result['backup_size'] = sti.st_size
self.logger.info("Created backup file '{}' with size {:,.2f} KB".format(result['backup_file'], (result['backup_size']/1024)))
return result
@@ -234,12 +241,12 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
"""
backup_files = sorted(glob.glob(backup_mask), reverse = True)
self.logger.debug("Found following local backup files >>>\n{}".format(pprint.pformat(backup_files, indent=4)))
- for bf in backup_files:
- if bf == backup_file:
+ for bfl in backup_files:
+ if bfl == backup_file:
self.logger.info("Keeping local copy of latest backup file '{}'".format(backup_file))
else:
- self.logger.info("Removing local copy of older backup file '{}'".format(bf))
- os.remove(bf)
+ self.logger.info("Removing local copy of older backup file '{}'".format(bfl))
+ os.remove(bfl)
def _analyze_remote(self, mount_point, mode):
"""
@@ -251,33 +258,33 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
if os.path.isdir(tgtdir):
self.logger.info("Analyzing remote backup directory '{}':".format(tgtdir))
backupfiles = os.listdir(tgtdir)
- ptrn = re.compile('mentatdb_' + mode + '_(\d{12})\.tar\.gz')
- for f in sorted(backupfiles, reverse=True):
- fn = os.path.join(tgtdir, f)
- if not os.path.isfile(fn):
+ ptrn = re.compile('mentatdb_' + mode + r'_(\d{12})\.tar\.gz')
+ for bfl in sorted(backupfiles, reverse=True):
+ bfp = os.path.join(tgtdir, bfl)
+ if not os.path.isfile(bfp):
continue
- m = ptrn.match(f)
- if not m:
+ match = ptrn.match(bfl)
+ if not match:
continue
- a = {
- 'file': f,
- 'path': fn,
- 'ts_raw': m.group(1),
- 'ts_str': time.strftime('%Y-%m-%d', time.strptime(m.group(1), '%Y%m%d%H%M')),
- 'ts': time.mktime(time.strptime(m.group(1), '%Y%m%d%H%M'))
+ anl = {
+ 'file': bfl,
+ 'path': bfp,
+ 'ts_raw': match.group(1),
+ 'ts_str': time.strftime('%Y-%m-%d', time.strptime(match.group(1), '%Y%m%d%H%M')),
+ 'ts': time.mktime(time.strptime(match.group(1), '%Y%m%d%H%M'))
}
- si = os.stat(fn)
- a['size'] = si.st_size
+ sti = os.stat(bfp)
+ anl['size'] = sti.st_size
if not previous:
- a['step'] = 0
- a['value'] = 1
+ anl['step'] = 0
+ anl['value'] = 1
else:
- a['step'] = int(previous['ts'] - a['ts'])
- a['value'] = int(previous['value'] + (a['step'] / 86400))
+ anl['step'] = int(previous['ts'] - anl['ts'])
+ anl['value'] = int(previous['value'] + (anl['step'] / 86400))
- previous = a
+ previous = anl
- result['backups'].append(a)
+ result['backups'].append(anl)
self.logger.debug("Backup analysis: {}".format(pprint.pformat(result)))
else:
self.logger.error("Remote backup directory '{}' does not exist".format(tgtdir))
@@ -308,16 +315,27 @@ class MentatBackupScript(pyzenkit.zenscript.ZenScript):
else:
self.logger.info("Remote storage already unmounted from path '{}'".format(mount_point))
+
+#
+# Execute the script.
+#
if __name__ == "__main__":
- """
- Execute the MentatBackupScript script.
- """
- script = MentatBackupScript(
+
+ SCRIPT = MentatBackupScript(
+
+ description = 'mentat-netmngr.py - Abuse group network management script for Mentat system database',
+
+ #
+ # Configure required script paths.
+ #
path_cfg = '/etc/mentat',
path_log = '/var/mentat/log',
path_run = '/var/mentat/run',
path_tmp = '/tmp',
+ #
+ # Override default configurations.
+ #
default_config_dir = '/etc/mentat/core',
)
- script.run()
+ SCRIPT.run()
diff --git a/bin/mentat-cleanup.py b/bin/mentat-cleanup.py
index ca3ed7c2f..0136e8cf9 100755
--- a/bin/mentat-cleanup.py
+++ b/bin/mentat-cleanup.py
@@ -7,15 +7,22 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
+
+"""
+Script providing Mentat system cleanup functions and features.
+"""
+
+
+__version__ = "0.1"
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
import os
-import re
import sys
import time
-import math
-import glob
import json
-import logging
-import pprint
+
#
# Custom libraries
@@ -24,6 +31,7 @@ import pyzenkit.zenscript
import pydgets.widgets
import mentat.storage
+
#
# Global variables.
#
@@ -31,9 +39,11 @@ SECS_DAY = 60 * 60 * 24 # Number of seconds in a day
SECS_WEEK = SECS_DAY * 7 # Number of seconds in a week
SECS_YEAR = 31556926 # Number of seconds in a year (approximately)
+
# Current time (second precission)
TS = int(time.time())
+
# List of possible item TTLs
TTLS = {
'y': SECS_YEAR,
@@ -78,10 +88,12 @@ THRESHOLDS = {
'd': {'l': 'items older than one day', 't': (TS - TTLS['d'])},
}
+
class MentatCleanupScript(pyzenkit.zenscript.ZenScript):
"""
- Script providing Mentat system cleanup functions and features
+ Script providing Mentat system cleanup functions and features.
"""
+
CONFIG_DB_PATH = 'db_path'
CONFIG_SIMULATE = 'simulate'
CONFIG_SOCKET_TIMEOUT = 'socket_timeout'
@@ -118,8 +130,8 @@ class MentatCleanupScript(pyzenkit.zenscript.ZenScript):
(self.CONFIG_COLLECTIONS, []),
(self.CONFIG_CACHES, []),
)
- for c in cfgs:
- config[c[0]] = kwargs.get('default_' + c[0], c[1])
+ for cfg in cfgs:
+ config[cfg[0]] = kwargs.get('default_' + cfg[0], cfg[1])
return config
def _init_custom(self, config, argparser, **kwargs):
@@ -213,25 +225,25 @@ class MentatCleanupScript(pyzenkit.zenscript.ZenScript):
"""
table_columns = [
- { 'label': 'Date' },
- { 'label': 'FS removed [#]', 'data_formating': '{:,d}', 'align': '>' },
- { 'label': 'DB removed [#]', 'data_formating': '{:,d}', 'align': '>' },
- ]
+ { 'label': 'Date' },
+ { 'label': 'FS removed [#]', 'data_formating': '{:,d}', 'align': '>' },
+ { 'label': 'DB removed [#]', 'data_formating': '{:,d}', 'align': '>' },
+ ]
table_data = []
- for an in evaluation[self.RLEVKEY_ANALYSES]:
- clrslt = an.get('cleanup', None)
+ for anl in evaluation[self.RLEVKEY_ANALYSES]:
+ clrslt = anl.get('cleanup', None)
if clrslt:
table_data.append(
[
- an['label'],
- an['cleanup']['caches']['removed_cnt'],
- an['cleanup']['collections']['removed_cnt'],
+ anl['label'],
+ anl['cleanup']['caches']['removed_cnt'],
+ anl['cleanup']['collections']['removed_cnt'],
]
)
else:
table_data.append(
[
- an['label'],
+ anl['label'],
0,
0,
]
@@ -261,14 +273,14 @@ class MentatCleanupScript(pyzenkit.zenscript.ZenScript):
# Perform cleanup od selected database collections
result['collections'] = []
for collection in self.c('collections'):
- r = self._cleanup_collection(**collection)
- result['collections'].append(r)
+ res = self._cleanup_collection(**collection)
+ result['collections'].append(res)
# Perform cleanup of selected folder caches
result['caches'] = []
for cache in self.c('caches'):
- r = self._cleanup_cache(**cache)
- result['caches'].append(r)
+ res = self._cleanup_cache(**cache)
+ result['caches'].append(res)
# Measure disk usage after cleanup
result['fsstats_post'] = self._fsstats(self.c('db_path'))
@@ -279,11 +291,12 @@ class MentatCleanupScript(pyzenkit.zenscript.ZenScript):
# INTERNAL METHODS
#---------------------------------------------------------------------------
- def _fsstats(self, fs):
+ @staticmethod
+ def _fsstats(fpath):
"""
Calculate the filesystem statistics.
"""
- fsstats = os.statvfs(fs)
+ fsstats = os.statvfs(fpath)
return dict(
zip(
('f_bsize', 'f_frsize', 'f_blocks', 'f_bfree', 'f_bavail', 'f_files', 'f_ffree', 'f_favail', 'f_flag', 'f_namemax'),
@@ -321,27 +334,25 @@ class MentatCleanupScript(pyzenkit.zenscript.ZenScript):
result['collection'],
result['threshold'],
result['threshold_str'],
- result['filter'])
- )
+ result['filter']))
- db = self.storage.database(database)
- result['stats_pre'] = db.collection_stats(collection)
+ dbh = self.storage.database(database)
+ result['stats_pre'] = dbh.collection_stats(collection)
if not self.c('simulate'):
- res = db.collection(collection).delete_many(filter_spec)
+ res = dbh.collection(collection).delete_many(filter_spec)
result['removed_cnt'] = res.deleted_count
else:
- result['removed_cnt'] = db.collection(collection).find(filter_spec).count()
+ result['removed_cnt'] = dbh.collection(collection).find(filter_spec).count()
- db = self.storage.database(database)
- result['stats_post'] = db.collection_stats(collection)
+ dbh = self.storage.database(database)
+ result['stats_post'] = dbh.collection_stats(collection)
self.logger.info("Collection '{}.{}' cleanup done, removed: {:,d} | kept: {:,d}".format(
result['database'],
result['collection'],
result['removed_cnt'],
- result['stats_post']['count'])
- )
+ result['stats_post']['count']))
# Except: pymongo.errors.OperationFailure: command SON([('collstats', 'logs')]) on namespace mentat.$cmd failed: Collection [mentat.logs] not found.
except:
@@ -374,39 +385,41 @@ class MentatCleanupScript(pyzenkit.zenscript.ZenScript):
}
self.logger.info("Cache '{}' cleanup started with threshold '{}': '{}'".format(result['cache'], result['threshold'], result['threshold_str']))
- for f in os.listdir(cache):
- fn = os.path.join(cache, f)
- if os.path.isfile(fn):
- st = os.stat(fn)
+ for fln in os.listdir(cache):
+ flp = os.path.join(cache, fln)
+ if os.path.isfile(flp):
+ fst = os.stat(flp)
result['files_cnt'] += 1
- result['files_bytes'] += st.st_size
- if st.st_mtime < threshold['t']:
+ result['files_bytes'] += fst.st_size
+ if fst.st_mtime < threshold['t']:
try:
if not self.c('simulate'):
- os.remove(fn)
+ os.remove(flp)
result['removed_cnt'] += 1
- result['removed_bytes'] += st.st_size
+ result['removed_bytes'] += fst.st_size
except:
- self.error("Unable to remove cache file '{}'".format(fn))
+ self.error("Unable to remove cache file '{}'".format(flp))
result['error_cnt'] += 1
- result['errors'].append(fn)
+ result['errors'].append(flp)
else:
result['kept_cnt'] += 1
- result['kept_bytes'] += st.st_size
+ result['kept_bytes'] += fst.st_size
self.logger.info("Cache '{}' cleanup done, removed: {:,d} | kept: {:,d} | errors: {:,d}".format(result['cache'], result['removed_cnt'], result['kept_cnt'], result['error_cnt']))
return result
+
+#
+# Execute the script.
+#
if __name__ == "__main__":
- """
- Execute the MentatCleanupScript script.
- """
- script = MentatCleanupScript(
+
+ SCRIPT = MentatCleanupScript(
description = 'mentat-cleanup.py - Mentat system database and cache folder cleanup script',
#
- # Configure required daemon paths
+ # Configure required script paths.
#
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
@@ -415,8 +428,8 @@ if __name__ == "__main__":
path_tmp = '/tmp',
#
- # Override default configurations
+ # Override default configurations.
#
default_config_dir = '/etc/mentat/core',
)
- script.run()
+ SCRIPT.run()
diff --git a/bin/mentat-controller.py b/bin/mentat-controller.py
index 78bf501d6..545c59977 100755
--- a/bin/mentat-controller.py
+++ b/bin/mentat-controller.py
@@ -7,6 +7,17 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
+
+"""
+Script providing Mentat system control functions and features.
+"""
+
+
+__version__ = "0.1"
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
import os
import re
import time
@@ -56,8 +67,9 @@ SIGNALS_TO_NAMES_DICT = dict((getattr(signal, n), n) \
class MentatControllerScript(pyzenkit.zenscript.ZenScript):
"""
- Script providing Mentat system control functions and features
+ Script providing Mentat system control functions and features.
"""
+
CONFIG_TARGET = 'target'
def _init_argparser(self, **kwargs):
@@ -80,8 +92,8 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
cfgs = (
(self.CONFIG_TARGET, None),
)
- for c in cfgs:
- config[c[0]] = kwargs.get('default_' + c[0], c[1])
+ for cfg in cfgs:
+ config[cfg[0]] = kwargs.get('default_' + cfg[0], cfg[1])
return config
def _get_target(self):
@@ -104,19 +116,19 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.modules = collections.OrderedDict()
# Analyze and initialize module configurations
- for c in self.c('modules'):
+ for mod in self.c('modules'):
record = {}
# Executable is mandatory attribute
- if c.get('exec'):
- record['exec'] = c['exec']
+ if mod.get('exec'):
+ record['exec'] = mod['exec']
else:
raise pyzenkit.baseapp.ZenAppSetupException("Module definition in configuration file is missing mandatory attribute 'exec'.")
- record['name'] = c.get('name', record['exec'])
- record['args'] = c.get('args', [])
- record['paralel'] = c.get('paralel', False)
- record['count'] = int(c.get('count', 0))
+ record['name'] = mod.get('name', record['exec'])
+ record['args'] = mod.get('args', [])
+ record['paralel'] = mod.get('paralel', False)
+ record['count'] = int(mod.get('count', 0))
if record['paralel'] and not record['count'] > 1:
raise pyzenkit.baseapp.ZenAppSetupException("Module '{}' is configured to run in paralel and the 'count' attribute is missing or below '2'.".format(record['name']))
@@ -131,13 +143,14 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
#---------------------------------------------------------------------------
- def _analyze_pid_file(self, pid_file, pid_file_path):
+ @staticmethod
+ def _analyze_pid_file(pid_file, pid_file_path):
"""
Analyze given PID file.
"""
try:
- m = re.search('^(.+?)(?:\.(\d+))?\.pid$', pid_file)
- if not m:
+ match = re.search(r'^(.+?)(?:\.(\d+))?\.pid$', pid_file)
+ if not match:
return None
pid = None
@@ -146,14 +159,13 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
pid = int(pid)
return {
- "name": m.group(1),
+ "name": match.group(1),
"file": pid_file,
"path": pid_file_path,
"pid": int(pid),
- "paralel": True if m.group(2) else False
+ "paralel": True if match.group(2) else False
}
except:
- daemon.logger.error("Unable to analyze PID file '{}'".format(pid_file_path))
return None
def _analyze_pid_files(self):
@@ -176,33 +188,35 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
pid_files[record['name']][record['pid']] = record
return pid_files
- def _analyze_process_ps(self, pid, process, cmdline):
+ @staticmethod
+ def _analyze_process_ps(pid, process, cmdline):
"""
Analyze given process (detected via ps utility).
"""
- m = re.match('([^\s]*(?:perl|python3))\s+(?:-T\s+)?[^\s]+(mentat-[^\s]+)(?:\s+(.*))?', cmdline)
- if not m:
+ match = re.match(r'([^\s]*(?:perl|python3))\s+(?:-T\s+)?[^\s]+(mentat-[^\s]+)(?:\s+(.*))?', cmdline)
+ if not match:
return None
- if m.group(2) in ['mentat-controller.py', 'mentat-statistician', 'mentat-reporter-ng']:
+ if match.group(2) in ['mentat-controller.py', 'mentat-statistician', 'mentat-reporter-ng']:
return None
record = {
- 'name': m.group(2),
- 'exec': m.group(2),
- 'args': m.group(3),
+ 'process': process,
+ 'name': match.group(2),
+ 'exec': match.group(2),
+ 'args': match.group(3),
'pid': int(pid),
'paralel': False
}
if record['args']:
# Attempt to parse custom name from command line arguments
- m = re.search('--name(?: |=)([^\s]+)', record['args'])
- if m:
- record['name'] = m.group(1)
+ match = re.search(r'--name(?: |=)([^\s]+)', record['args'])
+ if match:
+ record['name'] = match.group(1)
# Attempt to detect paralel mode switch from command line arguments
- m = re.search('--paralel', record['args'])
- if m:
+ match = re.search('--paralel', record['args'])
+ if match:
record['paralel'] = True
return record
@@ -212,23 +226,24 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
Analyze all running processes via ps utility.
"""
processes = collections.defaultdict(dict)
- s = subprocess.Popen(['/bin/ps axo pid,comm,args | grep mentat | grep -v grep'], stdout = subprocess.PIPE, shell = True)
- for line in s.stdout:
+ proc = subprocess.Popen(['/bin/ps axo pid,comm,args | grep mentat | grep -v grep'], stdout = subprocess.PIPE, shell = True)
+ for line in proc.stdout:
line = line.rstrip()
line = line.decode('utf-8')
- m = re.match('\s*(\d+)\s+([^\s]+)\s+(.+)$', line)
- if not m:
+ match = re.match(r'\s*(\d+)\s+([^\s]+)\s+(.+)$', line)
+ if not match:
continue
- record = self._analyze_process_ps(m.group(1), m.group(2), m.group(3))
+ record = self._analyze_process_ps(match.group(1), match.group(2), match.group(3))
if not record:
continue
processes[record['name']][record['pid']] = record
return processes
- def _module_status(self, problems, conf_data, pidf_data, proc_data):
+ @staticmethod
+ def _module_status(problems, conf_data, pidf_data, proc_data):
"""
Analyze status of given module.
"""
@@ -324,11 +339,11 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
# A) Analyze status of configured modules
for mname, mdata in self.modules.items():
status['modules'][mname] = self._module_status(
- status['problems'],
- mdata,
- status['pid_files'].get(mname, None),
- status['processes'].get(mname, None)
- )
+ status['problems'],
+ mdata,
+ status['pid_files'].get(mname, None),
+ status['processes'].get(mname, None)
+ )
# B) Analyze unconfigured PID files
for mname in sorted(status['pid_files'].keys()):
@@ -337,11 +352,11 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
continue
for pid in sorted(status['pid_files'][mname].keys()):
status['problems'].append((
- 'warning',
- 'pidfile',
- status['pid_files'][mname][pid]['path'],
- "Unknown pid file '{}' for PID '{}'".format(status['pid_files'][mname][pid]['path'], pid))
- )
+ 'warning',
+ 'pidfile',
+ status['pid_files'][mname][pid]['path'],
+ "Unknown pid file '{}' for PID '{}'".format(status['pid_files'][mname][pid]['path'], pid)
+ ))
# C) Analyze unconfigured running modules
for mname in sorted(status['processes'].keys()):
@@ -350,11 +365,11 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
continue
for pid in sorted(status['processes'][mname].keys()):
status['problems'].append((
- 'warning',
- 'pidfile',
- pid,
- "Unknown process '{}' with PID '{}'".format(status['processes'][mname][pid]['exec'], pid))
- )
+ 'warning',
+ 'pidfile',
+ pid,
+ "Unknown process '{}' with PID '{}'".format(status['processes'][mname][pid]['exec'], pid)
+ ))
# D) Determine overall status of the whole system
for mname, mstatus in status['modules'].items():
@@ -375,12 +390,12 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Status of configured Mentat modules:")
complist = self._get_target()
for mname in complist:
- s = status['modules'][mname]
- self.logger.info("Module '{}': '{}'".format(mname, s[1]))
+ stat = status['modules'][mname]
+ self.logger.info("Module '{}': '{}'".format(mname, stat[1]))
if status['problems']:
- for p in status['problems']:
- self.logger.warning(p[3])
+ for prob in status['problems']:
+ self.logger.warning(prob[3])
self.logger.info("Overall system status: '{}'".format(status['result'][1]))
self.rc = status['result'][0]
@@ -443,18 +458,18 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Starting all configured Mentat modules:")
complist = self._get_target()
for mname in complist:
- s = status['modules'][mname]
- f = status['pid_files'].get(mname, None)
- p = status['processes'].get(mname, None)
+ stat = status['modules'][mname]
+ pidf = status['pid_files'].get(mname, None)
+ proc = status['processes'].get(mname, None)
- if s[0] == STATUS_RUNNING_OK:
+ if stat[0] == STATUS_RUNNING_OK:
self.logger.info("Module '{}': Module is already running properly, nothing to do".format(mname))
- elif s[0] == STATUS_NOT_RUNNING:
+ elif stat[0] == STATUS_NOT_RUNNING:
self.logger.info("Module '{}': Launching module".format(mname))
- self._module_start(self.modules[mname], f, p)
- elif s[0] == STATUS_RUNNING_FEWER:
+ self._module_start(self.modules[mname], pidf, proc)
+ elif stat[0] == STATUS_RUNNING_FEWER:
self.logger.info("Module '{}': Module is running in fewer than required instances".format(mname))
- elif s[0] == STATUS_RUNNING_MORE:
+ elif stat[0] == STATUS_RUNNING_MORE:
self.logger.info("Module '{}': Module is running in more than required instances".format(mname))
else:
self.logger.error("Module '{}': Module is in weird state, unable to perform automatic startup".format(mname))
@@ -496,20 +511,20 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Stopping all configured Mentat modules:")
complist = self._get_target()
for mname in complist:
- s = status['modules'][mname]
- f = status['pid_files'].get(mname, None)
- p = status['processes'].get(mname, None)
+ stat = status['modules'][mname]
+ pidf = status['pid_files'].get(mname, None)
+ proc = status['processes'].get(mname, None)
- if s[0] == STATUS_NOT_RUNNING:
+ if stat[0] == STATUS_NOT_RUNNING:
self.logger.info("Module '{}': Module is already not running, nothing to do".format(mname))
- elif s[0] in (STATUS_RUNNING_OK,
- STATUS_RUNNING_PF_MISSING,
- STATUS_RUNNING_PID_MISSMATCH,
- STATUS_RUNNING_INCONSISTENT,
- STATUS_RUNNING_FEWER,
- STATUS_RUNNING_MORE):
+ elif stat[0] in (STATUS_RUNNING_OK,
+ STATUS_RUNNING_PF_MISSING,
+ STATUS_RUNNING_PID_MISSMATCH,
+ STATUS_RUNNING_INCONSISTENT,
+ STATUS_RUNNING_FEWER,
+ STATUS_RUNNING_MORE):
self.logger.info("Module '{}': Stopping module".format(mname))
- self._module_signal(self.modules[mname], f, p, signal.SIGINT)
+ self._module_signal(self.modules[mname], pidf, proc, signal.SIGINT)
else:
self.logger.error("Module '{}': Module is in weird state, unable to perform automatic shutdown".format(mname))
@@ -543,20 +558,20 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Killing all configured Mentat modules:")
complist = self._get_target()
for mname in complist:
- s = status['modules'][mname]
- f = status['pid_files'].get(mname, None)
- p = status['processes'].get(mname, None)
+ stat = status['modules'][mname]
+ pidf = status['pid_files'].get(mname, None)
+ proc = status['processes'].get(mname, None)
- if s[0] == STATUS_NOT_RUNNING:
+ if stat[0] == STATUS_NOT_RUNNING:
self.logger.info("Module '{}': Module is already not running, nothing to do".format(mname))
- elif s[0] in (STATUS_RUNNING_OK,
- STATUS_RUNNING_PF_MISSING,
- STATUS_RUNNING_PID_MISSMATCH,
- STATUS_RUNNING_INCONSISTENT,
- STATUS_RUNNING_FEWER,
- STATUS_RUNNING_MORE):
+ elif stat[0] in (STATUS_RUNNING_OK,
+ STATUS_RUNNING_PF_MISSING,
+ STATUS_RUNNING_PID_MISSMATCH,
+ STATUS_RUNNING_INCONSISTENT,
+ STATUS_RUNNING_FEWER,
+ STATUS_RUNNING_MORE):
self.logger.info("Module '{}': Killing module".format(mname))
- self._module_signal(self.modules[mname], f, p, signal.SIGKILL)
+ self._module_signal(self.modules[mname], pidf, proc, signal.SIGKILL)
else:
self.logger.error("Module '{}': Module is in weird state, unable to kill it".format(mname))
@@ -571,14 +586,14 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Sending SIGUSR1 signal to all configured Mentat modules:")
complist = self._get_target()
for mname in complist:
- s = status['modules'][mname]
- f = status['pid_files'].get(mname, None)
- p = status['processes'].get(mname, None)
+ stat = status['modules'][mname]
+ pidf = status['pid_files'].get(mname, None)
+ proc = status['processes'].get(mname, None)
- if not re.search('\.py$', self.modules[mname]['exec']):
+ if not re.search(r'\.py$', self.modules[mname]['exec']):
self.logger.warning("Module '{}': Module is unable to handle this signal".format(mname))
- elif s[0] == STATUS_RUNNING_OK:
- self._module_signal(self.modules[mname], f, p, signal.SIGUSR1)
+ elif stat[0] == STATUS_RUNNING_OK:
+ self._module_signal(self.modules[mname], pidf, proc, signal.SIGUSR1)
else:
self.logger.error("Module '{}': Unable to send signal".format(mname))
@@ -593,14 +608,14 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Sending SIGUSR2 signal to all configured Mentat modules:")
complist = self._get_target()
for mname in complist:
- s = status['modules'][mname]
- f = status['pid_files'].get(mname, None)
- p = status['processes'].get(mname, None)
+ stat = status['modules'][mname]
+ pidf = status['pid_files'].get(mname, None)
+ proc = status['processes'].get(mname, None)
- if not re.search('\.py$', self.modules[mname]['exec']):
+ if not re.search(r'\.py$', self.modules[mname]['exec']):
self.logger.warning("Module '{}': Module is unable to handle this signal".format(mname))
- elif s[0] == STATUS_RUNNING_OK:
- self._module_signal(self.modules[mname], f, p, signal.SIGUSR2)
+ elif stat[0] == STATUS_RUNNING_OK:
+ self._module_signal(self.modules[mname], pidf, proc, signal.SIGUSR2)
else:
self.logger.error("Module '{}': Unable to send signal".format(mname))
@@ -615,12 +630,12 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Sending SIGHUP signal to all configured Mentat modules:")
complist = self._get_target()
for mname in complist:
- s = status['modules'][mname]
- f = status['pid_files'].get(mname, None)
- p = status['processes'].get(mname, None)
+ stat = status['modules'][mname]
+ pidf = status['pid_files'].get(mname, None)
+ proc = status['processes'].get(mname, None)
- if s[0] == STATUS_RUNNING_OK:
- self._module_signal(self.modules[mname], f, p, signal.SIGHUP)
+ if stat[0] == STATUS_RUNNING_OK:
+ self._module_signal(self.modules[mname], pidf, proc, signal.SIGHUP)
else:
self.logger.error("Module '{}': Unable to send signal".format(mname))
@@ -632,17 +647,17 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
# status (return rc according to the overall status)
#
+#
+# Execute the script.
+#
if __name__ == "__main__":
- """
- Execute the script.
- """
- script = MentatControllerScript(
+ SCRIPT = MentatControllerScript(
description = 'mentat-controller.py - Mentat system control script',
#
- # Configure required deamon paths
+ # Configure required script paths.
#
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
@@ -651,8 +666,8 @@ if __name__ == "__main__":
path_tmp = '/tmp',
#
- # Override default configurations
+ # Override default configurations.
#
default_config_dir = '/etc/mentat/core',
)
- script.run()
+ SCRIPT.run()
diff --git a/bin/mentat-dbmngr.py b/bin/mentat-dbmngr.py
index 45872e738..de71930f4 100755
--- a/bin/mentat-dbmngr.py
+++ b/bin/mentat-dbmngr.py
@@ -7,15 +7,17 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
+
+"""
+Script providing Mentat system database management functions and features.
+"""
+
+
import os
import re
import json
import time
-import math
-import glob
-import logging
import pprint
-
import smtplib
from email.mime.text import MIMEText
@@ -27,6 +29,7 @@ import pyzenkit.zenscript
import mentat.const
import mentat.storage
+
class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
"""
Script providing Mentat system database management functions and features.
@@ -49,18 +52,18 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will determine the current status of all databases and its
collections.
"""
- db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
- db_schema = db_config[mentat.const.CKEY_CORE_DATABASE_SCHEMA]
- s = mentat.storage.Storage()
+ db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
+ db_schema = db_config[mentat.const.CKEY_CORE_DATABASE_SCHEMA]
+ db_storage = mentat.storage.Storage()
- for db_n in s.database_names():
+ for db_n in db_storage.database_names():
if db_n in ('admin','local'):
continue
self.logger.info("Inspecting database '{}'".format(db_n))
if not db_n in db_schema:
self.logger.warning("Database '{}' exists, but is not configured".format(db_n))
else:
- db_h = s.database(db_n)
+ db_h = db_storage.database(db_n)
db_c = db_schema[db_n]
for col_n in db_h.collection_names():
if not col_n in db_c['collections']:
@@ -77,7 +80,7 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
if not idx_n in index_list_current:
self.logger.info("Index '{}.{}':'{}' is missing".format(db_n, col_n, idx_n))
- s.close()
+ db_storage.close()
return self.RESULT_SUCCESS
def cbk_command_init(self):
@@ -86,13 +89,13 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will perform all necessary database initializations.
"""
- db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
- db_schema = db_config[mentat.const.CKEY_CORE_DATABASE_SCHEMA]
- s = mentat.storage.Storage()
+ db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
+ db_schema = db_config[mentat.const.CKEY_CORE_DATABASE_SCHEMA]
+ db_storage = mentat.storage.Storage()
for db_n in sorted(db_schema.keys()):
db_c = db_schema[db_n]
- db_h = s.database(db_n)
+ db_h = db_storage.database(db_n)
self.logger.info("Initializing database '{}'".format(db_n))
for col_n in sorted(db_c['collections'].keys()):
@@ -108,7 +111,7 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
else:
self.logger.info("Index '{}.{}':'{}' already exists".format(db_n, col_n, idx['name']))
- s.close()
+ db_storage.close()
return self.RESULT_SUCCESS
def cbk_command_watchdog(self):
@@ -117,10 +120,12 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will attempt to profile various collection data.
"""
- dbconfig = self.c(mentat.const.CKEY_CORE_DATABASE)
- storage = mentat.storage.Storage()
- database = storage.database('mentat')
- collection = database.collection('alerts')
+ db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
+ db_cfg = db_config[mentat.const.CKEY_CORE_DATABASE_CONFIG]
+ db_storage = mentat.storage.Storage()
+
+ database = db_storage.database(db_cfg['db'])
+ collection = database.collection(db_cfg['col_alerts'])
delta = 60 * 60 * 6
last_alert = None
@@ -133,7 +138,7 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
if storage_time:
age = time.time() - storage_time
if age > delta:
- self._email_alert_age(age)
+ self._email_alert(age)
return self.RESULT_SUCCESS
@@ -143,10 +148,12 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will attempt to profile various collection data.
"""
- dbconfig = self.c(mentat.const.CKEY_CORE_DATABASE)
- storage = mentat.storage.Storage()
- database = storage.database('mentat')
- collection = database.collection('alerts')
+ db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
+ db_cfg = db_config[mentat.const.CKEY_CORE_DATABASE_CONFIG]
+ db_storage = mentat.storage.Storage()
+
+ database = db_storage.database(db_cfg['db'])
+ collection = database.collection(db_cfg['col_alerts'])
profile_dir = '/tmp/mentat-db-profile'
if not os.path.exists(profile_dir):
@@ -157,8 +164,8 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
cursor = collection.find().sort("_CESNET.StorageTime", -1)
tcount = cursor.count()
try:
- ptrn_ip = re.compile('(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?:\/\d{1,2}|(?:-|..)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})?|[:a-fA-F0-9]+:[:a-fA-F0-9]*(?:\/\d{1,3}|(?:-|..)[:a-fA-F0-9]+:[:a-fA-F0-9]*)?)')
- ptrn_sc = re.compile('[ /.]+')
+ ptrn_ip = re.compile(r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?:\/\d{1,2}|(?:-|..)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})?|[:a-fA-F0-9]+:[:a-fA-F0-9]*(?:\/\d{1,3}|(?:-|..)[:a-fA-F0-9]+:[:a-fA-F0-9]*)?)')
+ ptrn_sc = re.compile(r'[ /.]+')
print("Found total of '{}' events".format(tcount))
for event in cursor:
counter += 1
@@ -172,25 +179,25 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
else:
key += '*'
if 'Node' in event:
- n = event['Node'][-1]
- #if 'Name' in n:
- # key += '_+++_' + n.get('Name', '???')
+ node = event['Node'][-1]
+ #if 'Name' in node:
+ # key += '_+++_' + node.get('Name', '???')
#else:
# key += '_+++_*_'
- if 'Type' in n:
- key += '_+++_' + ':'.join(sorted(n.get('Type', ['???'])))
+ if 'Type' in node:
+ key += '_+++_' + ':'.join(sorted(node.get('Type', ['???'])))
else:
key += '_+++_*_'
else:
key += '_+++_*_'
if 'Source' in event:
- s = set()
+ vset = set()
for src in event['Source']:
if 'Type' in src:
- for x in src.get('Type', ['???']):
- s.add(x)
- if len(s):
- key += '_+++_' + ':'.join(s)
+ for vtype in src.get('Type', ['???']):
+ vset.add(vtype)
+ if len(vset):
+ key += '_+++_' + ':'.join(vset)
else:
key += '_+++_*_'
else:
@@ -203,12 +210,12 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
else:
key += '_+++_*'
event_map[key] = event_map.get(key, 0) + 1
- if (event_map[key] == 1):
- fn = os.path.join(profile_dir, "{}.idea".format(key))
- fh = open(fn, 'w')
+ if event_map[key] == 1:
+ fln = os.path.join(profile_dir, "{}.idea".format(key))
+ flh = open(fln, 'w')
msgraw = event.get('msg_raw2', event.get('msg_raw'))
- json.dump(json.loads(msgraw), fh, sort_keys=True, indent=4)
- fh.close()
+ json.dump(json.loads(msgraw), flh, sort_keys=True, indent=4)
+ flh.close()
except Exception as exc:
print("Exception: {}".format(exc))
@@ -222,26 +229,28 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
#---------------------------------------------------------------------------
def _email_alert(self, age):
- msg = MIMEText("Attention: Last alert in database is older than 6 hours !!!")
+ msg = MIMEText("Attention: Last alert in database is older than 6 hours (age: {} seconds) !!!".format(age))
msg['Subject'] = 'Mentat watchdog alert'
msg['From'] = self.c('mail_from')
msg['To'] = self.c('mail_to')
- s = smtplib.SMTP('localhost')
- s.send_message(msg)
- s.quit()
+ smtp = smtplib.SMTP('localhost')
+ smtp.send_message(msg)
+ smtp.quit()
+
+#
+# Execute the script.
+#
if __name__ == "__main__":
- """
- Execute the MentatDbmngrScript script.
- """
- script = MentatDbmngrScript(
+
+ SCRIPT = MentatDbmngrScript(
description = 'mentat-dbmngr.py - Mentat system database management script',
#
- # Configure required daemon paths
+ # Configure required script paths.
#
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
@@ -250,8 +259,8 @@ if __name__ == "__main__":
path_tmp = '/tmp',
#
- # Override default configurations
+ # Override default configurations.
#
default_config_dir = '/etc/mentat/core',
)
- script.run()
+ SCRIPT.run()
diff --git a/bin/mentat-ideagen.py b/bin/mentat-ideagen.py
index bd0f56ca2..28473085f 100755
--- a/bin/mentat-ideagen.py
+++ b/bin/mentat-ideagen.py
@@ -7,31 +7,22 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
+"""
+Script for generating IDEA message, mainly fo testing and development purposes.
+"""
+
import os
-import re
-import sys
import time
-import math
-import glob
-import json
-import logging
import string
import random
from jinja2 import Environment, PackageLoader
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))
-sys.path.insert(0, lib)
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../bin'))
-sys.path.insert(0, lib)
-
#
# Custom libraries
#
-import pydgets.widgets
import pyzenkit.zenscript
-list_ips_source = [
+LIST_IPS_SOURCE = [
'192.168.0.1',
'10.0.0.1',
'192.168.0.2',
@@ -41,10 +32,15 @@ list_ips_source = [
'192.168.0.4',
'10.0.0.4',
'192.168.0.5',
- '10.0.0.5'
+ '10.0.0.5',
+ '195.113.144.234',
+ '195.113.144.194',
+ '195.113.144.201',
+ '195.113.144.230',
+ '78.128.211.97'
]
-list_ips_target = [
+LIST_IPS_TARGET = [
'192.168.5.1',
'10.10.0.1',
'192.168.5.2',
@@ -54,10 +50,15 @@ list_ips_target = [
'192.168.5.4',
'10.10.0.4',
'192.168.5.5',
- '10.10.0.5'
+ '10.10.0.5',
+ '195.113.144.234',
+ '195.113.144.194',
+ '195.113.144.201',
+ '195.113.144.230',
+ '78.128.211.97'
]
-list_node_sws = [
+LIST_NODE_SWS = [
'Kippo',
'Dionaea',
'LaBrea',
@@ -66,7 +67,7 @@ list_node_sws = [
'FlowMon'
]
-list_node_names = [
+LIST_NODE_NAMES = [
'cz.cesnet.lister',
'cz.cesnet.rimmer',
'cz.cesnet.kryten'
@@ -75,7 +76,7 @@ list_node_names = [
'cz.cesnet.queeg'
]
-list_categories = [
+LIST_CATEGORIES = [
'Recon.Scanning',
'Attempt.Exploit',
'Abusive.Spam',
@@ -86,8 +87,9 @@ list_categories = [
class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
"""
- Script providing Mentat system backup functions and features
+ Script for generating IDEA message, mainly fo testing and development purposes.
"""
+
CONFIG_COUNT = 'count'
CONFIG_BACKOFF = 'backoff'
CONFIG_STEADY = 'steady'
@@ -130,8 +132,8 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
(self.CONFIG_STEADY, False),
(self.CONFIG_QUEUE_DIR, '/var/tmp'),
)
- for c in cfgs:
- config[c[0]] = kwargs.get('default_' + c[0], c[1])
+ for cfg in cfgs:
+ config[cfg[0]] = kwargs.get('default_' + cfg[0], cfg[1])
return config
def get_default_command(self):
@@ -140,7 +142,8 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
"""
return 'generate'
- def _generate_id(self, size=6, chars=string.ascii_lowercase + string.digits):
+ @staticmethod
+ def _generate_id(size=6, chars=string.ascii_lowercase + string.digits):
return '{}{}'.format('testmsg-', ''.join(random.choice(chars) for _ in range(size)))
def _generate_variables(self):
@@ -150,11 +153,11 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
variables = {}
variables['message_id'] = self._generate_id(20)
variables['detect_time'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
- variables['category'] = random.choice(list_categories)
- variables['source_ip'] = random.choice(list_ips_source)
- variables['target_ip'] = random.choice(list_ips_target)
- variables['node_sw'] = random.choice(list_node_sws)
- variables['node_name'] = random.choice(list_node_names)
+ variables['category'] = random.choice(LIST_CATEGORIES)
+ variables['source_ip'] = random.choice(LIST_IPS_SOURCE)
+ variables['target_ip'] = random.choice(LIST_IPS_TARGET)
+ variables['node_sw'] = random.choice(LIST_NODE_SWS)
+ variables['node_name'] = random.choice(LIST_NODE_NAMES)
return variables
def _save_message(self, variables, msg):
@@ -178,15 +181,13 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
folder caches.
"""
template = self.templ_env.get_template('msg.01.idea.j2')
- msg_counter = 0
try:
while True:
- for i in range(self.c(self.CONFIG_COUNT)):
- msg_counter = msg_counter + 1
+ for idx in range(1, self.c(self.CONFIG_COUNT) + 1):
variables = self._generate_variables()
idea_message = template.render(**variables)
ifn = self._save_message(variables, idea_message)
- self.p("[{:0,d}] Generated new IDEA message into file '{}'".format(msg_counter,ifn))
+ self.p("[{:0,d}] Generated new IDEA message into file '{}'".format(idx, ifn))
if not self.c('steady'):
break
@@ -197,17 +198,28 @@ class MentatIdeagenScript(pyzenkit.zenscript.ZenScript):
except KeyboardInterrupt:
pass
+
+#
+# Execute the script.
+#
if __name__ == '__main__':
- """
- Execute the MentatCleanupScript script.
- """
- script = MentatIdeagenScript(
+
+ SCRIPT = MentatIdeagenScript(
+
+ description = 'mentat-dbmngr.py - Mentat system database management script',
+
+ #
+ # Configure required script paths.
+ #
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
path_log = '/var/mentat/log',
path_run = '/var/mentat/run',
path_tmp = '/tmp',
+ #
+ # Override default configurations.
+ #
default_config_dir = '/etc/mentat/core',
)
- script.run()
+ SCRIPT.run()
diff --git a/bin/mentat-inspector.py b/bin/mentat-inspector.py
index a90dc644b..d114ae46f 100755
--- a/bin/mentat-inspector.py
+++ b/bin/mentat-inspector.py
@@ -19,23 +19,10 @@ __author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-#
-# Python system libraries
-#
-import os
-import sys
-
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))
-sys.path.insert(0, lib)
-
-
#
# Custom libraries
#
-import pyzenkit.baseapp
-import pyzenkit.zendaemon
+import mentat.const
import mentat.daemon.piper
import mentat.daemon.component.parser
import mentat.daemon.component.inspector
@@ -44,20 +31,25 @@ import mentat.daemon.component.commiter
class MentatInspectorDaemon(mentat.daemon.piper.PiperDaemon):
+ """
+ Implementation of real-time message processing daemon capable of inspecting
+ `IDEA <https://idea.cesnet.cz/en/index>`__ messaes according to given
+ set of filtering rules and performing number of associated actions.
+ """
pass
+#
+# Execute the daemon.
+#
if __name__ == "__main__":
- """
- Execute the daemon.
- """
- daemon = MentatInspectorDaemon(
+ DAEMON = MentatInspectorDaemon(
- description = 'mentat-inspector - IDEA message inspection daemon',
+ description = 'mentat-inspector.py - IDEA message inspection daemon',
#
- # Configure required deamon paths
+ # Configure required deamon paths.
#
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
@@ -66,23 +58,25 @@ if __name__ == "__main__":
path_tmp = '/var/tmp',
#
- # Override default configurations
+ # Override default configurations.
#
default_config_dir = '/etc/mentat/core',
default_queue_in_dir = '/var/mentat/spool/mentat-inspector.py',
default_queue_out_dir = '/var/mentat/spool/__done_a__',
- default_stats_interval = 20,
+ default_stats_interval = mentat.const.DFLT_INTERVAL_STATISTICS,
#
- # Schedule initial events
+ # Schedule initial events.
#
- schedule = [('start',)],
+ schedule = [
+ (mentat.const.DFLT_EVENT_START,)
+ ],
schedule_after = [
- (20, 'log_statistics')
+ (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS)
],
#
- # Define required daemon components
+ # Define required daemon components.
#
components = [
mentat.daemon.component.parser.ParserDaemonComponent(),
@@ -91,4 +85,4 @@ if __name__ == "__main__":
mentat.daemon.component.commiter.CommiterDaemonComponent()
]
)
- daemon.run()
+ DAEMON.run()
diff --git a/bin/mentat-netmngr.py b/bin/mentat-netmngr.py
index f3247cde7..8e53023fd 100755
--- a/bin/mentat-netmngr.py
+++ b/bin/mentat-netmngr.py
@@ -7,29 +7,25 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
+
"""
Script providing functions for abuse group network management for Mentat system
database. Still work in progress, use with caution.
"""
+
__version__ = "0.1"
__author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-import os
-import re
-import json
-import time
-import math
-import glob
-import logging
+
import collections
import pprint
+
#
# Custom libraries
#
-import iprange
import pyzenkit.jsonconf
import pyzenkit.zenscript
import mentat.const
@@ -48,6 +44,7 @@ CONFIG_WHOIS_FILE = 'whois_file'
WHOIS_KEY_TYPE = '__whois_type__'
WHOIS_KEY_ABOUT_GENERIC = '__whois_about__'
WHOIS_KEY_ABOUT_NEGISTRY = '__negistry_about__'
+
WHOIS_TYPE_GENERIC = 'whois'
WHOIS_TYPE_NEGISTRY = 'negistry'
@@ -88,8 +85,8 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
(CONFIG_DB_ARGS, None),
(CONFIG_WHOIS_FILE, None)
)
- for c in cfgs:
- config[c[0]] = kwargs.get('default_' + c[0], c[1])
+ for cfg in cfgs:
+ config[cfg[0]] = kwargs.get('default_' + cfg[0], cfg[1])
return config
def _configure_postprocess(self):
@@ -105,13 +102,12 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
# Configure undefined database settings from core settings.
for cfg in (
- (CONFIG_DATABASE, db_config['db']),
- (CONFIG_COLLECTION, db_config['col_groups']),
- (CONFIG_DB_HOST, db_connection['host']),
- (CONFIG_DB_PORT, db_connection['port']),
- (CONFIG_DB_TIMEOUT, db_connection['timeout']),
- (CONFIG_DB_ARGS, db_connection['args'])
- ):
+ (CONFIG_DATABASE, db_config['db']),
+ (CONFIG_COLLECTION, db_config['col_groups']),
+ (CONFIG_DB_HOST, db_connection['host']),
+ (CONFIG_DB_PORT, db_connection['port']),
+ (CONFIG_DB_TIMEOUT, db_connection['timeout']),
+ (CONFIG_DB_ARGS, db_connection['args'])):
if self.config[cfg[0]] is None:
self.config[cfg[0]] = cfg[1]
@@ -176,18 +172,19 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
self.logger.info("Loaded reference whois file '{} :: {}' ({})".format(whois_file, whois_file_type, whois_file_about))
return (whois_file_type, whois_file_data)
- def _process_whois_data(self, whois_file_data, whois_file_type):
+ @staticmethod
+ def _process_whois_data(whois_file_data, whois_file_type):
"""
Process reference whois file.
"""
# Load whois data into dictionary of dictionaries for easy searching and
# comparisons.
processed_data = collections.defaultdict(dict)
- for network_key, network_data in whois_file_data.items():
- nr = mentat.whois.NetworkRecord(network_data, source = whois_file_type)
- nrkey = str(nr)
+ for network_data in whois_file_data.values():
+ nwr = mentat.whois.NetworkRecord(network_data, source = whois_file_type)
+ nwrkey = str(nwr)
for abuse_group in network_data['resolved_abuses']:
- processed_data[abuse_group][nrkey] = nr
+ processed_data[abuse_group][nwrkey] = nwr
return processed_data
#---------------------------------------------------------------------------
@@ -199,25 +196,25 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
for group_name in sorted(wi_file_data.keys()):
if not group_name in abuse_group_dict:
self.logger.warning("'{}' Inserting missing abuse group defined in loaded '{}' whois file".format(group_name, wi_file_type))
- ag = mentat.whois.AbuseGroup({
- '_id': group_name,
- 'networks': wi_file_data[group_name].values()
- })
- pprint.pprint(ag)
- pprint.pprint(ag['networks'][0]['_id'])
- self.collection.insert_one(ag.export())
+ abg = mentat.whois.AbuseGroup({
+ '_id': group_name,
+ 'networks': wi_file_data[group_name].values()
+ })
+ pprint.pprint(abg)
+ pprint.pprint(abg['networks'][0]['_id'])
+ self.collection.insert_one(abg.export())
def _groups_remove_extra(self, abuse_group_dict, wi_file_data, wi_file_type):
"""
Remove extra abuse groups from database.
"""
for group_name in sorted(abuse_group_dict.keys()):
- ag = abuse_group_dict[group_name]
- if ag['source'] == wi_file_type and not group_name in wi_file_data:
+ abg = abuse_group_dict[group_name]
+ if abg['source'] == wi_file_type and not group_name in wi_file_data:
self.logger.warning("'{}' Consider deletion of extra abuse group missing in loaded '{}' whois file".format(group_name, wi_file_type))
# Do not delete yet.
- #self.collection.delete_one({'_id': ag['_id']})
+ #self.collection.delete_one({'_id': abg['_id']})
def _groups_update_existing(self, abuse_group_dict, wi_file_data, wi_file_type):
"""
@@ -226,14 +223,14 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
for group_name in sorted(abuse_group_dict.keys()):
if group_name in wi_file_data:
self.logger.info("'{}' Updating abuse group according to loaded '{}' whois file".format(group_name, wi_file_type))
- ag = abuse_group_dict[group_name]
- changelog = ag.update_networks(wi_file_data[group_name].values(), wi_file_type)
+ abg = abuse_group_dict[group_name]
+ changelog = abg.update_networks(wi_file_data[group_name].values(), wi_file_type)
if changelog:
- for ch in changelog:
- self.logger.warning("'{}' Abuse group network change according to loaded '{}' whois file: {}".format(group_name, wi_file_type, ch))
- pprint.pprint(ag)
- pprint.pprint(ag['_id'])
- self.collection.update_one({'_id': ag['_id']}, {'$set': {'networks': ag.export()['networks']}})
+ for chl in changelog:
+ self.logger.warning("'{}' Abuse group network change according to loaded '{}' whois file: {}".format(group_name, wi_file_type, chl))
+ pprint.pprint(abg)
+ pprint.pprint(abg['_id'])
+ self.collection.update_one({'_id': abg['_id']}, {'$set': {'networks': abg.export()['networks']}})
#---------------------------------------------------------------------------
# OPERATION CALLBACK IMPLEMENTATIONS
@@ -273,7 +270,8 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
# Check 01: The abuse group should have any related networks defined,
# otherwise it might be useless (with exception of '__UNKNOWN__' group).
#
- if not 'networks' in abuse_group or not len(abuse_group['networks']):
+ network_count = len(abuse_group['networks'])
+ if not 'networks' in abuse_group or not network_count > 0:
self.logger.warning("'{}::{}' Abuse group does not have any networks defined, consider deletion".format(group_source, group_name))
continue
@@ -292,9 +290,9 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
#
for network in abuse_group['networks']:
try:
- nr = mentat.whois.NetworkRecord(network)
- nrkey = str(nr)
- self.logger.debug("'{}::{}' Network record '{}::{}'".format(group_source, group_name, nr['source'], nrkey))
+ nwr = mentat.whois.NetworkRecord(network)
+ nwrkey = str(nwr)
+ self.logger.debug("'{}::{}' Network record '{}::{}'".format(group_source, group_name, nwr['source'], nwrkey))
if wi_file_data and group_name in wi_file_data:
@@ -302,23 +300,23 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
# Point out network records marked as coming from given
# source and not present in that particular source.
#
- if nr['source'] == wi_file_type and not nrkey in wi_file_data[group_name]:
- self.logger.warning("'{}::{}' Network record '{}::{}' is not defined in loaded '{}' whois file, consider deletion".format(group_source, group_name, nr['source'], nrkey, wi_file_type))
- elif nr['source'] == wi_file_type:
- del wi_file_data[group_name][nrkey]
+ if nwr['source'] == wi_file_type and not nwrkey in wi_file_data[group_name]:
+ self.logger.warning("'{}::{}' Network record '{}::{}' is not defined in loaded '{}' whois file, consider deletion".format(group_source, group_name, nwr['source'], nwrkey, wi_file_type))
+ elif nwr['source'] == wi_file_type:
+ del wi_file_data[group_name][nwrkey]
else:
- self.logger.info("'{}::{}' Extra network record '{}::{}'".format(group_source, group_name, nr['source'], nrkey))
+ self.logger.info("'{}::{}' Extra network record '{}::{}'".format(group_source, group_name, nwr['source'], nwrkey))
- except ValueError as e:
- self.logger.warning("'{}::{}' Invalid network record {} - '{}'".format(group_source, group_name, pprint.pformat(network), str(e)))
+ except ValueError as exc:
+ self.logger.warning("'{}::{}' Invalid network record {} - '{}'".format(group_source, group_name, pprint.pformat(network), str(exc)))
if wi_file_data:
for group_name in wi_file_data.keys():
if not group_name in abuse_group_dict:
self.logger.warning("'{}' Abuse group defined in loaded '{}' whois file but missing in database".format(group_name, wi_file_type))
- for nrkey in wi_file_data[group_name].keys():
- self.logger.warning("'{}' Missing network record '{}::{}'".format(group_name, wi_file_data[group_name][nrkey]['source'], nrkey))
+ for nwrkey in wi_file_data[group_name].keys():
+ self.logger.warning("'{}' Missing network record '{}::{}'".format(group_name, wi_file_data[group_name][nwrkey]['source'], nwrkey))
return self.RESULT_SUCCESS
@@ -344,16 +342,18 @@ class MentatNetmngrScript(pyzenkit.zenscript.ZenScript):
self._groups_remove_extra(abuse_group_dict, wi_file_data, wi_file_type)
self._groups_update_existing(abuse_group_dict, wi_file_data, wi_file_type)
+
+#
+# Execute the script.
+#
if __name__ == "__main__":
- """
- Execute the MentatNetmngrScript script.
- """
- script = MentatNetmngrScript(
+
+ SCRIPT = MentatNetmngrScript(
description = 'mentat-netmngr.py - Abuse group network management script for Mentat system database',
#
- # Configure required daemon paths
+ # Configure required script paths.
#
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
@@ -362,8 +362,8 @@ if __name__ == "__main__":
path_tmp = '/tmp',
#
- # Override default configurations
+ # Override default configurations.
#
default_config_dir = '/etc/mentat/core',
)
- script.run()
+ SCRIPT.run()
diff --git a/bin/mentat-sampler.py b/bin/mentat-sampler.py
index 829d692a5..4132c02d7 100755
--- a/bin/mentat-sampler.py
+++ b/bin/mentat-sampler.py
@@ -7,30 +7,31 @@
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
-import pprint
-import os
-import sys
+"""
+Real-time message processing daemon capable of sampling `IDEA <https://idea.cesnet.cz/en/index>`__
+messages according to various policies.
+"""
+
+
+__version__ = "0.1"
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))
-sys.path.insert(0, lib)
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../bin'))
-sys.path.insert(0, lib)
#
# Custom libraries
#
-import pyzenkit.baseapp
-import pyzenkit.zendaemon
import mentat.daemon.piper
import mentat.daemon.component.parser
import mentat.daemon.component.sampler
import mentat.daemon.component.commiter
+
class MentatSamplerDaemon(mentat.daemon.piper.PiperDaemon):
"""
- Daemon for message inspections.
+ Implementation of real-time message processing daemon capable of sampling
+ `IDEA <https://idea.cesnet.cz/en/index>`__ messages according to various policies.
"""
CONFIG_SAMPLING_KEYS = 'sampling_keys'
@@ -58,36 +59,54 @@ class MentatSamplerDaemon(mentat.daemon.piper.PiperDaemon):
(self.CONFIG_SAMPLING_LIMIT, 10),
(self.CONFIG_SAMPLING_POLICY, 'simple'),
)
- for c in cfgs:
- config[c[0]] = kwargs.pop('default_' + c[0], c[1])
+ for cfg in cfgs:
+ config[cfg[0]] = kwargs.pop('default_' + cfg[0], cfg[1])
return config
+
+#
+# Execute the daemon.
+#
if __name__ == "__main__":
- """
- Execute the daemon.
- """
- # Perform test run.
- daemon = MentatSamplerDaemon(
+
+ DAEMON = MentatSamplerDaemon(
+
+ description = 'mentat-sampler.py - IDEA message sampling daemon',
+
+ #
+ # Configure required deamon paths.
+ #
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
path_log = '/var/mentat/log',
path_run = '/var/mentat/run',
path_tmp = '/tmp',
+ #
+ # Override default configurations.
+ #
default_config_dir = '/etc/mentat/core',
default_queue_in_dir = '/var/mentat/spool/mentat-sampler.py',
- default_queue_out_dir = '/var/mentat/spool/mentat-sampler.py/tmp',
- default_stats_interval = 20,
+ default_queue_out_dir = None,
+ default_stats_interval = mentat.const.DFLT_INTERVAL_STATISTICS,
- description = 'mentat-sampler - Message sampling daemon',
- schedule = [('start',)],
+ #
+ # Schedule initial events.
+ #
+ schedule = [
+ (mentat.const.DFLT_EVENT_START,)
+ ],
schedule_after = [
- (10, 'log_statistics')
+ (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS)
],
+
+ #
+ # Define required daemon components.
+ #
components = [
mentat.daemon.component.parser.ParserDaemonComponent(),
mentat.daemon.component.sampler.SamplerDaemonComponent(),
mentat.daemon.component.commiter.CommiterDaemonComponent()
]
)
- daemon.run()
+ DAEMON.run()
diff --git a/bin/mentat-storage.py b/bin/mentat-storage.py
index 9b43963f2..b021f41ae 100755
--- a/bin/mentat-storage.py
+++ b/bin/mentat-storage.py
@@ -65,24 +65,9 @@ __author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-#
-# Python system libraries.
-#
-import os
-import sys
-
-
-# Generate the path to custom 'lib' directory. This is usefull when launching
-# modules from development directory tree.
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib'))
-sys.path.insert(0, lib)
-
-
#
# Custom libraries.
#
-import pyzenkit.baseapp
-import pyzenkit.zendaemon
import mentat.const
import mentat.daemon.piper
import mentat.daemon.component.parser
@@ -91,6 +76,11 @@ import mentat.daemon.component.commiter
class MentatStorageDaemon(mentat.daemon.piper.PiperDaemon):
+ """
+ Implementation of real-time message processing daemon capable of storing
+ `IDEA <https://idea.cesnet.cz/en/index>`__ messages into persistent storage.
+ Currently only `MongoDB <https://www.mongodb.com/>`__ NoSQL database is supported.
+ """
def _init_argparser(self, **kwargs):
"""
@@ -120,8 +110,8 @@ class MentatStorageDaemon(mentat.daemon.piper.PiperDaemon):
(mentat.daemon.component.storage.CONFIG_DB_TIMEOUT, None),
(mentat.daemon.component.storage.CONFIG_DB_ARGS, None)
)
- for c in cfgs:
- config[c[0]] = kwargs.get('default_' + c[0], c[1])
+ for cfg in cfgs:
+ config[cfg[0]] = kwargs.get('default_' + cfg[0], cfg[1])
return config
def _configure_postprocess(self):
@@ -137,27 +127,27 @@ class MentatStorageDaemon(mentat.daemon.piper.PiperDaemon):
# Configure undefined database settings from core settings.
for cfg in (
- (mentat.daemon.component.storage.CONFIG_DATABASE, db_config['db']),
- (mentat.daemon.component.storage.CONFIG_COLLECTION, db_config['col_alerts']),
- (mentat.daemon.component.storage.CONFIG_DB_HOST, db_connection['host']),
- (mentat.daemon.component.storage.CONFIG_DB_PORT, db_connection['port']),
- (mentat.daemon.component.storage.CONFIG_DB_TIMEOUT, db_connection['timeout']),
- (mentat.daemon.component.storage.CONFIG_DB_ARGS, db_connection['args'])
- ):
+ (mentat.daemon.component.storage.CONFIG_DATABASE, db_config['db']),
+ (mentat.daemon.component.storage.CONFIG_COLLECTION, db_config['col_alerts']),
+ (mentat.daemon.component.storage.CONFIG_DB_HOST, db_connection['host']),
+ (mentat.daemon.component.storage.CONFIG_DB_PORT, db_connection['port']),
+ (mentat.daemon.component.storage.CONFIG_DB_TIMEOUT, db_connection['timeout']),
+ (mentat.daemon.component.storage.CONFIG_DB_ARGS, db_connection['args'])):
if self.config[cfg[0]] is None:
self.config[cfg[0]] = cfg[1]
+
+#
+# Execute the daemon.
+#
if __name__ == "__main__":
- """
- Execute the daemon.
- """
- daemon = MentatStorageDaemon(
+ DAEMON = MentatStorageDaemon(
- description = 'mentat-storage - IDEA message storing daemon',
+ description = 'mentat-storage.py - IDEA message storing daemon',
#
- # Configure required deamon paths
+ # Configure required deamon paths.
#
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
@@ -166,7 +156,7 @@ if __name__ == "__main__":
path_tmp = '/var/tmp',
#
- # Override default configurations
+ # Override default configurations.
#
default_config_dir = '/etc/mentat/core',
default_queue_in_dir = '/var/mentat/spool/mentat-storage.py',
@@ -174,17 +164,17 @@ if __name__ == "__main__":
default_stats_interval = mentat.const.DFLT_INTERVAL_STATISTICS,
#
- # Schedule initial events
+ # Schedule initial events.
#
schedule = [
(mentat.const.DFLT_EVENT_START,)
],
schedule_after = [
- (mentat.const.DFLT_INTERVAL_STATISTICS, 'log_statistics')
+ (mentat.const.DFLT_INTERVAL_STATISTICS, mentat.const.DFLT_EVENT_LOG_STATISTICS)
],
#
- # Define required daemon components
+ # Define required daemon components.
#
components = [
mentat.daemon.component.parser.ParserDaemonComponent(),
@@ -192,4 +182,4 @@ if __name__ == "__main__":
mentat.daemon.component.commiter.CommiterDaemonComponent()
]
)
- daemon.run()
+ DAEMON.run()
--
GitLab