Code style improvements with pylint tool.

Removed all reported errors for library code reported by pylint, with additional fixes of minor problems. (Redmine issue: #3443)

.pylintrc-lib

@@ -297,7 +297,7 @@ ignored-classes=optparse.Values,thread._local,_thread._local
 # (useful for modules/projects where namespaces are manipulated during runtime
 # and thus existing member attributes cannot be deduced by static analysis. It
 # supports qualified module names, as well as Unix pattern matching.
-ignored-modules=
+ignored-modules=rrdtool
 
 # Show a hint with possible names when a member name was not found. The aspect
 # of finding the hint is based on edit distance.

lib/mentat/datatype/internal.py

@@ -40,11 +40,15 @@ import re
 import time
 import random
 import string
-import ipranges
-import typedcols
 import datetime
 import pprint
 
+#
+# Custom libraries.
+#
+import ipranges
+import typedcols
+
 
 NR_TYPE_IPV4 = 'ipv4'
 NR_TYPE_IPV6 = 'ipv6'
@@ -69,8 +73,13 @@ RE_TIMESTAMP = re.compile(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})[Tt ]([0-9]{2}):([0
 # HELPER FUNCTIONS
 #-------------------------------------------------------------------------------
 
+def list_factory(name, item_type):
+    """
+    Default implementation of list factory.
+    """
+    return typedcols.typed_list(name, item_type)
 
-def list_types(flavour, list_factory=None):
+def list_types(flavour, cb_list_factory=None):
     """
     Generate list type flavours from given scalar flavour definitions.
 
@@ -80,12 +89,11 @@ def list_types(flavour, list_factory=None):
     :return: list flavour
     :rtype: dict
     """
-    if list_factory is None:
-        def list_factory(name, item_type):
-            return typedcols.typed_list(name, item_type)
+    if cb_list_factory is None:
+        cb_list_factory = list_factory
     lists = {}
-    for t in ('Integer', 'String', 'NetworkRecord', 'ReportingFilter', 'SavedQuery'):
-        lists[t] = list_factory(t, flavour[t])
+    for tpe in ('Integer', 'String', 'NetworkRecord', 'ReportingFilter', 'SavedQuery'):
+        lists[tpe] = cb_list_factory(tpe, flavour[tpe])
     return lists
 
 def to_net4(val):
@@ -204,12 +212,12 @@ def t_datetime(val):
     res = RE_TIMESTAMP.match(val)
     if res is not None:
         year, month, day, hour, minute, second = (int(n or 0) for n in res.group(*range(1, 7)))
-        us_str = (res.group(7) or '0')[:6].ljust(6, '0')
-        us = int(us_str)
+        usec_str = (res.group(7) or '0')[:6].ljust(6, '0')
+        usec = int(usec_str)
         zonestr = res.group(8)
         zonespl = (0, 0) if zonestr in ['z', 'Z'] else [int(i) for i in zonestr.split(':')]
         zonediff = datetime.timedelta(minutes = zonespl[0]*60+zonespl[1])
-        return datetime.datetime(year, month, day, hour, minute, second, us) - zonediff
+        return datetime.datetime(year, month, day, hour, minute, second, usec) - zonediff
     else:
         raise ValueError("Invalid datetime '{:s}'".format(val))
 
@@ -270,8 +278,8 @@ def t_network_record(val, source = None):
 
     # 'ip4_start' and 'ip4_end' come from Negistry whois file
     elif 'ip4_start' in val and 'ip4_end' in val:
-            record['nrobj'] = to_net4("{}-{}".format(val['ip4_start'], val['ip4_end']))
-            record['type']  = NR_TYPE_IPV4
+        record['nrobj'] = to_net4("{}-{}".format(val['ip4_start'], val['ip4_end']))
+        record['type']  = NR_TYPE_IPV4
 
     # 'ip6_addr' and 'ip6_prefix' come from Negistry whois file
     elif 'ip6_addr' in val and 'ip6_prefix' in val:
@@ -312,8 +320,7 @@ def t_network_record(val, source = None):
 
     if record['type'] == NR_TYPE_IPV4:
         return NetworkRecordIP4(record)
-    else:
-        return NetworkRecordIP6(record)
+    return NetworkRecordIP6(record)
 
 def t_reporting_mode(val):
     """
@@ -362,6 +369,9 @@ class NetworkRecord(typedcols.TypedDict):
     allow_unknown = False
 
     def fingerprint(self):
+        """
+        Return network fingerprint (concatenation of network and source).
+        """
         return '{:s}:{:s}'.format(self['network'], self['source'])
 
 types_internal = {

lib/mentat/idea/internal.py

@@ -7,6 +7,7 @@
 # Use of this source is governed by the MIT license, see LICENSE file.
 #-------------------------------------------------------------------------------
 
+
 """
 This module provides classess and tools for object representation of
 `IDEA <https://idea.cesnet.cz/en/index>`__ messages in Mentat project.
@@ -93,11 +94,11 @@ class CESNETDict(typedcols.TypedDict):
     *_CESNET* in message root dictionary.
     """
     typedef = cesnet_dict_typedef(
-                    idea.lite.idea_types,
-                    idea.lite.idea_lists,
-                    typedcols.typed_list("InspectionErrorsList", str),
-                    typedcols.typed_list("ResolvedAbusesList", str)
-                )
+        idea.lite.idea_types,
+        idea.lite.idea_lists,
+        typedcols.typed_list("InspectionErrorsList", str),
+        typedcols.typed_list("ResolvedAbusesList", str)
+    )
 
 
 def internal_base_addon_typedef(flavour, list_flavour, cesnet_dict, addon=None):
@@ -122,11 +123,7 @@ def internal_base_addon_typedef(flavour, list_flavour, cesnet_dict, addon=None):
     if addon is not None:
         addon_def.update(addon)
     return addon_def
-"""
-This definition will make use of *addon* feature of ``typedcols`` library
-and alter existing typedef definition of root message element, which is
-coming from ``idea.lite`` module.
-"""
+
 
 class Idea(idea.lite.Idea):
     """
@@ -147,21 +144,22 @@ class Idea(idea.lite.Idea):
         typedcols.typed_list("AttachList", idea.lite.AttachDict),
         typedcols.typed_list("NodeList", idea.lite.NodeDict),
         internal_base_addon_typedef(
-                idea.lite.idea_types,
-                idea.lite.idea_lists,
-                CESNETDict
-            )
+            idea.lite.idea_types,
+            idea.lite.idea_lists,
+            CESNETDict
+        )
     )
     def to_string(self, **kwargs):
         """
         Dump message object to string.
         """
-        if not 'sort_keys' in kwargs:
+        if 'sort_keys' not in kwargs:
             kwargs['sort_keys'] = True
-        if not 'default' in kwargs:
+        if 'default' not in kwargs:
             kwargs['default']   = json_default
         return json.dumps(self, **kwargs)
 
+
 json_default = idea.lite.json_default
 """
 Helper method for JSON serialization of :py:class:`mentat.idea.internal.Idea`

lib/mentat/idea/mongodb.py

@@ -7,6 +7,7 @@
 # Use of this source is governed by the MIT license, see LICENSE file.
 #-------------------------------------------------------------------------------
 
+
 """
 This module provides classess for object representation and conversion
 of `IDEA <https://idea.cesnet.cz/en/index>`__  messages in
@@ -69,8 +70,8 @@ import typedcols
 import ipranges
 import idea.base
 import idea.lite
-import mentat.idea.internal
 from idea.base import unicode
+import mentat.idea.internal
 
 
 MONGODB_BINARY_TYPE = bson.binary.BINARY_SUBTYPE
@@ -100,24 +101,24 @@ NTPSTAMP_FUSIZE = int(NTPSTAMP_FPRECISION / NTPSTAMP_FRACTION)
 #-------------------------------------------------------------------------------
 
 
-def _to_bin(t, size):
+def _to_bin(val, size):
     """
     Convert given argument to binary representation.
 
     Reference: `link <http://pclib.github.io/safari/program/python-cookbook/Text/ch03s05.html>`__
     """
-    return bson.Binary(t.to_bytes(size, 'big'),
+    return bson.Binary(val.to_bytes(size, 'big'),
                        subtype = MONGODB_BINARY_TYPE)
 
-def _from_bin(t):
+def _from_bin(val):
     """
     Convert given argument from binary representation.
 
     Reference: `link <http://pclib.github.io/safari/program/python-cookbook/Text/ch03s05.html>`__
     """
-    if isinstance(t, bson.Binary):
-        t = bytes(t)
-    return int.from_bytes(t, 'big')
+    if isinstance(val, bson.Binary):
+        val = bytes(val)
+    return int.from_bytes(val, 'big')
 
 
 #-------------------------------------------------------------------------------
@@ -125,28 +126,28 @@ def _from_bin(t):
 #-------------------------------------------------------------------------------
 
 
-def Timestamp(t):
+def Timestamp(val):
     """
     Conversion of IDEA Timestamp datatype into appropriate MongoDB
     representation.
 
     Forward conversion.
     """
-    if isinstance(t, datetime.datetime):
+    if isinstance(val, datetime.datetime):
         # Take datetime as unix timestamp, convert to NTP epoch and shift
         # to upper 32bit part of 64bit integer.
-        r = (int(t.timestamp()) + NTP_TO_EPOCH_DELTA) << 32
+        res = (int(val.timestamp()) + NTP_TO_EPOCH_DELTA) << 32
 
         # Add microsecond part of NTP stamp to lower 32bit part of 64bit
         # integer
-        r = r + (t.microsecond * NTPSTAMP_FUSIZE)
+        res = res + (val.microsecond * NTPSTAMP_FUSIZE)
 
         # Convert to binary
-        return _to_bin(r, 8)
+        return _to_bin(res, 8)
     raise TypeError("Wrong Timestamp, expecting only 'datetime' object")
 
 
-def Timestamp_inv(t):
+def Timestamp_inv(val):
     """
     Convert MongoDB Timestamp datatype into appropriate IDEA object
     representation.
@@ -154,7 +155,7 @@ def Timestamp_inv(t):
     Inverse conversion.
     """
     # Unpack binary data directly into upper and lower part.
-    high, low = struct.unpack(">II", t)
+    high, low = struct.unpack(">II", val)
 
     # Convert NTP epoch to Unix epoch, deal with microseconds.
     stamp = high - NTP_TO_EPOCH_DELTA + ((float)(low&0xffffffff))/NTPSTAMP_FPRECISION
@@ -167,20 +168,20 @@ def Timestamp_inv(t):
     return idea.lite.Timestamp(stamp)
 
 
-def Duration(t):
+def Duration(val):
     """
     Conversion of IDEA Duration datatype into appropriate MongoDB
     representation.
 
     Forward conversion.
     """
-    if isinstance(t, datetime.timedelta):
-        t = t.total_seconds()
-        return t
+    if isinstance(val, datetime.timedelta):
+        val = val.total_seconds()
+        return val
     raise TypeError("Wrong Duration, expecting only 'timedelta' object")
 
 
-def Duration_inv(t):
+def Duration_inv(val):
     """
     Convert MongoDB Duration datatype into appropriate IDEA object
     representation.
@@ -188,14 +189,14 @@ def Duration_inv(t):
     Inverse conversion.
     """
     try:
-        t = datetime.timedelta(seconds=float(t))
-        return t
+        val = datetime.timedelta(seconds=float(val))
+        return val
     except:
         pass
     raise TypeError("Wrong Duration, expecting only number of seconds as 'float'")
 
 
-def Net4(ip):
+def Net4(val):
     """
     Convert IDEA Net4 datatypes into appropriate MongoDB
     representation.
@@ -203,23 +204,23 @@ def Net4(ip):
     Forward conversion.
     """
     # Single IP4 address is the most common, so check for that first.
-    if isinstance(ip, ipranges.IP4):
-        b = _to_bin(ip.ip, 4)
+    if isinstance(val, ipranges.IP4):
+        bval = _to_bin(val.ip, 4)
         return {
-            'min': b,
-            'max': b,
-            'ip':  b,
+            'min': bval,
+            'max': bval,
+            'ip':  bval,
         }
     # There is no difference between storage of IP4Net and IP4Range.
-    elif isinstance(ip, (ipranges.IP4Net, ipranges.IP4Range)):
+    elif isinstance(val, (ipranges.IP4Net, ipranges.IP4Range)):
         return {
-            'min': _to_bin(ip.low(),  4),
-            'max': _to_bin(ip.high(), 4),
+            'min': _to_bin(val.low(),  4),
+            'max': _to_bin(val.high(), 4),
         }
     raise TypeError("Wrong Net4, expecting one of 'ipranges.IP4...' objects")
 
 
-def Net4_inv(ip):
+def Net4_inv(val):
     """
     Convert MongoDB Net4 datatypes into appropriate IDEA object
     representation.
@@ -227,22 +228,22 @@ def Net4_inv(ip):
     Inverse conversion.
     """
     # Single IP4 address is the most common, so check for that first.
-    if 'ip' in ip:
-        return ipranges.IP4(_from_bin(ip['ip']))
+    if 'ip' in val:
+        return ipranges.IP4(_from_bin(val['ip']))
 
     # Otherwise it will always be a network range
-    l =  _from_bin(ip['min'])
-    h =  _from_bin(ip['max'])
-    r = ipranges.IP4Range((l,h))
+    low  = _from_bin(val['min'])
+    high = _from_bin(val['max'])
+    rng  = ipranges.IP4Range((low, high))
 
     # Or we can do even better and it is CIDR network
     try:
-        return ipranges.IP4Net(r)
+        return ipranges.IP4Net(rng)
     except:
-        return r
+        return rng
 
 
-def Net6(ip):
+def Net6(val):
     """
     Convert IDEA Net6 datatypes into appropriate MongoDB
     representation.
@@ -250,23 +251,23 @@ def Net6(ip):
     Forward conversion.
     """
     # Single IP6 address is the most common, so check for that first.
-    if isinstance(ip, ipranges.IP6):
-        b = _to_bin(ip.ip, 16)
+    if isinstance(val, ipranges.IP6):
+        bval = _to_bin(val.ip, 16)
         return {
-            'min': b,
-            'max': b,
-            'ip':  b,
+            'min': bval,
+            'max': bval,
+            'ip':  bval,
         }
     # There is no difference between storage of IP6Net and IP6Range.
-    elif isinstance(ip, (ipranges.IP6Net, ipranges.IP6Range)):
+    elif isinstance(val, (ipranges.IP6Net, ipranges.IP6Range)):
         return {
-            'min': _to_bin(ip.low(),  16),
-            'max': _to_bin(ip.high(), 16),
+            'min': _to_bin(val.low(),  16),
+            'max': _to_bin(val.high(), 16),
         }
     raise TypeError("Wrong Net6, expecting one of 'ipranges.IP6...' objects")
 
 
-def Net6_inv(ip):
+def Net6_inv(val):
     """
     Convert MongoDB Net6 datatypes into appropriate IDEA object
     representation.
@@ -274,19 +275,19 @@ def Net6_inv(ip):
     Inverse conversion.
     """
     # Single IP6 address is the most common, so check for that first.
-    if 'ip' in ip:
-        return ipranges.IP6(_from_bin(ip['ip']))
+    if 'ip' in val:
+        return ipranges.IP6(_from_bin(val['ip']))
 
     # Otherwise it will always be a network range
-    l =  _from_bin(ip['min'])
-    h =  _from_bin(ip['max'])
-    r = ipranges.IP6Range((l,h))
+    low  = _from_bin(val['min'])
+    high = _from_bin(val['max'])
+    rng  = ipranges.IP6Range((low, high))
 
     # Or we can do even better and it is CIDR network
     try:
-        return ipranges.IP6Net(r)
+        return ipranges.IP6Net(rng)
     except:
-        return r
+        return rng
 
 
 #-------------------------------------------------------------------------------
@@ -387,10 +388,10 @@ class CESNETDictIn(typedcols.TypedDict):
 # addon typedef.
 #
 idea_in_addon = mentat.idea.internal.internal_base_addon_typedef(
-                    idea_types_in,
-                    idea_lists_in,
-                    simplify(CESNETDictIn),
-                )
+    idea_types_in,
+    idea_lists_in,
+    simplify(CESNETDictIn),
+)
 
 
 class IdeaIn(idea.base.IdeaBase):
@@ -429,10 +430,9 @@ def json_default_in(o):
     """
     if isinstance(o, uuid.UUID):
         return str(o)
-    elif isinstance(o, bson.Binary):
+    if isinstance(o, bson.Binary):
         return repr(o)
-    else:
-        return idea.base.json_default(o)
+    return idea.base.json_default(o)
 
 
 #-------------------------------------------------------------------------------
@@ -500,11 +500,11 @@ class NodeDictOut(typedcols.TypedDict):
 class CESNETDictOut(typedcols.TypedDict):
     allow_unknown = True
     typedef = mentat.idea.internal.cesnet_dict_typedef(
-                    idea_types_out,
-                    idea_lists_out,
-                    typedcols.typed_list("InspectionErrorsList", str),
-                    typedcols.typed_list("ResolvedAbusesList", str)
-                )
+        idea_types_out,
+        idea_lists_out,
+        typedcols.typed_list("InspectionErrorsList", str),
+        typedcols.typed_list("ResolvedAbusesList", str)
+    )
 
 #
 # Addon for patching IdeaOut.
@@ -515,10 +515,10 @@ class CESNETDictOut(typedcols.TypedDict):
 # and will be deprecated and removed in the future.
 #
 idea_out_addon = mentat.idea.internal.internal_base_addon_typedef(
-                        idea_types_out,
-                        idea_lists_out,
-                        CESNETDictOut
-                    )
+    idea_types_out,
+    idea_lists_out,
+    CESNETDictOut
+)
 idea_out_addon['_id']      = typedcols.Discard
 idea_out_addon['class2']   = typedcols.Discard
 idea_out_addon['msg_raw2'] = typedcols.Discard

lib/mentat/stats/rrd.py

@@ -115,10 +115,10 @@ FILE_SPARKCHARTLIST = '_sparkchartlist.json'  # Name of the spark chart list fil
 #
 # Precompiled regular expression patterns for handling RRD database file names.
 #
-PTRN_RRDFILE = re.compile('(.+)\.rrd$')
-PTRN_TOTALDB = re.compile('{}$'.format(DB_TOTALS_NAME))
-PTRN_NAMERPL = re.compile('\.')
-PTRN_NAMECLN = re.compile('[^-_a-zA-Z0-9]')
+PTRN_RRDFILE = re.compile(r'(.+)\.rrd$')
+PTRN_TOTALDB = re.compile(r'{}$'.format(DB_TOTALS_NAME))
+PTRN_NAMERPL = re.compile(r'\.')
+PTRN_NAMECLN = re.compile(r'[^-_a-zA-Z0-9]')
 
 
 #
@@ -528,7 +528,7 @@ class RrdStats:
             return (rrddb, False)
 
         # Calculate steps per hour, will be used later during initialization.
-        sph = int(3600 / self.step);
+        sph = int(3600 / self.step)
 
         # Calculate the size for three month storage.
         rows = sph * 24 * 7 * 31 * 3