Merge branch 'devel' into 'master'

0.21

Closes #1

See merge request 709/mentat/pynspect!16

.gitlab-ci.yml

 # Official language image. Look for the different tagged releases at:
 # https://hub.docker.com/r/library/python/tags/
-image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.6
+image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:latest
 
 # Change pip's cache directory to be inside the project directory since we can
 # only cache local items.
@@ -15,7 +15,6 @@ variables:
 cache:
   paths:
     - .cache/pip
-    - venv/
 
 before_script:
   - pip install virtualenv
@@ -26,13 +25,50 @@ before_script:
 
 stages:          # List of stages for jobs, and their order of execution
   - test
+  - check-warnings
   - build
   - deploy
 
 unit-test-job:  
   stage: test
   script:
-    - make test
+    - make test 2>&1 | tee errors.log
+  artifacts:
+    when: always
+    paths:
+      - errors.log
+    reports:
+      junit: nose2-junit.xml
+
+unit-test-3.7-job:
+  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.7
+  stage: test
+  script:
+    - make test 2>&1 | tee errors-3.7.log
+  artifacts:
+    when: always
+    paths:
+      - errors-3.7.log
+
+unit-test-3.8-job:
+  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.8
+  stage: test
+  script:
+    - make test 2>&1 | tee errors-3.8.log
+  artifacts:
+    when: always
+    paths:
+      - errors-3.8.log
+
+unit-test-3.9-job:
+  image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.9
+  stage: test
+  script:
+    - make test 2>&1 | tee errors-3.9.log
+  artifacts:
+    when: always
+    paths:
+      - errors-3.9.log
 
 pylint-test-job:
   stage: test
@@ -44,6 +80,34 @@ pyflakes-test-job:
   script:
     - make pyflakes
 
+check-deprecation-warnings:
+  before_script: []
+  stage: check-warnings
+  script:
+    - "if [[ $(grep DeprecationWarning errors.log) ]]; then cat errors.log; exit 1; fi"
+  allow_failure: true
+
+check-deprecation-warnings-3.7:
+  before_script: []
+  stage: check-warnings
+  script:
+    - "if [[ $(grep DeprecationWarning errors-3.7.log) ]]; then cat errors-3.7.log; exit 1; fi"
+  allow_failure: true
+
+check-deprecation-warnings-3.8:
+  before_script: []
+  stage: check-warnings
+  script:
+    - "if [[ $(grep DeprecationWarning errors-3.8.log) ]]; then cat errors-3.8.log; exit 1; fi"
+  allow_failure: true
+
+check-deprecation-warnings-3.9:
+  before_script: []
+  stage: check-warnings
+  script:
+    - "if [[ $(grep DeprecationWarning errors-3.9.log) ]]; then cat errors-3.9.log; exit 1; fi"
+  allow_failure: true
+
 build-job:
   stage: build
   script:

Makefile

@@ -169,7 +169,7 @@ pylint-test: FORCE
 
 test: FORCE
 	@echo "\n${GREEN}*** Checking code with nosetests ***${NC}\n"
-	@$(NOSETESTS)
+	@$(PYTHON) -W always::DeprecationWarning -m nose2 --junit-xml
 
 
 #-------------------------------------------------------------------------------

README.rst

@@ -56,3 +56,19 @@ Copyright
 | Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com>
 | Use of this package is governed by the MIT license, see LICENSE file.
 |
+
+
+Changelog
+--------------------------------------------------------------------------------
+
+
+Version 0.21
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Released 2022-06-28
+
+-   Dropped support for Python 3.6.
+-   Fixed deprecation warnings for Python 3.7+ regarding ``collections.abc``.
+-   Added a config file for GitLab CI/CD.
+-   Updated the repository information.
+-   Updated packages versions.

nose2.cfg

+[unittest]
+plugins = nose2.plugins.junitxml

pynspect/__init__.py

@@ -16,4 +16,4 @@ data structures.
 """
 
 
-__version__ = "0.20"
+__version__ = "0.21"

pynspect/jpath.py

@@ -133,7 +133,7 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>"
 
 
 import re
-import collections
+from collections.abc import Mapping, MutableSequence
 
 
 #
@@ -273,7 +273,7 @@ def jpath_values(structure, jpath):
         # Process all currently active nodes.
         for node in nodes_a:
             key = chnk['n']
-            if not isinstance(node, dict) and not isinstance(node, collections.Mapping):
+            if not isinstance(node, dict) and not isinstance(node, Mapping):
                 continue
 
             # Process indexed nodes.
@@ -281,7 +281,7 @@ def jpath_values(structure, jpath):
                 idx = chnk['i']
                 # Skip the node, if the key does not exist, the value is not
                 # a list-like object or the list is empty.
-                if not key in node or not (isinstance(node[key], (list, collections.MutableSequence))) or not node[key]:
+                if not key in node or not (isinstance(node[key], (list, MutableSequence))) or not node[key]:
                     continue
                 try:
                     # Handle '*' special index - append all nodes.
@@ -300,7 +300,7 @@ def jpath_values(structure, jpath):
                     continue
 
                 # Handle list values - expand them.
-                if isinstance(node[key], (list, collections.MutableSequence)):
+                if isinstance(node[key], (list, MutableSequence)):
                     for i in node[key]:
                         nodes_b.append(i)
                 # Handle scalar values.
@@ -369,7 +369,7 @@ def jpath_set(structure, jpath, value, overwrite = True, unique = False):
     for i, chnk in enumerate(chunks):
         key = chnk['n']
 
-        if not isinstance(current, dict) and not isinstance(current, collections.Mapping):
+        if not isinstance(current, dict) and not isinstance(current, Mapping):
             raise JPathException("Expected dict-like structure to attach node '{}'".format(chnk['p']))
 
         # Process indexed nodes.
@@ -379,7 +379,7 @@ def jpath_set(structure, jpath, value, overwrite = True, unique = False):
             # Automatically create nodes for non-existent keys.
             if not key in current:
                 current[key] = []
-            if not isinstance(current[key], list) and not isinstance(current[key], collections.MutableSequence):
+            if not isinstance(current[key], list) and not isinstance(current[key], MutableSequence):
                 raise JPathException("Expected list-like object under structure key '{}'".format(key))
 
             # Detection of the last JPath chunk - node somewhere in the middle.
@@ -426,7 +426,7 @@ def jpath_set(structure, jpath, value, overwrite = True, unique = False):
                 # Automatically create nodes for non-existent keys.
                 if not key in current:
                     current[key] = {}
-                if not isinstance(current[key], dict) and not isinstance(current[key], collections.Mapping):
+                if not isinstance(current[key], dict) and not isinstance(current[key], Mapping):
                     raise JPathException("Expected dict-like object under structure key '{}'".format(key))
 
                 current = current[key]
@@ -464,7 +464,7 @@ def jpath_unset(structure, jpath):
         for node in nodes_a:
             key = chnk['n']
 
-            if not isinstance(node, dict) and not isinstance(node, collections.Mapping):
+            if not isinstance(node, dict) and not isinstance(node, Mapping):
                 raise JPathException("Expected dict-like structure to drop node '{}'".format(chnk['p']))
 
             # Process indexed nodes.
@@ -474,7 +474,7 @@ def jpath_unset(structure, jpath):
                 # Skip nodes for non-existent keys.
                 if not key in node:
                     continue
-                if not isinstance(node[key], list) and not isinstance(node[key], collections.MutableSequence):
+                if not isinstance(node[key], list) and not isinstance(node[key], MutableSequence):
                     raise JPathException("Expected list-like object under structure key '{}'".format(key))
 
                 # Detection of the last JPath chunk - node somewhere in the middle.
@@ -513,7 +513,7 @@ def jpath_unset(structure, jpath):
                     if isinstance(node[key], list):
                         nodes_b.extend(node[key])
                         continue
-                    if not isinstance(node[key], dict) and not isinstance(node[key], collections.Mapping):
+                    if not isinstance(node[key], dict) and not isinstance(node[key], Mapping):
                         raise JPathException("Expected dict-like object under structure key '{}'".format(key))
                     nodes_b.append(node[key])
 

pynspect/tests/test_compilers.py

@@ -130,23 +130,23 @@ class TestIDEAFilterCompiler(unittest.TestCase):
         )
         self.assertEqual(
             repr(compile_timedelta(NumberRule('3600'))),
-            "TIMEDELTA(datetime.timedelta(0, 3600))"
+            "TIMEDELTA(datetime.timedelta(seconds=3600))"
         )
         self.assertEqual(
             repr(compile_timedelta(ConstantRule('3600'))),
-            "TIMEDELTA(datetime.timedelta(0, 3600))"
+            "TIMEDELTA(datetime.timedelta(seconds=3600))"
         )
         self.assertEqual(
             repr(compile_timedelta(ConstantRule('15:15:15'))),
-            "TIMEDELTA(datetime.timedelta(0, 54915))"
+            "TIMEDELTA(datetime.timedelta(seconds=54915))"
         )
         self.assertEqual(
             repr(compile_timedelta(ConstantRule('15D15:15:15'))),
-            "TIMEDELTA(datetime.timedelta(15, 54915))"
+            "TIMEDELTA(datetime.timedelta(days=15, seconds=54915))"
         )
         self.assertEqual(
             repr(compile_timedelta(ConstantRule('15d15:15:15'))),
-            "TIMEDELTA(datetime.timedelta(15, 54915))"
+            "TIMEDELTA(datetime.timedelta(days=15, seconds=54915))"
         )
         self.assertEqual(
             repr(compile_timeoper(ConstantRule('2016-06-21T13:08:27Z'))),
@@ -158,19 +158,19 @@ class TestIDEAFilterCompiler(unittest.TestCase):
         )
         self.assertEqual(
             repr(compile_timeoper(NumberRule('3600'))),
-            "TIMEDELTA(datetime.timedelta(0, 3600))"
+            "TIMEDELTA(datetime.timedelta(seconds=3600))"
         )
         self.assertEqual(
             repr(compile_timeoper(ConstantRule('15:15:15'))),
-            "TIMEDELTA(datetime.timedelta(0, 54915))"
+            "TIMEDELTA(datetime.timedelta(seconds=54915))"
         )
         self.assertEqual(
             repr(compile_timeoper(ConstantRule('15D15:15:15'))),
-            "TIMEDELTA(datetime.timedelta(15, 54915))"
+            "TIMEDELTA(datetime.timedelta(days=15, seconds=54915))"
         )
         self.assertEqual(
             repr(compile_timeoper(ConstantRule('15d15:15:15'))),
-            "TIMEDELTA(datetime.timedelta(15, 54915))"
+            "TIMEDELTA(datetime.timedelta(days=15, seconds=54915))"
         )
 
 
@@ -347,12 +347,12 @@ class TestIDEAFilterCompiler(unittest.TestCase):
         rule = psr.parse('(DetectTime < (utcnow() - 3600))')
         self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_MINUS INTEGER(3600)))")
         res = cpl.compile(rule)
-        self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(0, 3600))))")
+        self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(seconds=3600))))")
 
         rule = psr.parse('(DetectTime + 3600) > utcnow()')
         self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('DetectTime') OP_PLUS INTEGER(3600)) OP_GT FUNCTION(utcnow()))")
         res = cpl.compile(rule)
-        self.assertEqual(repr(res), "COMPBINOP(MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(0, 3600))) OP_GT FUNCTION(utcnow()))")
+        self.assertEqual(repr(res), "COMPBINOP(MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(seconds=3600))) OP_GT FUNCTION(utcnow()))")
 
 
 #-------------------------------------------------------------------------------

pynspect/tests/test_filters_idea.py

@@ -315,7 +315,7 @@ class TestDataObjectFilterIDEA(unittest.TestCase):
         self.maxDiff = None
 
         rule = self.build_rule('DetectTime + 3600')
-        self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(0, 3600)))")
+        self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(seconds=3600)))")
         expected_res = (datetime.datetime(2016, 6, 21, 13, 8, 27) + datetime.timedelta(seconds = 3600))
         self.assertEqual(self.check_rule(rule), expected_res)
 
@@ -344,10 +344,10 @@ class TestDataObjectFilterIDEA(unittest.TestCase):
         self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LE DATETIME(datetime.datetime(2016, 6, 21, 14, 8, 27)))")
         self.assertEqual(self.check_rule(rule), True)
         rule = self.build_rule('DetectTime < (utcnow() + 05:00:00)')
-        self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_PLUS TIMEDELTA(datetime.timedelta(0, 18000))))")
+        self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_PLUS TIMEDELTA(datetime.timedelta(seconds=18000))))")
         self.assertEqual(self.check_rule(rule), True)
         rule = self.build_rule('DetectTime > (utcnow() - 05:00:00)')
-        self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_GT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(0, 18000))))")
+        self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_GT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(seconds=18000))))")
         self.assertEqual(self.check_rule(rule), False)
 
         rule = self.build_rule('(Source.IP4 == 188.14.166.39)')

pynspect/traversers.py

@@ -41,8 +41,8 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea
 
 
 import re
-import collections
 import datetime
+from collections.abc import MutableSequence
 
 from pynspect.rules import FilteringRuleException
 
@@ -524,7 +524,7 @@ def _to_numeric(val):
     return float(val)
 
 
-class ListIP(collections.MutableSequence):
+class ListIP(MutableSequence):
     """
     Special list implementation designed to provide special handling of 'IN' operator.
     When item is being compared using 'IN' operator with this list, the IN operation

requirements-dev.pip

@@ -2,8 +2,8 @@ setuptools
 wheel
 twine
 docutils<0.18
-nose==1.3.7
-pyflakes==2.1.0
-pylint==2.2.2
-sphinx==1.8.4
-sphinx-rtd-theme==0.4.2
+nose2
+pyflakes
+pylint
+sphinx
+sphinx-rtd-theme

setup.py

@@ -52,17 +52,18 @@ setup(
         'Programming Language :: Python'
     ],
     keywords = 'library',
-    url = 'https://gitlab.cesnet.cz/709/mentat/pynspect',
+    url = 'https://pypi.org/project/pynspect/',
+    project_urls={
+        'Documentation': 'https://709.gitlab-pages.cesnet.cz/mentat/pynspect/master/html/manual.html',
+        'Source': 'https://gitlab.cesnet.cz/709/mentat/pynspect',
+        'Tracker': 'https://gitlab.cesnet.cz/709/mentat/pynspect/-/issues'
+    },
     author = 'Jan Mach',
     author_email = 'honza.mach.ml@gmail.com',
     license = 'MIT',
     packages = [
         'pynspect'
     ],
-    test_suite = 'nose.collector',
-    tests_require = [
-        'nose'
-    ],
     install_requires=[
         'ipranges',
         'ply',