diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f1723d7772df9d8b0ad5fe6c0551619aebb38da5..f518e1d9b01cff7e7efd330dec5cf64bdc228bd6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ # Official language image. Look for the different tagged releases at: # https://hub.docker.com/r/library/python/tags/ -image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.6 +image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:latest # Change pip's cache directory to be inside the project directory since we can # only cache local items. @@ -15,7 +15,6 @@ variables: cache: paths: - .cache/pip - - venv/ before_script: - pip install virtualenv @@ -26,13 +25,50 @@ before_script: stages: # List of stages for jobs, and their order of execution - test + - check-warnings - build - deploy unit-test-job: stage: test script: - - make test + - make test 2>&1 | tee errors.log + artifacts: + when: always + paths: + - errors.log + reports: + junit: nose2-junit.xml + +unit-test-3.7-job: + image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.7 + stage: test + script: + - make test 2>&1 | tee errors-3.7.log + artifacts: + when: always + paths: + - errors-3.7.log + +unit-test-3.8-job: + image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.8 + stage: test + script: + - make test 2>&1 | tee errors-3.8.log + artifacts: + when: always + paths: + - errors-3.8.log + +unit-test-3.9-job: + image: ${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/python:3.9 + stage: test + script: + - make test 2>&1 | tee errors-3.9.log + artifacts: + when: always + paths: + - errors-3.9.log pylint-test-job: stage: test @@ -44,6 +80,34 @@ pyflakes-test-job: script: - make pyflakes +check-deprecation-warnings: + before_script: [] + stage: check-warnings + script: + - "if [[ $(grep DeprecationWarning errors.log) ]]; then cat errors.log; exit 1; fi" + allow_failure: true + +check-deprecation-warnings-3.7: + before_script: [] + stage: check-warnings + script: + - "if [[ $(grep DeprecationWarning errors-3.7.log) ]]; then cat errors-3.7.log; exit 1; fi" + allow_failure: true + +check-deprecation-warnings-3.8: + before_script: [] + stage: check-warnings + script: + - "if [[ $(grep DeprecationWarning errors-3.8.log) ]]; then cat errors-3.8.log; exit 1; fi" + allow_failure: true + +check-deprecation-warnings-3.9: + before_script: [] + stage: check-warnings + script: + - "if [[ $(grep DeprecationWarning errors-3.9.log) ]]; then cat errors-3.9.log; exit 1; fi" + allow_failure: true + build-job: stage: build script: @@ -105,4 +169,4 @@ pages: - public only: - master - - devel \ No newline at end of file + - devel diff --git a/Makefile b/Makefile index 610c3c0f122201579eb4e8d5bf74348893b597cf..7a340340536d6f6d675c6a84ea3ecc1e2224ed42 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,7 @@ pylint-test: FORCE test: FORCE @echo "\n${GREEN}*** Checking code with nosetests ***${NC}\n" - @$(NOSETESTS) + @$(PYTHON) -W always::DeprecationWarning -m nose2 --junit-xml #------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 1d2e5c524f3506c0b19a5910a41a06869f43407b..d67cdd278a8882397d96d99a66ed745853b8e2dd 100644 --- a/README.rst +++ b/README.rst @@ -56,3 +56,19 @@ Copyright | Copyright (C) since 2016 Jan Mach <honza.mach.ml@gmail.com> | Use of this package is governed by the MIT license, see LICENSE file. | + + +Changelog +-------------------------------------------------------------------------------- + + +Version 0.21 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Released 2022-06-28 + +- Dropped support for Python 3.6. +- Fixed deprecation warnings for Python 3.7+ regarding ``collections.abc``. +- Added a config file for GitLab CI/CD. +- Updated the repository information. +- Updated packages versions. diff --git a/nose2.cfg b/nose2.cfg new file mode 100644 index 0000000000000000000000000000000000000000..a7850f000fd8fb9a4b96d43d4bb81e8122cdc388 --- /dev/null +++ b/nose2.cfg @@ -0,0 +1,2 @@ +[unittest] +plugins = nose2.plugins.junitxml diff --git a/pynspect/__init__.py b/pynspect/__init__.py index 34dfcb69a1740656648f36f4787608cb1a105d54..ccd0225d112992d5d991c8c9bcae4a4cb7285f79 100644 --- a/pynspect/__init__.py +++ b/pynspect/__init__.py @@ -16,4 +16,4 @@ data structures. """ -__version__ = "0.20" +__version__ = "0.21" diff --git a/pynspect/jpath.py b/pynspect/jpath.py index 2286507868254e47ed5195ce64b0d275fca2207e..41edff3d4032fb76243e5ce3fa9921f354d74c7f 100644 --- a/pynspect/jpath.py +++ b/pynspect/jpath.py @@ -133,7 +133,7 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>" import re -import collections +from collections.abc import Mapping, MutableSequence # @@ -273,7 +273,7 @@ def jpath_values(structure, jpath): # Process all currently active nodes. for node in nodes_a: key = chnk['n'] - if not isinstance(node, dict) and not isinstance(node, collections.Mapping): + if not isinstance(node, dict) and not isinstance(node, Mapping): continue # Process indexed nodes. @@ -281,7 +281,7 @@ def jpath_values(structure, jpath): idx = chnk['i'] # Skip the node, if the key does not exist, the value is not # a list-like object or the list is empty. - if not key in node or not (isinstance(node[key], (list, collections.MutableSequence))) or not node[key]: + if not key in node or not (isinstance(node[key], (list, MutableSequence))) or not node[key]: continue try: # Handle '*' special index - append all nodes. @@ -300,7 +300,7 @@ def jpath_values(structure, jpath): continue # Handle list values - expand them. - if isinstance(node[key], (list, collections.MutableSequence)): + if isinstance(node[key], (list, MutableSequence)): for i in node[key]: nodes_b.append(i) # Handle scalar values. @@ -369,7 +369,7 @@ def jpath_set(structure, jpath, value, overwrite = True, unique = False): for i, chnk in enumerate(chunks): key = chnk['n'] - if not isinstance(current, dict) and not isinstance(current, collections.Mapping): + if not isinstance(current, dict) and not isinstance(current, Mapping): raise JPathException("Expected dict-like structure to attach node '{}'".format(chnk['p'])) # Process indexed nodes. @@ -379,7 +379,7 @@ def jpath_set(structure, jpath, value, overwrite = True, unique = False): # Automatically create nodes for non-existent keys. if not key in current: current[key] = [] - if not isinstance(current[key], list) and not isinstance(current[key], collections.MutableSequence): + if not isinstance(current[key], list) and not isinstance(current[key], MutableSequence): raise JPathException("Expected list-like object under structure key '{}'".format(key)) # Detection of the last JPath chunk - node somewhere in the middle. @@ -426,7 +426,7 @@ def jpath_set(structure, jpath, value, overwrite = True, unique = False): # Automatically create nodes for non-existent keys. if not key in current: current[key] = {} - if not isinstance(current[key], dict) and not isinstance(current[key], collections.Mapping): + if not isinstance(current[key], dict) and not isinstance(current[key], Mapping): raise JPathException("Expected dict-like object under structure key '{}'".format(key)) current = current[key] @@ -464,7 +464,7 @@ def jpath_unset(structure, jpath): for node in nodes_a: key = chnk['n'] - if not isinstance(node, dict) and not isinstance(node, collections.Mapping): + if not isinstance(node, dict) and not isinstance(node, Mapping): raise JPathException("Expected dict-like structure to drop node '{}'".format(chnk['p'])) # Process indexed nodes. @@ -474,7 +474,7 @@ def jpath_unset(structure, jpath): # Skip nodes for non-existent keys. if not key in node: continue - if not isinstance(node[key], list) and not isinstance(node[key], collections.MutableSequence): + if not isinstance(node[key], list) and not isinstance(node[key], MutableSequence): raise JPathException("Expected list-like object under structure key '{}'".format(key)) # Detection of the last JPath chunk - node somewhere in the middle. @@ -513,7 +513,7 @@ def jpath_unset(structure, jpath): if isinstance(node[key], list): nodes_b.extend(node[key]) continue - if not isinstance(node[key], dict) and not isinstance(node[key], collections.Mapping): + if not isinstance(node[key], dict) and not isinstance(node[key], Mapping): raise JPathException("Expected dict-like object under structure key '{}'".format(key)) nodes_b.append(node[key]) diff --git a/pynspect/tests/test_compilers.py b/pynspect/tests/test_compilers.py index 167fbb1b295e2e4346340ec84352a074ffcddcbd..2309b9441245e6d8d66932919912c2b2c1d7ac5d 100644 --- a/pynspect/tests/test_compilers.py +++ b/pynspect/tests/test_compilers.py @@ -130,23 +130,23 @@ class TestIDEAFilterCompiler(unittest.TestCase): ) self.assertEqual( repr(compile_timedelta(NumberRule('3600'))), - "TIMEDELTA(datetime.timedelta(0, 3600))" + "TIMEDELTA(datetime.timedelta(seconds=3600))" ) self.assertEqual( repr(compile_timedelta(ConstantRule('3600'))), - "TIMEDELTA(datetime.timedelta(0, 3600))" + "TIMEDELTA(datetime.timedelta(seconds=3600))" ) self.assertEqual( repr(compile_timedelta(ConstantRule('15:15:15'))), - "TIMEDELTA(datetime.timedelta(0, 54915))" + "TIMEDELTA(datetime.timedelta(seconds=54915))" ) self.assertEqual( repr(compile_timedelta(ConstantRule('15D15:15:15'))), - "TIMEDELTA(datetime.timedelta(15, 54915))" + "TIMEDELTA(datetime.timedelta(days=15, seconds=54915))" ) self.assertEqual( repr(compile_timedelta(ConstantRule('15d15:15:15'))), - "TIMEDELTA(datetime.timedelta(15, 54915))" + "TIMEDELTA(datetime.timedelta(days=15, seconds=54915))" ) self.assertEqual( repr(compile_timeoper(ConstantRule('2016-06-21T13:08:27Z'))), @@ -158,19 +158,19 @@ class TestIDEAFilterCompiler(unittest.TestCase): ) self.assertEqual( repr(compile_timeoper(NumberRule('3600'))), - "TIMEDELTA(datetime.timedelta(0, 3600))" + "TIMEDELTA(datetime.timedelta(seconds=3600))" ) self.assertEqual( repr(compile_timeoper(ConstantRule('15:15:15'))), - "TIMEDELTA(datetime.timedelta(0, 54915))" + "TIMEDELTA(datetime.timedelta(seconds=54915))" ) self.assertEqual( repr(compile_timeoper(ConstantRule('15D15:15:15'))), - "TIMEDELTA(datetime.timedelta(15, 54915))" + "TIMEDELTA(datetime.timedelta(days=15, seconds=54915))" ) self.assertEqual( repr(compile_timeoper(ConstantRule('15d15:15:15'))), - "TIMEDELTA(datetime.timedelta(15, 54915))" + "TIMEDELTA(datetime.timedelta(days=15, seconds=54915))" ) @@ -347,12 +347,12 @@ class TestIDEAFilterCompiler(unittest.TestCase): rule = psr.parse('(DetectTime < (utcnow() - 3600))') self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_MINUS INTEGER(3600)))") res = cpl.compile(rule) - self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(0, 3600))))") + self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(seconds=3600))))") rule = psr.parse('(DetectTime + 3600) > utcnow()') self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('DetectTime') OP_PLUS INTEGER(3600)) OP_GT FUNCTION(utcnow()))") res = cpl.compile(rule) - self.assertEqual(repr(res), "COMPBINOP(MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(0, 3600))) OP_GT FUNCTION(utcnow()))") + self.assertEqual(repr(res), "COMPBINOP(MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(seconds=3600))) OP_GT FUNCTION(utcnow()))") #------------------------------------------------------------------------------- diff --git a/pynspect/tests/test_filters_idea.py b/pynspect/tests/test_filters_idea.py index 6e753a537cb7dcbeae07ab5122781e711c6532da..7424e28fd54f2c5f04ab3289401230c42af934d2 100644 --- a/pynspect/tests/test_filters_idea.py +++ b/pynspect/tests/test_filters_idea.py @@ -315,7 +315,7 @@ class TestDataObjectFilterIDEA(unittest.TestCase): self.maxDiff = None rule = self.build_rule('DetectTime + 3600') - self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(0, 3600)))") + self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(seconds=3600)))") expected_res = (datetime.datetime(2016, 6, 21, 13, 8, 27) + datetime.timedelta(seconds = 3600)) self.assertEqual(self.check_rule(rule), expected_res) @@ -344,10 +344,10 @@ class TestDataObjectFilterIDEA(unittest.TestCase): self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LE DATETIME(datetime.datetime(2016, 6, 21, 14, 8, 27)))") self.assertEqual(self.check_rule(rule), True) rule = self.build_rule('DetectTime < (utcnow() + 05:00:00)') - self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_PLUS TIMEDELTA(datetime.timedelta(0, 18000))))") + self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_PLUS TIMEDELTA(datetime.timedelta(seconds=18000))))") self.assertEqual(self.check_rule(rule), True) rule = self.build_rule('DetectTime > (utcnow() - 05:00:00)') - self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_GT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(0, 18000))))") + self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_GT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(seconds=18000))))") self.assertEqual(self.check_rule(rule), False) rule = self.build_rule('(Source.IP4 == 188.14.166.39)') diff --git a/pynspect/traversers.py b/pynspect/traversers.py index a6ce8a6634572e127ef2818ca379ec9c1261ca71..bf494612b4632f3bf1b408ffcc4c53c6b10b4095 100644 --- a/pynspect/traversers.py +++ b/pynspect/traversers.py @@ -41,8 +41,8 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea import re -import collections import datetime +from collections.abc import MutableSequence from pynspect.rules import FilteringRuleException @@ -524,7 +524,7 @@ def _to_numeric(val): return float(val) -class ListIP(collections.MutableSequence): +class ListIP(MutableSequence): """ Special list implementation designed to provide special handling of 'IN' operator. When item is being compared using 'IN' operator with this list, the IN operation diff --git a/requirements-dev.pip b/requirements-dev.pip index e92d9cb3c7fc873c0b1fe2936e839410bac9f1ed..67caa2415dccb8ca41123a036f0e606d547b4c1d 100644 --- a/requirements-dev.pip +++ b/requirements-dev.pip @@ -2,8 +2,8 @@ setuptools wheel twine docutils<0.18 -nose==1.3.7 -pyflakes==2.1.0 -pylint==2.2.2 -sphinx==1.8.4 -sphinx-rtd-theme==0.4.2 +nose2 +pyflakes +pylint +sphinx +sphinx-rtd-theme diff --git a/setup.py b/setup.py index 5161ada2e6af611c04d3dd733d98c60aef758c34..dcd70e11965008db88aa776656b5ebaa9528bdf7 100644 --- a/setup.py +++ b/setup.py @@ -52,17 +52,18 @@ setup( 'Programming Language :: Python' ], keywords = 'library', - url = 'https://gitlab.cesnet.cz/709/mentat/pynspect', + url = 'https://pypi.org/project/pynspect/', + project_urls={ + 'Documentation': 'https://709.gitlab-pages.cesnet.cz/mentat/pynspect/master/html/manual.html', + 'Source': 'https://gitlab.cesnet.cz/709/mentat/pynspect', + 'Tracker': 'https://gitlab.cesnet.cz/709/mentat/pynspect/-/issues' + }, author = 'Jan Mach', author_email = 'honza.mach.ml@gmail.com', license = 'MIT', packages = [ 'pynspect' ], - test_suite = 'nose.collector', - tests_require = [ - 'nose' - ], install_requires=[ 'ipranges', 'ply',