diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000000000000000000000000000000000000..dae9093320d1b520f831d4ac0c6bf96645014edf
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,6 @@
+[submodule "submodules/pyzenkit"]
+ path = submodules/pyzenkit
+ url = https://github.com/honzamach/pyzenkit.git
+[submodule "submodules/pynspect"]
+ path = submodules/pynspect
+ url = https://github.com/honzamach/pynspect.git
diff --git a/deploy/pynspect/LICENSE.txt b/deploy/pynspect/LICENSE.txt
deleted file mode 100644
index 18202f1e946329bf84dd5dbfc4a259af90cfe234..0000000000000000000000000000000000000000
--- a/deploy/pynspect/LICENSE.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (C) since 2011 CESNET, z.s.p.o
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/deploy/pynspect/Makefile b/deploy/pynspect/Makefile
deleted file mode 100644
index f25681a9eb6c309337ab80011c42c9e210fd28c5..0000000000000000000000000000000000000000
--- a/deploy/pynspect/Makefile
+++ /dev/null
@@ -1,71 +0,0 @@
-#-------------------------------------------------------------------------------
-# Copyright (c) since 2016, CESNET, z. s. p. o.
-# Authors: Pavel Kácha <pavel.kacha@cesnet.cz>
-# Jan Mach <jan.mach@cesnet.cz>
-# Use of this source is governed by an ISC license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-all: archive sync bdist deploy
-
-build: archive sync bdist
-
-buildbot: sync bdist
-
-help:
- $(info List of possible make targets:)
- $(info )
- $(info * all: archive previous packages, build new distribution and deploy to PyPI [default])
- $(info * build: archive previous packages and build new distribution)
- $(info * test: run unit tests)
- $(info * archive: archive previous packages)
- $(info * sync: synchronize source code)
- $(info * bdist: build new distribution)
- $(info * install: install distribution on local machine)
- $(info * deploy: deploy to PyPI)
- $(info )
-
-# Perform unit tests
-test: FORCE
- $(info Testing source code)
- nosetests
-
-# Move old distribution files to archive directory
-archive: FORCE
- $(info Checking if dist archivation is needed)
- @if ! [ `ls dist/pynspect* | wc -l` = "0" ]; then\
- echo "Moving old distribution files to local archive";\
- mv -f dist/pynspect* archive;\
- fi
-
-# Synchronize directory contents
-sync: FORCE
- $(info Syncing source code)
- rsync -r --progress --archive --update --delete --force ../../lib/pynspect ./
- find ./ -type f -name *.pyc -exec rm -f {} \;
-
-# Build various Python package distributions
-bdist:
- $(info Building distributions)
-
- # Build Python3 only wheel and egg.
- python3 setup.py sdist bdist_wheel
-
- # Or build universal wheel and egg.
- #python3 setup.py sdist bdist_wheel --universal
-
-# Perform installation from local files for both Python 2 and 3
-install: FORCE
- $(info Local installation)
- #pip install dist/pynspect*.whl
- pip3 install dist/pynspect*.whl
-
-# Deploy latest packages to PyPI
-deploy: FORCE
- $(info PyPI deployment)
-
- # Secure upload with Twine
- twine upload dist/pynspect*
-
-# Empty rule as dependency wil force make to always perform target
-# Source: https://www.gnu.org/software/make/manual/html_node/Force-Targets.html
-FORCE:
diff --git a/deploy/pynspect/README.rst b/deploy/pynspect/README.rst
deleted file mode 100644
index 230af994ee87ee4e133add7cf21b3921daa90641..0000000000000000000000000000000000000000
--- a/deploy/pynspect/README.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-pynspect
-================================================================================
-
-Python 3 library for filtering, querying or inspecting almost arbitrary data
-structures.
-
-This README file is work in progress, for more information please consult source
-code and unit tests.
diff --git a/deploy/pynspect/archive/.gitplaceholder b/deploy/pynspect/archive/.gitplaceholder
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/deploy/pynspect/dist/.gitplaceholder b/deploy/pynspect/dist/.gitplaceholder
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/deploy/pynspect/setup.py b/deploy/pynspect/setup.py
deleted file mode 100644
index 15fcc0023f2fcb45f35aca485125cb51a6c77775..0000000000000000000000000000000000000000
--- a/deploy/pynspect/setup.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# Copyright (c) since 2016, CESNET, z. s. p. o.
-# Authors: Jan Mach <jan.mach@cesnet.cz>
-# Pavel Kácha <pavel.kacha@cesnet.cz>
-# Use of this source is governed by an ISC license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-# Resources:
-# https://packaging.python.org/distributing/
-# http://python-packaging-user-guide.readthedocs.io/distributing/
-
-# Always prefer setuptools over distutils
-from setuptools import setup, find_packages
-# To use a consistent encoding
-from codecs import open
-from os import path
-
-here = path.abspath(path.dirname(__file__))
-
-# Get the long description from the README file
-with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
- long_description = f.read()
-
-setup(
- name = 'pynspect',
- version = '0.4',
- description = 'Python 3 library for filtering, querying or inspecting almost arbitrary data structures.',
- long_description = long_description,
- classifiers = [
- 'Development Status :: 3 - Alpha',
- 'License :: OSI Approved :: MIT License',
- 'Programming Language :: Python :: 3 :: Only'
- ],
- keywords = 'library',
- url = 'https://homeproj.cesnet.cz/git/mentat-ng.git',
- author = 'Jan Mach',
- author_email = 'jan.mach@cesnet.cz',
- license = 'MIT',
- packages = ['pynspect'],
- test_suite = 'nose.collector',
- tests_require = [
- 'nose'
- ],
- install_requires=[
- 'ipranges'
- ],
- zip_safe = True
-)
diff --git a/doc/sphinx/_doclib/lib.pynspect.filters.rst b/doc/sphinx/_doclib/lib.pynspect.filters.rst
deleted file mode 100644
index 9dc77a30676e5dca050e29fe483e04b0c0dcb8ae..0000000000000000000000000000000000000000
--- a/doc/sphinx/_doclib/lib.pynspect.filters.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-pynspect.filters module
-================================================================================
-
-.. automodule:: pynspect.filters
- :show-inheritance:
- :members:
- :undoc-members:
diff --git a/doc/sphinx/_doclib/lib.pynspect.gparser.rst b/doc/sphinx/_doclib/lib.pynspect.gparser.rst
deleted file mode 100644
index f63c4135de9ead5f33d7bb680fd2dfbc2af63ab6..0000000000000000000000000000000000000000
--- a/doc/sphinx/_doclib/lib.pynspect.gparser.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-pynspect.gparser module
-================================================================================
-
-.. automodule:: pynspect.gparser
- :show-inheritance:
diff --git a/doc/sphinx/_doclib/lib.pynspect.jpath.rst b/doc/sphinx/_doclib/lib.pynspect.jpath.rst
deleted file mode 100644
index 0484b30f29e9333ee0f48725c16ed79fdcf5f575..0000000000000000000000000000000000000000
--- a/doc/sphinx/_doclib/lib.pynspect.jpath.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-pynspect.jpath module
-================================================================================
-
-.. automodule:: pynspect.jpath
- :members:
- :undoc-members:
- :show-inheritance:
diff --git a/doc/sphinx/_doclib/lib.pynspect.lexer.rst b/doc/sphinx/_doclib/lib.pynspect.lexer.rst
deleted file mode 100644
index 9088a14cbde683ec9981bc688f710779701932d3..0000000000000000000000000000000000000000
--- a/doc/sphinx/_doclib/lib.pynspect.lexer.rst
+++ /dev/null
@@ -1,5 +0,0 @@
-pynspect.lexer module
-================================================================================
-
-.. automodule:: pynspect.lexer
- :show-inheritance:
diff --git a/doc/sphinx/_doclib/lib.pynspect.rules.rst b/doc/sphinx/_doclib/lib.pynspect.rules.rst
deleted file mode 100644
index 81dc0f0f06871b05e4f6fa1e005f729cd8ba3692..0000000000000000000000000000000000000000
--- a/doc/sphinx/_doclib/lib.pynspect.rules.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-pynspect.rules module
-================================================================================
-
-.. automodule:: pynspect.rules
- :show-inheritance:
- :members:
- :undoc-members:
diff --git a/lib/pynspect/__init__.py b/lib/pynspect/__init__.py
deleted file mode 100644
index f04b228f73878c52d39d43cbb1e92857c28ca5c1..0000000000000000000000000000000000000000
--- a/lib/pynspect/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-__version__ = "0.3"
diff --git a/lib/pynspect/benchmark/bench_jpath.py b/lib/pynspect/benchmark/bench_jpath.py
deleted file mode 100644
index 6673bf579d24c2f91770cdad06d5c2e22ff9138e..0000000000000000000000000000000000000000
--- a/lib/pynspect/benchmark/bench_jpath.py
+++ /dev/null
@@ -1,143 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-import os
-import sys
-import shutil
-import random
-import string
-import timeit
-from pprint import pformat, pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../lib'))
-sys.path.insert(0, lib)
-
-from pynspect.jpath import *
-
-#-------------------------------------------------------------------------------
-# HELPER FUNCTIONS
-#-------------------------------------------------------------------------------
-
-def random_jpath(depth = 3):
- """
- Generate random JPath with given node depth.
- """
- chunks = []
- while depth > 0:
- length = random.randint(5, 15)
- ident = ''.join(random.choice(string.ascii_uppercase + string.ascii_lowercase) for _ in range(length))
- if random.choice((True, False)):
- index = random.randint(0, 10)
- ident = "{:s}[{:d}]".format(ident, index)
- chunks.append(ident)
- depth -= 1
- return ".".join(chunks)
-
-RANDOM_JPATHS = [random_jpath(random.randint(1,5)) for i in range(50)]
-"""Pregenerated list of random JPaths."""
-
-#-------------------------------------------------------------------------------
-# BENCHMARK TESTS
-#-------------------------------------------------------------------------------
-
-b001 = jpath_parse
-b002 = jpath_parse_c
-
-def b003():
- jpath = random.choice(RANDOM_JPATHS)
- return jpath_parse(jpath)
-
-def b004():
- jpath = random.choice(RANDOM_JPATHS)
- return jpath_parse_c(jpath)
-
-#-------------------------------------------------------------------------------
-
-if __name__ == "__main__":
- """
- Performance benchmarking of :py:mod:`pynspect.jpath` module.
- """
-
- print("\n BENCHMARKING MENTAT.FILTERING.JPATH MODULE\n")
-
- print("=" * 84)
- print(" {:22s} | {:16s} | {:20s} | {:20s}".format(
- "Name",
- "Iterations (#)",
- "Duration (s)",
- "Speed (#/s)"))
- print("=" * 84)
- format_ptrn = " {:22s} | {:16,d} | {:20.10f} | {:15,.3f}"
-
- #---------------------------------------------------------------------------
-
- iterations = 1000000
-
- """
- Parsing of single reasonably complex JPath without caching.
- """
- result = timeit.timeit('b001("Long[*].Test.Path[*]")', number = iterations, setup = "from __main__ import b001")
- speed = iterations / result
- print(
- format_ptrn.format(
- "jpath_parse",
- iterations,
- result,
- speed
- )
- )
- """
- Parsing of single reasonably complex JPath with caching.
- """
- result = timeit.timeit('b002("Long[*].Test.Path[*]")', number = iterations, setup = "from __main__ import b002")
- speed = iterations / result
- print(
- format_ptrn.format(
- "jpath_parse_c",
- iterations,
- result,
- speed
- )
- )
-
- #---------------------------------------------------------------------------
-
- iterations = 1000000
-
- """
- Parsing of random reasonably complex JPath without caching.
- """
- result = timeit.timeit('b003()', number = iterations, setup = "from __main__ import b003")
- speed = iterations / result
- print(
- format_ptrn.format(
- "jpath_parse (random)",
- iterations,
- result,
- speed
- )
- )
- """
- Parsing of random reasonably complex JPath with caching.
- """
- result = timeit.timeit('b004()', number = iterations, setup = "from __main__ import b004")
- speed = iterations / result
- print(
- format_ptrn.format(
- "jpath_parse_c (random)",
- iterations,
- result,
- speed
- )
- )
-
- #---------------------------------------------------------------------------
-
- print("=" * 84)
diff --git a/lib/pynspect/filters.py b/lib/pynspect/filters.py
deleted file mode 100644
index 05d4de97fc039717c188041ceb848618d292ec48..0000000000000000000000000000000000000000
--- a/lib/pynspect/filters.py
+++ /dev/null
@@ -1,267 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-"""
-This module provides tools for data filtering based on filtering and query
-grammar.
-
-The filtering grammar is thoroughly described in following module:
-
-* :py:mod:`pynspect.lexer`
-
- Lexical analyzer, descriptions of valid grammar tokens.
-
-* :py:mod:`pynspect.gparser`
-
- Grammar parser, language grammar description
-
-* :py:mod:`pynspect.rules`
-
- Object representation of grammar rules, interface definition
-
-* :py:mod:`pynspect.jpath`
-
- The addressing language JPath.
-
-Please refer to appropriate module for more in-depth information.
-
-There are two main tools in this package:
-
-* :py:class:`DataObjectFilter`
-
- Tool capable of filtering data structures according to given filtering rules.
-
-* :py:class:`IDEAFilterCompiler`
-
- Filter compiler, that ensures appropriate data types for correct variable
- comparison evaluation.
-
-.. todo::
-
- There is quite a lot of code that needs to be written before actual filtering
- can take place. In the future, there should be some kind of object, that will
- be tailored for immediate processing and will take care of initializing
- uderlying parser, compiler and filter. This object will be designed later.
-
-"""
-
-__version__ = "0.3"
-__author__ = "Jan Mach <jan.mach@cesnet.cz>"
-__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-
-import re
-import ipranges
-
-from pynspect.rules import *
-from pynspect.jpath import *
-
-class DataObjectFilter(RuleTreeTraverser):
- """
- Rule tree traverser implementing default object filtering logic.
-
- Following example demonstrates DataObjectFilter usage in conjuction with
- MentatFilterParser::
-
- >>> flt = DataObjectFilter()
- >>> psr = MentatFilterParser()
- >>> psr.build()
- >>> rule = psr.parse('ID like "e214d2d9"')
- >>> result = flt.filter(rule, test_msg)
-
- Alternativelly rule tree can be created by hand/programatically:
-
- >>> rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(1))
- >>> result = flt.filter(rule, test_msg1)
- """
- def filter(self, rule, data):
- """
- Apply given filtering rule to given data structure.
-
- :param Rule rule: filtering rule to be checked
- :param any data: data structure to check against rule, ussually dict
- :return: True or False or expression result
- :rtype: bool or any
- """
- return rule.traverse(self, obj = data)
-
- #---------------------------------------------------------------------------
-
- def ipv4(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return rule.value
- def ipv6(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return rule.value
- def integer(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return rule.value
- def constant(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return rule.value
- def variable(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return jpath_values(kwargs['obj'], rule.value)
- def list(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return [i.value for i in rule.value]
- def binary_operation_logical(self, rule, left, right, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return self.evaluate_binop_logical(rule.operation, left, right, **kwargs)
- def binary_operation_comparison(self, rule, left, right, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return self.evaluate_binop_comparison(rule.operation, left, right, **kwargs)
- def binary_operation_math(self, rule, left, right, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return self.evaluate_binop_math(rule.operation, left, right, **kwargs)
- def unary_operation(self, rule, right, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return self.evaluate_unop(rule.operation, right, **kwargs)
-
-def compile_ip_v4(rule):
- """
- Compiler helper method: attempt to compile constant into object representing
- IPv4 address to enable relations and thus simple comparisons using Python
- operators.
- """
- if isinstance(rule.value, ipranges.Range):
- return rule
- return IPV4Rule(ipranges.from_str_v4(rule.value))
-
-def compile_ip_v6(rule):
- """
- Compiler helper method: attempt to compile constant into object representing
- IPv6 address to enable relations and thus simple comparisons using Python
- operators.
- """
- if isinstance(rule.value, ipranges.Range):
- print("IPv6 {} already compiled".format(rule.value))
- return rule
- print("Compiling IPv6 {} to Range object".format(rule.value))
- return IPV6Rule(ipranges.from_str_v6(rule.value))
-
-CVRE = re.compile('\[\d+\]')
-def clean_variable(var):
- """
- Remove any array indices from variable name to enable indexing into :py:data:`COMPILATIONS`
- callback dictionary.
-
- This dictionary contains postprocessing callback appropriate for opposing
- operand of comparison operation for variable on given JPath.
- """
- return CVRE.sub('', var)
-
-COMPILATIONS = {
- 'Source.IP4': compile_ip_v4,
- 'Target.IP4': compile_ip_v4,
- 'Source.IP6': compile_ip_v6,
- 'Target.IP6': compile_ip_v6,
-}
-
-class IDEAFilterCompiler(RuleTreeTraverser):
- """
- Rule tree traverser implementing IDEA filter compilation algorithm.
-
- Following example demonstrates DataObjectFilter usage in conjuction with
- MentatFilterParser::
-
- >>> msg_idea = lite.Idea(test_msg)
- >>> flt = DataObjectFilter()
- >>> cpl = IDEAFilterCompiler()
- >>> psr = MentatFilterParser()
- >>> psr.build()
- >>> rule = psr.parse('ID like "e214d2d9"')
- >>> rule = cpl.compile(rule)
- >>> result = flt.filter(rule, test_msg)
- """
- def compile(self, rule):
- """
- Compile given filtering rule into format appropriate for processing IDEA
- messages.
-
- :param Rule rule: filtering rule to be compiled
- :return: compiled filtering rule
- :rtype: Rule
- """
- return rule.traverse(self)
-
- #---------------------------------------------------------------------------
-
- def ipv4(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- rule = compile_ip_v4(rule)
- return rule
- def ipv6(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- rule = compile_ip_v4(rule)
- return rule
- def integer(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- rule.value = int(rule.value)
- return rule
- def constant(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return rule
- def variable(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return rule
- def list(self, rule, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return rule
- def binary_operation_logical(self, rule, left, right, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return LogicalBinOpRule(rule.operation, left, right)
- def binary_operation_comparison(self, rule, left, right, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- var = val = None
- if isinstance(left, VariableRule) and not isinstance(right, VariableRule):
- var = left
- val = right
- elif isinstance(right, VariableRule) and not isinstance(left, VariableRule):
- var = right
- val = left
- if var and val:
- p = clean_variable(var.value)
- if p in COMPILATIONS.keys():
- if isinstance(val, ListRule):
- result = []
- for v in val.value:
- result.append(COMPILATIONS[p](v))
- right = ListRule(result)
- else:
- right = COMPILATIONS[p](val)
- return ComparisonBinOpRule(rule.operation, left, right)
- def binary_operation_math(self, rule, left, right, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- if isinstance(left, IntegerRule) and isinstance(right, IntegerRule):
- result = self.evaluate_binop_math(rule.operation, left.value, right.value)
- if isinstance(result, list):
- return ListRule([IntegerRule(r) for r in result])
- else:
- return IntegerRule(result)
- elif isinstance(left, NumberRule) and isinstance(right, NumberRule):
- result = self.evaluate_binop_math(rule.operation, left.value, right.value)
- if isinstance(result, list):
- return ListRule([FloatRule(r) for r in result])
- else:
- return FloatRule(result)
- return MathBinOpRule(rule.operation, left, right)
- def unary_operation(self, rule, right, **kwargs):
- """Implementation of :py:class:`pynspect.rules.RuleTreeTraverser` interface"""
- return UnaryOperationRule(rule.operation, right)
-
-if __name__ == "__main__":
- """
- Perform the demonstration.
- """
- import pprint
-
- data = {"Test": 15, "Attr": "ABC"}
- rule = ComparisonBinOpRule('OP_GT', VariableRule("Test"), IntegerRule(10))
- flt = DataObjectFilter()
- pprint.pprint(flt.filter(rule, data))
diff --git a/lib/pynspect/gparser.py b/lib/pynspect/gparser.py
deleted file mode 100644
index a253b9fca13afa8691828439260e1f5696c356e4..0000000000000000000000000000000000000000
--- a/lib/pynspect/gparser.py
+++ /dev/null
@@ -1,332 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-"""
-This module contains object encapsulation of `PLY <http://www.dabeaz.com/ply/>`__
-parser for filtering and query language grammar used in Mentat project.
-
-Grammar features
-^^^^^^^^^^^^^^^^
-
-* Logical operations: ``and or xor not exists``
-
- All logical operations support upper case and lower case name variants.
- Additionally, there are also symbolic variants ``|| ^^ && ! ?`` with higher
- priority and which can be used in some cases instead of parentheses.
-
-* Comparison operations: ``like in is eq ne gt ge lt le``
-
- All comparison operations support upper case and lower case name variants.
- Additionally, there are also symbolic variants.
-
-* Mathematical operations: ``+ - * / %``
-* JPath variables: ``Source[0].IP4[1]``
-* Directly recognized constants:
-
- * IPv4: ``127.0.0.1 127.0.0.1/32 127.0.0.1-127.0.0.5 127.0.0.1..127.0.0.5``
- * IPv6: ``::1 ::1/64 ::1-::5 ::1..::5``
- * integer: ``0 1 42``
- * float: ``3.14159``
-* Quoted literal constants: ``"double quotes"`` or ``'single quotes'``
-
-For more details on supported grammar token syntax please see the documentation
-of :py:mod:`pynspect.lexer` module.
-
-Currently implemented grammar
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. code-block:: bnf
-
- expression : xor_expression OP_OR expression
- | xor_expression
-
- xor_expression : and_expression OP_XOR xor_expression
- | and_expression
-
- and_expression : or_p_expression OP_AND and_expression
- | or_p_expression
-
- or_p_expression : xor_p_expression OP_OR_P or_p_expression
- | xor_p_expression
-
- xor_p_expression : and_p_expression OP_XOR_P xor_p_expression
- | and_p_expression
-
- and_p_expression : not_expression OP_AND_P and_p_expression
- | not_expression
-
- not_expression : OP_NOT ex_expression
- | ex_expression
-
- ex_expression : OP_EXISTS cmp_expression
- | cmp_expression
-
- cmp_expression : term OP_LIKE cmp_expression
- | term OP_IN cmp_expression
- | term OP_IS cmp_expression
- | term OP_EQ cmp_expression
- | term OP_NE cmp_expression
- | term OP_GT cmp_expression
- | term OP_GE cmp_expression
- | term OP_LT cmp_expression
- | term OP_LE cmp_expression
- | term
-
- term : factor OP_PLUS term
- | factor OP_MINUS term
- | factor OP_TIMES term
- | factor OP_DIVIDE term
- | factor OP_MODULO term
- | factor
-
- factor : IPV4
- | IPV6
- | INTEGER
- | FLOAT
- | VARIABLE
- | CONSTANT
- | LBRACK list RBRACK
- | LPAREN expression RPAREN
-
- list : IPV4
- | IPV6
- | INTEGER
- | FLOAT
- | VARIABLE
- | CONSTANT
- | IPV4 COMMA list
- | IPV6 COMMA list
- | INTEGER COMMA list
- | FLOAT COMMA list
- | VARIABLE COMMA list
- | CONSTANT COMMA list
-
-.. note::
-
- Implementation of this module is very *PLY* specific, please read the
- appropriate `documentation <http://www.dabeaz.com/ply/ply.html#ply_nn3>`__
- to understand it.
-
-"""
-
-__version__ = "0.3"
-__author__ = "Jan Mach <jan.mach@cesnet.cz>"
-__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-
-import re, logging
-import ply.yacc
-
-from pynspect.lexer import MentatFilterLexer
-from pynspect.rules import *
-
-class MentatFilterParser():
- """
- Object encapsulation of *PLY* parser implementation for filtering and
- query language grammar used in Mentat project.
- """
-
- def build(self, **kwargs):
- """
- Build/rebuild the parser object
- """
- self.logger = logging.getLogger('ply_parser')
-
- self.lexer = MentatFilterLexer()
- self.lexer.build()
-
- self.tokens = self.lexer.tokens
-
- self.parser = ply.yacc.yacc(
- module=self
- #start='statements',
- #debug=yacc_debug,
- #optimize=yacc_optimize,
- #tabmodule=yacctab
- )
-
- def parse(self, data, filename='', debuglevel=0):
- """
- Parse given data.
-
- data:
- A string containing the filter definition
- filename:
- Name of the file being parsed (for meaningful
- error messages)
- debuglevel:
- Debug level to yacc
- """
- self.lexer.filename = filename
- self.lexer.reset_lineno()
- if not data or data.isspace():
- return []
- else:
- return self.parser.parse(data, lexer=self.lexer, debug=debuglevel)
-
- #---------------------------------------------------------------------------
-
- def _create_factor_rule(self, t):
- """
- Simple helper method for creating factor node objects based on node name.
- """
- if (t[0] == 'IPV4'):
- return IPV4Rule(t[1])
- elif (t[0] == 'IPV6'):
- return IPV6Rule(t[1])
- elif (t[0] == 'INTEGER'):
- return IntegerRule(t[1])
- elif (t[0] == 'FLOAT'):
- return FloatRule(t[1])
- elif (t[0] == 'VARIABLE'):
- return VariableRule(t[1])
- else:
- return ConstantRule(t[1])
-
- def p_expression(self, t):
- """expression : xor_expression OP_OR expression
- | xor_expression"""
- if (len(t) == 4):
- t[0] = LogicalBinOpRule(t[2], t[1], t[3])
- else:
- t[0] = t[1]
-
- def p_xor_expression(self, t):
- """xor_expression : and_expression OP_XOR xor_expression
- | and_expression"""
- if (len(t) == 4):
- t[0] = LogicalBinOpRule(t[2], t[1], t[3])
- else:
- t[0] = t[1]
-
- def p_and_expression(self, t):
- """and_expression : or_p_expression OP_AND and_expression
- | or_p_expression"""
- if (len(t) == 4):
- t[0] = LogicalBinOpRule(t[2], t[1], t[3])
- else:
- t[0] = t[1]
-
- def p_or_p_expression(self, t):
- """or_p_expression : xor_p_expression OP_OR_P or_p_expression
- | xor_p_expression"""
- if (len(t) == 4):
- t[0] = LogicalBinOpRule(t[2], t[1], t[3])
- else:
- t[0] = t[1]
-
- def p_xor_p_expression(self, t):
- """xor_p_expression : and_p_expression OP_XOR_P xor_p_expression
- | and_p_expression"""
- if (len(t) == 4):
- t[0] = LogicalBinOpRule(t[2], t[1], t[3])
- else:
- t[0] = t[1]
-
- def p_and_p_expression(self, t):
- """and_p_expression : not_expression OP_AND_P and_p_expression
- | not_expression"""
- if (len(t) == 4):
- t[0] = LogicalBinOpRule(t[2], t[1], t[3])
- else:
- t[0] = t[1]
-
- def p_not_expression(self, t):
- """not_expression : OP_NOT ex_expression
- | ex_expression"""
- if (len(t) == 3):
- t[0] = UnaryOperationRule(t[1], t[2])
- else:
- t[0] = t[1]
-
- def p_ex_expression(self, t):
- """ex_expression : OP_EXISTS cmp_expression
- | cmp_expression"""
- if (len(t) == 3):
- t[0] = UnaryOperationRule(t[1], t[2])
- else:
- t[0] = t[1]
-
- def p_cmp_expression(self, t):
- """cmp_expression : term OP_LIKE cmp_expression
- | term OP_IN cmp_expression
- | term OP_IS cmp_expression
- | term OP_EQ cmp_expression
- | term OP_NE cmp_expression
- | term OP_GT cmp_expression
- | term OP_GE cmp_expression
- | term OP_LT cmp_expression
- | term OP_LE cmp_expression
- | term"""
- if (len(t) == 4):
- t[0] = ComparisonBinOpRule(t[2], t[1], t[3])
- else:
- t[0] = t[1]
-
- def p_term(self, t):
- """term : factor OP_PLUS term
- | factor OP_MINUS term
- | factor OP_TIMES term
- | factor OP_DIVIDE term
- | factor OP_MODULO term
- | factor"""
- if (len(t) == 4):
- t[0] = MathBinOpRule(t[2], t[1], t[3])
- else:
- t[0] = t[1]
-
- def p_factor(self, t):
- """factor : IPV4
- | IPV6
- | INTEGER
- | FLOAT
- | VARIABLE
- | CONSTANT
- | LBRACK list RBRACK
- | LPAREN expression RPAREN"""
- if (len(t) == 2):
- t[0] = self._create_factor_rule(t[1])
- else:
- t[0] = t[2]
-
- def p_list(self, t):
- """list : IPV4
- | IPV6
- | INTEGER
- | FLOAT
- | VARIABLE
- | CONSTANT
- | IPV4 COMMA list
- | IPV6 COMMA list
- | INTEGER COMMA list
- | FLOAT COMMA list
- | VARIABLE COMMA list
- | CONSTANT COMMA list"""
- n = self._create_factor_rule(t[1])
- if (len(t) == 2):
- t[0] = ListRule(n)
- else:
- t[0] = ListRule(n, t[3])
-
- def p_error(self, t):
- print("Syntax error at '%s'" % t.value)
-
-if __name__ == "__main__":
- """
- Perform the demonstration.
- """
- import pprint
-
- data = "1 and 1 or 1 xor 1"
-
- # Build the parser and try it out
- m = MentatFilterParser()
- m.build()
-
- print("Parsing: {}".format(data))
- pprint.pprint(m.parse(data))
diff --git a/lib/pynspect/jpath.py b/lib/pynspect/jpath.py
deleted file mode 100644
index afa6533d05f1b2bbee2887e1afad423970f3f800..0000000000000000000000000000000000000000
--- a/lib/pynspect/jpath.py
+++ /dev/null
@@ -1,474 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-"""
-This module provides tools for parsing **JPaths** and setting or retrieving values
-on given **JPath** within data structures.
-
-*JPath* is simplified version of `JSONPath <http://goessner.net/articles/JsonPath/>`__
-and can be used to addressing nodes within arbitrary data structure composed
-of dict-like and list-like objects. Basically it can be used for any data
-structure of objects implementing Python 3 list and/or dict interface.
-
-The motivation for implementing this module were following two use cases:
-
-* Enable writing of simple rules in various filtering expressions, for example::
-
- Source.IP4 in [192.168.0.0/24, 192.168.0.0/24]
-
-* Enable simple message modifications based on key => value rules, for example::
-
- "Source[1].Type[*]" = "source type tag"
-
-The obvious first choice for a solution was the `jsonpath-rw <https://pypi.python.org/pypi/jsonpath-rw>`__
-library. The full *JSONPath* however seems to be too big of a gun for our needs and
-in some cases it could even enable users to cut the branch they are sitting on. For
-this reason we have designed this simplified version with only required set of basic features.
-
-*JPath* syntax uses only dot characters ``.`` as node delimiters. Each node name may
-contain only one or more of the following characters::
-
- [a-zA-Z0-9_]+
-
-Node delimiters implicitly work with nested dictionaries and using delimiter
-results in appending new dictionary as a value of given key in parent disctionary.
-Working with lists is enabled by using indices. List indices must be enclosed in
-brackets '[' and ']' and may contain one of the following values:
-
-* ``[int]`` - precise index (negative values not permitted, numbering starts with 1)
-* ``[#]`` - last (because you might not know number of nodes)
-* ``[*]`` - all nodes
-* (index omitted)
-
-When retrieving value(s) at given *JPath*, use of indices will have following effects:
-
-* ``[int]`` - Return node at particular list position (starting with 1)
-* ``[#]`` - Return last node
-* ``[*]`` - Return all nodes (same as omitting)
-* (index omitted) - Return all nodes (same as '*')
-
-When setting value(s) to given *JPath*, use of indices will have following effects:
-
-* ``[int]`` - Set value to particular list node (starting with 1)
-* ``[#]`` - Set value to already existing last node, or append new one to an empty list
-* ``[*]`` - Append new value to a list
-* (index omitted) - This will result in a dictionary key instead of a list
-
-Consider following examples::
-
- >>> msg = {
- 'Format': 'IDEA0',
- 'ID': 'MESSAGE_ID',
- 'DetectTime': 'DETECT TIME',
- 'Category': ['CATEGORY'],
- 'ConnCount': 633,
- 'Description': 'Ping scan',
- 'Source': [
- {
- 'IP4': ['192.168.1.1', '192.168.1.2'],
- 'Proto': ['icmp']
- },
- {
- 'IP4': ['192.168.2.1', '192.168.2.2'],
- 'Proto': ['icmp']
- }
- ],
- 'Target': [
- {
- 'Proto': ['icmp'],
- 'IP4': ['192.168.3.1', '192.168.3.2'],
- 'Anonymised': True
- }
- ],
- 'Node': [
- {
- 'SW' : ['KIPPO'],
- 'Name' : 'NODE_NAME'
- }
- ]
- }
-
- >>> jpath_value(msg, 'Format')
- 'IDEA0'
- >>> jpath_value(msg, 'Category')
- 'CATEGORY'
- >>> jpath_value(msg, 'Node.Name')
- 'NODE_NAME'
- >>> jpath_value(msg, 'Source.IP4')
- '192.168.1.1'
-
- >>> jpath_values(msg, 'Format')
- ['IDEA0']
- >>> jpath_values(msg, 'Category')
- ['CATEGORY']
- >>> jpath_values(msg, 'Node.Name')
- ['NODE_NAME']
- >>> jpath_values(msg, 'Source.IP4')
- ['192.168.1.1', '192.168.1.2', '192.168.2.1', '192.168.2.2']
-
-The current implementation has following known drawbacks:
-
-* Toplevel element must be a dict-like object
-* Nested list-like objects are not possible: ``[[1,2],[3,4]]``
-* It is not possible to set value to multiple elements at once
-* It is not possible to customize type of created structure, only lists and
- dicts are always created
-
-"""
-
-__version__ = "0.3"
-__author__ = "Jan Mach <jan.mach@cesnet.cz>"
-__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-
-
-import re
-import collections
-
-
-#
-# Define global constants.
-#
-
-#: Status code for ``success``, returned by function :py:func:`jpath_set`
-RC_VALUE_SET = 0
-
-#: Status code for ``already-exists``, returned by function :py:func:`jpath_set`
-RC_VALUE_EXISTS = 1
-
-#: Status code for ``not-unique``, returned by function :py:func:`jpath_set`
-RC_VALUE_DUPLICATE = 2
-
-#: Regular expression for single JPath chunk.
-RE_JPATH_CHUNK = re.compile(r"^([a-zA-Z0-9_]+)(\[(#|\*|\d+)\])?$")
-
-#: Internal cache for parsed JPaths.
-_JPATH_CACHE = {}
-
-
-class JPathException(Exception):
- """
- Custom JPath specific exception.
-
- This exception will be thrown on module specific errors.
- """
- def __init__(self, description):
- self._description = description
- def __str__(self):
- return repr(self._description)
-
-def cache_size():
- """
- Return the size of internal JPath cache.
-
- :return: Cache size
- :rtype: int
- """
- return len(_JPATH_CACHE)
-
-def cache_clear():
- """
- Clear internal JPath cache.
- """
- global _JPATH_CACHE
- _JPATH_CACHE = {}
-
-def jpath_parse(jpath):
- """
- Parse given JPath into chunks.
-
- Returns list of dictionaries describing all of the JPath chunks.
-
- :param str jpath: JPath to be parsed into chunks
- :return: JPath chunks as list of dicts
- :rtype: :py:class:`list`
- :raises JPathException: in case of invalid JPath syntax
- """
- result = []
- breadcrumbs = []
-
- # Split JPath into chunks based on '.' character
- chunks = jpath.split('.')
- for ch in chunks:
- match = RE_JPATH_CHUNK.match(ch)
- if match:
- r = {}
-
- # Record whole match
- r['m'] = ch
-
- # Record breadcrumb path
- breadcrumbs.append(ch)
- r['p'] = '.'.join(breadcrumbs)
-
- # Handle node name
- r['n'] = match.group(1)
-
- # Handle node index (optional, may be omitted)
- if match.group(2):
- r['i'] = match.group(3)
- if str(r['i']) == '#':
- r['i'] = -1
- elif str(r['i']) == '*':
- pass
- else:
- r['i'] = int(r['i']) - 1
-
- result.append(r)
- else:
- raise JPathException("Invalid JPath chunk '{}'".format(ch))
- return result
-
-def jpath_parse_c(jpath):
- """
- Caching variant of :py:func:`jpath_parse` function. Same arguments and return
- value.
-
- For performance reasons thee is no copying and all returned values are
- references to internal cache. Treat the returned values as read only, or
- suffer the consequences.
- """
- if not jpath in _JPATH_CACHE:
- _JPATH_CACHE[jpath] = jpath_parse(jpath)
- return _JPATH_CACHE[jpath]
-
-def jpath_values(structure, jpath):
- """
- Return all values at given JPath within given data structure.
-
- For performance reasons this method is intentionally not written as
- recursive.
-
- :param str structure: data structure to be searched
- :param str jpath: JPath to be evaluated
- :return: found values as a list
- :rtype: :py:class:`list`
- """
- # Current working node set
- nodes_a = [structure]
-
- # Next iteration working node set
- nodes_b = []
-
- # Process sequentially all JPath chunks
- chunks = jpath_parse_c(jpath)
- for ch in chunks:
- # Process all currently active nodes
- for node in nodes_a:
- key = ch['n']
- if not isinstance(node, dict) and not isinstance(node, collections.Mapping):
- continue
-
- # Process indexed nodes
- if 'i' in ch:
- idx = ch['i']
- # Skip the node, if the key does not exist, the value is not
- # a list-like object or the list is empty
- if not key in node or not (isinstance(node[key], list) or isinstance(node[key], collections.MutableSequence)) or not len(node[key]):
- continue
- try:
- # Handle '*' special index - append all nodes
- if str(idx) == '*':
- for i in node[key]:
- nodes_b.append(i)
- # Append only node at particular index
- else:
- nodes_b.append(node[key][idx])
- except:
- pass
-
- # Process unindexed nodes
- else:
- # Skip the node, if the key does not exist
- if not key in node:
- continue
-
- # Handle list values - expand them
- if isinstance(node[key], list) or isinstance(node[key], collections.MutableSequence):
- for i in node[key]:
- nodes_b.append(i)
- # Handle scalar values
- else:
- nodes_b.append(node[key])
-
- nodes_a = nodes_b
- nodes_b = []
-
- return nodes_a
-
-def jpath_value(structure, jpath):
- """
- Return single value or first value from list at given JPath within
- given data structure.
-
- This method returns None for non-existent JPaths.
-
- :param str structure: data structure to be searched
- :param str jpath: JPath to be evaluated
- :return: None or found value
- """
- values = jpath_values(structure, jpath)
- if values:
- return values[0]
- return None
-
-def jpath_exists(structure, jpath):
- """
- Check if node at given JPath within given data structure does exist.
-
- :param str structure: data structure to be searched
- :param str jpath: JPath to be evaluated
- :return: True or False
- :rtype: bool
- """
- result = jpath_value(structure, jpath)
- if not result is None:
- return True
- return False
-
-def jpath_set(structure, jpath, value, overwrite = True, unique = False):
- """
- Set given JPath to given value within given structure.
-
- For performance reasons this method is intentionally not written as
- recursive.
-
- :param str structure: data structure to be searched
- :param str jpath: JPath to be evaluated
- :param any value: value of any type to be set at given path
- :param bool overwrite: enable/disable overwriting of already existing value
- :param bool unique: ensure uniqueness of value, works only for lists
- :return: numerical return code, one of the (:py:data:`RC_VALUE_SET`,
- :py:data:`RC_VALUE_EXISTS`, :py:data:`RC_VALUE_DUPLICATE`)
- :rtype: int
- """
- chunks = jpath_parse_c(jpath)
- size = len(chunks) - 1
- current = structure
-
- # Process chunks in order, enumeration is used for detection of the last JPath chunk.
- for i, ch in enumerate(chunks):
- key = ch['n']
-
- if not isinstance(current, dict) and not isinstance(current, collections.Mapping):
- raise JPathException("Expected dict-like structure to attach node '{}'".format(ch['p']))
-
- # Process indexed nodes
- if 'i' in ch:
- idx = ch['i']
-
- # Automatically create nodes for non-existent keys
- if not key in current:
- current[key] = []
- if not isinstance(current[key], list) and not isinstance(current[key], collections.MutableSequence):
- raise JPathException("Expected list-like object under structure key '{}'".format(key))
-
- # Detection of the last JPath chunk - node somewhere in the middle
- if i != size:
- # Attempt to access node at given index
- try:
- current = current[key][idx]
- # IndexError: list index out of range
- # Node at given index does not exist, append new one. Using insert()
- # does not work, item is appended to the end of the list anyway.
- # TypeError: list indices must be integers or slices, not str
- # In the case list index was '*', we are appending to the end of
- # list.
- except (IndexError, TypeError):
- current[key].append({})
- current = current[key][-1]
-
- # Detection of the last JPath chunk - node at the end
- else:
- # Attempt to insert value at given index
- try:
- if overwrite or not current[key][idx]:
- current[key][idx] = value
- else:
- return RC_VALUE_EXISTS
- # IndexError: list index out of range
- # Node at given index does not exist, append new one. Using insert()
- # does not work, item is appended to the end of the list anyway.
- # TypeError: list indices must be integers or slices, not str
- # In the case list index was '*', we are appending to the end of
- # list.
- except (IndexError, TypeError):
- # At this point only deal with unique, overwrite does not make
- # sense, because we would not be here otherwise.
- if not unique or not value in current[key]:
- current[key].append(value)
- else:
- return RC_VALUE_DUPLICATE
-
- # Process unindexed nodes
- else:
- # Detection of the last JPath chunk - node somewhere in the middle
- if i != size:
- # Automatically create nodes for non-existent keys
- if not key in current:
- current[key] = {}
- if not isinstance(current[key], dict) and not isinstance(current[key], collections.Mapping):
- raise JPathException("Expected dict-like object under structure key '{}'".format(key))
-
- current = current[key]
-
- # Detection of the last JPath chunk - node at the end
- else:
- if overwrite or not key in current:
- current[key] = value
- else:
- return RC_VALUE_EXISTS
- return RC_VALUE_SET
-
-if __name__ == "__main__":
- """
- Perform the demonstration.
- """
- import pprint
-
- print("Path parsing:")
- pprint.pprint(jpath_parse("Test"))
- pprint.pprint(jpath_parse("Test.Path"))
- pprint.pprint(jpath_parse("Long.Test.Path"))
- pprint.pprint(jpath_parse("Long[1].Test.Path"))
- pprint.pprint(jpath_parse("Long.Test[2].Path"))
- pprint.pprint(jpath_parse("Long.Test.Path[3]"))
- pprint.pprint(jpath_parse("Long[*].Test.Path"))
- pprint.pprint(jpath_parse("Long.Test[*].Path"))
- pprint.pprint(jpath_parse("Long.Test.Path[*]"))
- pprint.pprint(jpath_parse("Long[#].Test.Path"))
- pprint.pprint(jpath_parse("Long.Test[#].Path"))
- pprint.pprint(jpath_parse("Long.Test.Path[#]"))
-
- print("Path fetching:")
- msg = {
- 'TestA': { 'ValueA1': 'A1', 'ValueA2': 'A2' },
- 'TestB': { 'ValueB1': 'B1', 'ValueB2': 'B2' },
- 'TestC': { 'ValueC1': 'C1', 'ValueC2': 'C2' },
- 'TestD': { 'ValueD1': ['D11','D12'], 'ValueC2': 'C2' }
- }
- pprint.pprint(jpath_values(msg, 'TestD.ValueD1'))
- pprint.pprint(jpath_values(msg, 'TestD.ValueD1[1]'))
- pprint.pprint(jpath_values(msg, 'TestD.ValueD1[2]'))
- pprint.pprint(jpath_values(msg, 'TestD.ValueD1[#]'))
-
- print("Path seting:")
- msg = {
- 'TestA': { 'ValueA1': 'A1', 'ValueA2': 'A2' },
- 'TestB': { 'ValueB1': 'B1', 'ValueB2': 'B2' },
- 'TestC': { 'ValueC1': 'C1', 'ValueC2': 'C2' },
- 'TestD': { 'ValueD1': ['D11','D12'], 'ValueC2': 'C2' }
- }
- pprint.pprint(jpath_set(msg, 'TestE.ValueE1', "Added value"))
- pprint.pprint(jpath_set(msg, 'TestE.ValueE2[1]', "Added value 2"))
- pprint.pprint(jpath_set(msg, 'TestE.ValueE2[2]', "Added value 3"))
- pprint.pprint(jpath_set(msg, 'TestE.ValueE2[#]', "Added value 4"))
- pprint.pprint(jpath_set(msg, 'TestE.ValueE3[1].Subval1', "Added subvalue 11"))
- pprint.pprint(jpath_set(msg, 'TestE.ValueE3[1].Subval2[1]', "Added subval 21"))
- pprint.pprint(jpath_set(msg, 'TestE.ValueE3[#].Subval2[2]', "Added subval 22"))
- pprint.pprint(msg)
diff --git a/lib/pynspect/lexer.py b/lib/pynspect/lexer.py
deleted file mode 100644
index 52321d05570552410359ceb1d7e49dd8b5582f4f..0000000000000000000000000000000000000000
--- a/lib/pynspect/lexer.py
+++ /dev/null
@@ -1,327 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-"""
-This module contains object encapsulation of `PLY <http://www.dabeaz.com/ply/>`__
-lexical analyzer for filtering and query language grammar used in Mentat
-project.
-
-Currently recognized tokens
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. code-block:: python
-
- # Mathematical operation tokens
- OP_PLUS = '\+'
- OP_MINUS = '-'
- OP_TIMES = '\*'
- OP_DIVIDE = '/'
- OP_MODULO = '%'
-
- # Logical operation tokens
- OP_OR = '(or|OR)'
- OP_XOR = '(xor|XOR)'
- OP_AND = '(and|AND)'
- OP_NOT = '(not|NOT)'
- OP_EXISTS = '(exists|EXISTS|\?)'
-
- # Priority logical operation tokens
- OP_OR_P = '\|\|'
- OP_XOR_P = '\^\^'
- OP_AND_P = '&&'
-
- # Comparison operation tokens
- OP_LIKE = '(like|LIKE|=~)'
- OP_IN = '(in|IN|~~)'
- OP_IS = '(is|IS)'
- OP_EQ = '(eq|EQ|==)'
- OP_NE = '(ne|NE|!=|<>)'
- OP_GT = '(gt|GT|>)'
- OP_GE = '(ge|GE|>=)'
- OP_LT = '(lt|LT|<)'
- OP_LE = '(le|LE|<=)'
-
- # Special tokens
- COMMA = '\s*,\s*|\s*;\s*'
- LPAREN = '\('
- RPAREN = '\)'
- LBRACK = '\['
- RBRACK = '\]'
-
- # Contant and variable tokens
- IPV4 = '\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?:\/\d{1,2}|(?:-|..)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})?'
- IPV6 = '[:a-fA-F0-9]+:[:a-fA-F0-9]*(?:\/\d{1,3}|(?:-|..)[:a-fA-F0-9]+:[:a-fA-F0-9]*)?'
- INTEGER = '\d+'
- FLOAT = '\d+\.\d+'
- CONSTANT = '"([^"]+)"|\'([^\']+)\''
- VARIABLE = '[_a-zA-Z][-_a-zA-Z0-9]*(?:\[(?:\d+|-\d+|\#)\])?(?:\.?[a-zA-Z][-_a-zA-Z0-9]*(?:\[(?:\d+|-\d+|\#)\])?)*'
-
-.. note::
-
- Implementation of this module is very *PLY* specific, please read the
- appropriate `documentation <http://www.dabeaz.com/ply/ply.html#ply_nn3>`__
- to understand it.
-
-.. todo::
-
- Consider following options:
-
- * Support negative integers and floats
- * Recognize RFC timestamps as constant without quotes for better
- time value input
- * Support functions (count, max, min, first, last, time, etc.)
-
-"""
-
-__version__ = "0.3"
-__author__ = "Jan Mach <jan.mach@cesnet.cz>"
-__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-
-import re
-import ply.lex as lex
-
-class MentatFilterLexer():
- """
- Object encapsulation of *PLY* lexical analyzer implementation for
- filtering and query language grammar.
- """
-
- # List of all reserved words.
- reserved = {
- 'or': 'OP_OR',
- 'xor': 'OP_XOR',
- 'and': 'OP_AND',
- 'not': 'OP_NOT',
- 'exists': 'OP_EXISTS',
-
- 'like': 'OP_LIKE',
- 'in': 'OP_IN',
- 'is': 'OP_IS',
- 'eq': 'OP_EQ',
- 'ne': 'OP_NE',
- 'gt': 'OP_GT',
- 'ge': 'OP_GE',
- 'lt': 'OP_LT',
- 'le': 'OP_LE',
-
- 'OR': 'OP_OR',
- 'XOR': 'OP_XOR',
- 'AND': 'OP_AND',
- 'NOT': 'OP_NOT',
- 'EXISTS': 'OP_EXISTS',
-
- 'LIKE': 'OP_LIKE',
- 'IN': 'OP_IN',
- 'IS': 'OP_IS',
- 'EQ': 'OP_EQ',
- 'NE': 'OP_NE',
- 'GT': 'OP_GT',
- 'GE': 'OP_GE',
- 'LT': 'OP_LT',
- 'LE': 'OP_LE',
-
- '||': 'OP_OR_P',
- '^^': 'OP_XOR_P',
- '&&': 'OP_AND_P',
- '!': 'OP_NOT',
- '?': 'OP_EXISTS',
-
- '=~': 'OP_LIKE',
- '~~': 'OP_IN',
- '==': 'OP_EQ',
- '!=': 'OP_NE',
- '<>': 'OP_NE',
- '>': 'OP_GT',
- '>=': 'OP_GE',
- '<': 'OP_LT',
- '<=': 'OP_LE',
-
- '+': 'OP_PLUS',
- '-': 'OP_MINUS',
- '*': 'OP_TIMES',
- '/': 'OP_DIVIDE',
- '%': 'OP_MODULO'
- }
-
- # List of grammar token names.
- tokens = [
- 'EXP_ALL',
-
- 'OP_PLUS',
- 'OP_MINUS',
- 'OP_TIMES',
- 'OP_DIVIDE',
- 'OP_MODULO',
-
- 'OP_OR',
- 'OP_XOR',
- 'OP_AND',
- 'OP_NOT',
- 'OP_EXISTS',
-
- 'OP_OR_P',
- 'OP_XOR_P',
- 'OP_AND_P',
-
- 'OP_LIKE',
- 'OP_IN',
- 'OP_IS',
- 'OP_EQ',
- 'OP_NE',
- 'OP_GT',
- 'OP_GE',
- 'OP_LT',
- 'OP_LE',
-
- 'COMMA',
- 'LPAREN',
- 'RPAREN',
- 'LBRACK',
- 'RBRACK',
-
- 'IPV4',
- 'IPV6',
- 'INTEGER',
- 'FLOAT',
- 'CONSTANT',
- 'VARIABLE'
- ]
-
- # Regular expressions for simple tokens
- t_COMMA = r'\s*,\s*|\s*;\s*'
- t_LPAREN = r'\('
- t_RPAREN = r'\)'
- t_LBRACK = r'\['
- t_RBRACK = r'\]'
-
- # Regular expression for ignored tokens
- t_ignore = ' \t'
-
- def build(self, **kwargs):
- """
- Build/rebuild the lexer object.
-
- (Re)Initialize internal PLY lexer object.
- """
- self.lexer = lex.lex(module=self, **kwargs)
-
- def test(self, data, separator = ''):
- """
- Test the lexer on given input string.
-
-
- """
- self.lexer.input(data)
- result = ''
- while True:
- tok = self.lexer.token()
- if not tok:
- break
- result = '{}{}{}'.format(result, tok, separator)
- return result
-
- #---------------------------------------------------------------------------
-
- # According to the documentation, section 4.3 Specification of tokens
- # (http://www.dabeaz.com/ply/ply.html#ply_nn6), best practice is to
- # reduce the number of required regular expressions. So following
- # is the ugly as hell uber regular expression for unary and binary
- # operators.
- def t_EXP_ALL(self, t):
- r'(-|\+|\*|/|%|like|LIKE|=~|in|IN|~~|is|IS|eq|EQ|==|ne|NE|!=|<>|ge|GE|>=|gt|GT|>|le|LE|<=|lt|LT|<|or|OR|\|\||xor|XOR|\^\^|and|AND|&&|not|NOT|!|exists|EXISTS|\?)'
- t.type = self.reserved.get(t.value)
- t.value = t.type
- return t
-
- def t_IPV4(self, t):
- r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?:\/\d{1,2}|(?:-|..)\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})?'
- t.value = (t.type, t.value)
- return t
-
- def t_IPV6(self, t):
- r'[:a-fA-F0-9]+:[:a-fA-F0-9]*(?:\/\d{1,3}|(?:-|..)[:a-fA-F0-9]+:[:a-fA-F0-9]*)?'
- t.value = (t.type, t.value)
- return t
-
- def t_FLOAT(self, t):
- r'\d+\.\d+'
- t.value = (t.type, float(t.value))
- return t
-
- def t_INTEGER(self, t):
- r'\d+'
- t.value = (t.type, int(t.value))
- return t
-
- def t_VARIABLE(self, t):
- r'[_a-zA-Z][-_a-zA-Z0-9]*(?:\[(?:\d+|-\d+|\#)\])?(?:\.?[a-zA-Z][-_a-zA-Z0-9]*(?:\[(?:\d+|-\d+|\#)\])?)*'
- t.value = (t.type, t.value)
- return t
-
- def t_CONSTANT(self, t):
- r'"([^"]+)"|\'([^\']+)\''
- t.value = (t.type, re.sub('["\']', '', t.value))
- return t
-
- def t_newline(self, t):
- r'\n+'
- t.lexer.lineno += len(t.value)
-
- def t_error(self, t):
- print("Illegal character '%s'" % t.value[0])
- t.lexer.skip(1)
-
- def reset_lineno(self):
- """
- Reset internal line counter.
- """
- self.lexer.lineno = 1
-
- def input(self, text):
- """
- Proxy method for underlying Lexer object interface.
- """
- self.lexer.input(text)
-
- def token(self):
- """
- Proxy method for underlying Lexer object interface.
- """
- g = self.lexer.token()
- return g
-
-if __name__ == "__main__":
- """
- Perform the demonstration by parsing text containing all possible
- tokens.
- """
- data = """
- 1 + 1 - 1 * 1 % 1
- OR 2 or 2 || 2
- XOR 3 xor 3 ^^ 3
- AND 4 and 4 && 4
- NOT 5 not 5 ! 5
- EXISTS 6 exists 4 ? 6
- LIKE 7 like 7 =~ 7
- IN 8 in 8 ~~ 8
- IS 9 is 9
- EQ 10 eq 10 == 10
- NE 11 ne 11 <> 11 != 11
- GT 12 gt 12 > 12
- GE 13 ge 13 >= 13
- LT 14 lt 14 < 14
- LE 15 le 15 <= 15
- (127.0.0.1 eq ::1 eq 2001:afdc::58 eq Source.Node eq "Value 525.89:X><" eq 'Value 525.89:X><')
- [1, 2, 3 , 4]
- """
-
- # Build the lexer and try it out
- m = MentatFilterLexer()
- m.build()
- print(m.test(data, "\n"))
diff --git a/lib/pynspect/rules.py b/lib/pynspect/rules.py
deleted file mode 100644
index 4a9c665202763ab0262b56081c896d6a0111e9be..0000000000000000000000000000000000000000
--- a/lib/pynspect/rules.py
+++ /dev/null
@@ -1,578 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-"""
-This module contains implementation of object representations of filtering
-and query language grammar.
-
-There is a separate class defined for each grammar rule. There are following
-classes representing all possible constant and variable values (tree leaves,
-without child nodes):
-
-* :py:class:`VariableRule`
-* :py:class:`ConstantRule`
-* :py:class:`IPv4Rule`
-* :py:class:`IPv6Rule`
-* :py:class:`IntegerRule`
-* :py:class:`FloatRule`
-* :py:class:`ListRule`
-
-There are following classes representing various binary and unary operations:
-
-* :py:class:`LogicalBinOpRule`
-* :py:class:`ComparisonBinOpRule`
-* :py:class:`MathBinOpRule`
-* :py:class:`UnaryOperationRule`
-
-Desired hierarchical rule tree can be created either programatically, or by
-parsing string rules using :py:mod:`pynspect.gparser`.
-
-Working with rule tree is then done via objects implementing rule tree
-traverser interface:
-
-* :py:class:`RuleTreeTraverser`
-
-The provided :py:class:`RuleTreeTraverser` class contains also implementation of
-all necessary evaluation methods.
-
-There is a simple example implementation of rule tree traverser capable of
-printing rule tree into a formated string:
-
-* :py:class:`PrintingTreeTraverser`
-
-Rule evaluation
-^^^^^^^^^^^^^^^
-
-* Logical operations ``and or xor not exists``
-
- There is no special handling for operands of logical operations. Operand(s) are
- evaluated in logical expression exactly as they are received, there is no
- mangling involved.
-
-* Comparison operations
-
- All comparison operations are designed to work with lists as both operands.
- This is because :py:func:`pynspect.jpath.jpath_values` function is
- used to retrieve variable values and this function always returns list.
-
- * Operation: ``is``
-
- Like in the case of logical operations, there is no mangling involved when
- evaluating this operation. Both operands are compared using Python`s native
- ``is`` operation and result is returned.
-
- * Operation: ``in``
-
- In this case left operand is iterated and each value is compared using Python`s
- native ``in`` operation with right operand. First ``True`` result wins and
- operation immediatelly returns ``True``, ``False`` is returned otherwise.
-
- * Any other operation: ``like eq ne gt ge lt le``
-
- In case of this operation both of the operands are iterated and each one is
- compared with each other. First ``True`` result wins and operation immediatelly
- returns ``True``, ``False`` is returned otherwise.
-
- * Math operations: ``+ - * / %``
-
- Current math operation implementation supports following options:
-
- * Both operands are lists of the same length. In this case corresponding
- elements at certain position within the list are evaluated with given
- operation. Result is a list.
-
- * One of the operands is a list, second is scalar value or list of the
- size 1. In this case given operation is evaluated with each element of
- the longer list. Result is a list.
-
- * Operands are lists of the different size. This option is **forbidden**
- and the result is ``None``.
-
-"""
-
-__version__ = "0.3"
-__author__ = "Jan Mach <jan.mach@cesnet.cz>"
-__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-
-import collections
-import re
-import datetime
-
-class FilteringRuleException(Exception):
- """
- Custom filtering rule specific exception.
-
- This exception will be thrown on module specific errors.
- """
- def __init__(self, description):
- self._description = description
- def __str__(self):
- return repr(self._description)
-
-class Rule():
- """
- Base class for all filter tree rules.
- """
- pass
-
-class ValueRule(Rule):
- """
- Base class for all filter tree value rules.
- """
- pass
-
-class VariableRule(ValueRule):
- """
- Class for expression variables.
- """
- def __init__(self, value):
- """
- Initialize the variable with given path value.
- """
- self.value = value
-
- def __str__(self):
- return "{}".format(self.value)
-
- def __repr__(self):
- return "VARIABLE({})".format(repr(self.value))
-
- def traverse(self, traverser, **kwargs):
- return traverser.variable(self, **kwargs)
-
-class ConstantRule(ValueRule):
- """
- Class for all expression constant values.
- """
- def __init__(self, value):
- """
- Initialize the constant with given value.
- """
- self.value = value
-
- def __str__(self):
- return '"{}"'.format(self.value)
-
- def __repr__(self):
- return "CONSTANT({})".format(repr(self.value))
-
- def traverse(self, traverser, **kwargs):
- return traverser.constant(self, **kwargs)
-
-class IPV4Rule(ConstantRule):
- """
- Class for IPv4 address constants.
- """
- def __str__(self):
- return '{}'.format(self.value)
-
- def __repr__(self):
- return "IPV4({})".format(repr(self.value))
-
- def traverse(self, traverser, **kwargs):
- return traverser.ipv4(self, **kwargs)
-
-class IPV6Rule(ConstantRule):
- """
- Class for IPv6 address constants.
- """
- def __str__(self):
- return '{}'.format(self.value)
-
- def __repr__(self):
- return "IPV6({})".format(repr(self.value))
-
- def traverse(self, traverser, **kwargs):
- return traverser.ipv6(self, **kwargs)
-
-class NumberRule(ConstantRule):
- """
- Base class for all numerical constants.
- """
- pass
-
-class IntegerRule(NumberRule):
- """
- Class for integer constants.
- """
- def __str__(self):
- return '{}'.format(self.value)
-
- def __repr__(self):
- return "INTEGER({})".format(repr(self.value))
-
- def traverse(self, traverser, **kwargs):
- return traverser.integer(self, **kwargs)
-
-class FloatRule(NumberRule):
- """
- Class for float constants.
- """
- def __str__(self):
- return '{}'.format(self.value)
-
- def __repr__(self):
- return "FLOAT({})".format(repr(self.value))
-
- def traverse(self, traverser, **kwargs):
- return traverser.float(self, **kwargs)
-
-class ListRule(ValueRule):
- """
- Base class for all filter tree rules.
- """
- def __init__(self, rule, next_rule = None):
- """
- Initialize the constant with given value.
- """
- if not isinstance(rule, list):
- rule = [rule]
- self.value = rule
- if next_rule:
- self.value += next_rule.value
-
- def __str__(self):
- return '[{}]'.format(', '.join([str(v) for v in self.value]))
-
- def __repr__(self):
- return "LIST({})".format(', '.join([repr(v) for v in self.value]))
-
- def traverse(self, traverser, **kwargs):
- return traverser.list(self, **kwargs)
-
-class OperationRule(Rule):
- """
- Base class for all expression operations (both unary and binary).
- """
- pass
-
-class BinaryOperationRule(OperationRule):
- """
- Base class for all binary operations.
- """
- def __init__(self, operation, left, right):
- """
- Initialize the object with operation type and both operands.
- """
- self.operation = operation
- self.left = left
- self.right = right
-
- def __str__(self):
- return "({} {} {})".format(str(self.left), str(self.operation), str(self.right))
-
-class LogicalBinOpRule(BinaryOperationRule):
- """
- Base class for all logical binary operations.
- """
- def __repr__(self):
- return "LOGBINOP({} {} {})".format(repr(self.left), str(self.operation), repr(self.right))
-
- def traverse(self, traverser, **kwargs):
- lr = self.left.traverse(traverser, **kwargs)
- rr = self.right.traverse(traverser, **kwargs)
- return traverser.binary_operation_logical(self, lr, rr, **kwargs)
-
-class ComparisonBinOpRule(BinaryOperationRule):
- """
- Base class for all comparison binary operations.
- """
- def __repr__(self):
- return "COMPBINOP({} {} {})".format(repr(self.left), str(self.operation), repr(self.right))
-
- def traverse(self, traverser, **kwargs):
- lr = self.left.traverse(traverser, **kwargs)
- rr = self.right.traverse(traverser, **kwargs)
- return traverser.binary_operation_comparison(self, lr, rr, **kwargs)
-
-class MathBinOpRule(BinaryOperationRule):
- """
- Base class for all mathematical binary operations.
- """
- def __repr__(self):
- return "MATHBINOP({} {} {})".format(repr(self.left), str(self.operation), repr(self.right))
-
- def traverse(self, traverser, **kwargs):
- lr = self.left.traverse(traverser, **kwargs)
- rr = self.right.traverse(traverser, **kwargs)
- return traverser.binary_operation_math(self, lr, rr, **kwargs)
-
-class UnaryOperationRule(OperationRule):
- """
- Base class for all unary operations.
- """
- def __init__(self, operation, right):
- """
- Initialize the object with operation type operand.
- """
- self.operation = operation
- self.right = right
-
- def __str__(self):
- return "({} {})".format(str(self.operation), str(self.right))
-
- def __repr__(self):
- return "UNOP({} {})".format(str(self.operation), repr(self.right))
-
- def traverse(self, traverser, **kwargs):
- rr = self.right.traverse(traverser, **kwargs)
- return traverser.unary_operation(self, rr, **kwargs)
-
-def _to_numeric(val):
- """
- Helper function for conversion of various data types into numeric representation.
- """
- if isinstance(val, int) or isinstance(val, float):
- return val
- if isinstance(val, datetime.datetime):
- return val.timestamp()
- return float(val)
-
-class RuleTreeTraverser():
- """
- Base class for all rule tree traversers.
- """
-
- """
- Definitions of all logical binary operations.
- """
- binops_logical = {
- 'OP_OR': lambda x, y : x or y,
- 'OP_XOR': lambda x, y : (x and not y) or (not x and y),
- 'OP_AND': lambda x, y : x and y,
- 'OP_OR_P': lambda x, y : x or y,
- 'OP_XOR_P': lambda x, y : (x and not y) or (not x and y),
- 'OP_AND_P': lambda x, y : x and y,
- }
-
- """
- Definitions of all comparison binary operations.
- """
- binops_comparison = {
- 'OP_LIKE': lambda x, y : re.search(y, x),
- 'OP_IN': lambda x, y : x in y,
- 'OP_IS': lambda x, y : x == y,
- 'OP_EQ': lambda x, y : x == y,
- 'OP_NE': lambda x, y : x != y,
- 'OP_GT': lambda x, y : x > y,
- 'OP_GE': lambda x, y : x >= y,
- 'OP_LT': lambda x, y : x < y,
- 'OP_LE': lambda x, y : x <= y,
- }
-
- """
- Definitions of all mathematical binary operations.
- """
- binops_math = {
- 'OP_PLUS': lambda x, y : x + y,
- 'OP_MINUS': lambda x, y : x - y,
- 'OP_TIMES': lambda x, y : x * y,
- 'OP_DIVIDE': lambda x, y : x / y,
- 'OP_MODULO': lambda x, y : x % y,
- }
-
- """
- Definitions of all unary operations.
- """
- unops = {
- 'OP_NOT': lambda x : not x,
- 'OP_EXISTS': lambda x : x,
- }
-
- def evaluate_binop_logical(self, operation, left, right, **kwargs):
- """
- Evaluate given logical binary operation with given operands.
- """
- if not operation in self.binops_logical:
- raise Exception("Invalid logical binary operation '{}'".format(operation))
- result = self.binops_logical[operation](left, right)
- if result:
- return True
- else:
- return False
-
- def evaluate_binop_comparison(self, operation, left, right, **kwargs):
- """
- Evaluate given comparison binary operation with given operands.
- """
- if not operation in self.binops_comparison:
- raise Exception("Invalid comparison binary operation '{}'".format(operation))
- if left is None or right is None:
- return None
- if not isinstance(left, list):
- left = [left]
- if not isinstance(right, list):
- right = [right]
- if not len(left) or not len(right):
- return None
- if operation in ['OP_IS']:
- res = self.binops_comparison[operation](left, right)
- if res:
- return True
- elif operation in ['OP_IN']:
- for l in left:
- res = self.binops_comparison[operation](l, right)
- if res:
- return True
- else:
- for l in left:
- if l is None:
- continue
- for r in right:
- if r is None:
- continue
- res = self.binops_comparison[operation](l, r)
- if res:
- return True
- return False
-
- def _calculate_vector(self, operation, left, right):
- """
-
- """
- result = []
- if len(right) == 1:
- right = _to_numeric(right[0])
- for l in left:
- l = _to_numeric(l)
- result.append(self.binops_math[operation](l, right))
- elif len(left) == 1:
- left = _to_numeric(left[0])
- for r in right:
- r = _to_numeric(r)
- result.append(self.binops_math[operation](left, r))
- elif len(left) == len(right):
- for l, r in zip(left, right):
- l = _to_numeric(l)
- r = _to_numeric(r)
- result.append(self.binops_math[operation](l, r))
- else:
- raise FilteringRuleException("Uneven lenth of math operation '{}' operands".format(operation))
- return result
-
- def evaluate_binop_math(self, operation, left, right, **kwargs):
- """
- Evaluate given mathematical binary operation with given operands.
- """
- if not operation in self.binops_math:
- raise Exception("Invalid math binary operation '{}'".format(operation))
- if left is None or right is None:
- return None
- if not isinstance(left, list):
- left = [left]
- if not isinstance(right, list):
- right = [right]
- if not len(left) or not len(right):
- return None
- try:
- v = self._calculate_vector(operation, left, right)
- if len(v) > 1:
- return v
- else:
- return v[0]
- except:
- return None
-
- def evaluate_unop(self, operation, right, **kwargs):
- """
- Evaluate given unary operation with given operand.
- """
- if not operation in self.unops:
- raise Exception("Invalid unary operation '{}'".format(operation))
- if right is None:
- return None
- return self.unops[operation](right)
-
- def evaluate(self, operation, *args):
- """
- Master method for evaluating any operation (both unary and binary).
- """
- if operation in self.binops_comparison:
- return self.evaluate_binop_comparison(rule, *args)
- if operation in self.binops_logical:
- return self.evaluate_binop_logical(rule, *args)
- if operation in self.binops_math:
- return self.evaluate_binop_math(rule, *args)
- if operation in self.unops:
- return self.evaluate_unop(rule, *args)
- raise Exception("Invalid operation '{}'".format(operation))
-
-class PrintingTreeTraverser(RuleTreeTraverser):
- """
- Demonstation of simple rule tree traverser - printing traverser.
- """
- def ipv4(self, rule, **kwargs):
- return "IPV4({})".format(rule.value)
- def ipv6(self, rule, **kwargs):
- return "IPV6({})".format(rule.value)
- def integer(self, rule, **kwargs):
- return "INTEGER({})".format(rule.value)
- def float(self, rule, **kwargs):
- return "FLOAT({})".format(rule.value)
- def constant(self, rule, **kwargs):
- return "CONSTANT({})".format(rule.value)
- def variable(self, rule, **kwargs):
- return "VARIABLE({})".format(rule.value)
- def list(self, rule, **kwargs):
- return "LIST({})".format(', '.join([str(v) for v in rule.value]))
- def binary_operation_logical(self, rule, left, right, **kwargs):
- return "LOGBINOP({};{};{})".format(rule.operation, left, right)
- def binary_operation_comparison(self, rule, left, right, **kwargs):
- return "COMPBINOP({};{};{})".format(rule.operation, left, right)
- def binary_operation_math(self, rule, left, right, **kwargs):
- return "MATHBINOP({};{};{})".format(rule.operation, left, right)
- def unary_operation(self, rule, right, **kwargs):
- return "UNOP({};{})".format(rule.operation, right)
-
-if __name__ == "__main__":
- """
- Perform the demonstration.
- """
- print("* Rule usage:")
- rule_var = VariableRule("Test")
- print("STR: {}".format(str(rule_var)))
- print("REPR: {}".format(repr(rule_var)))
- rule_const = ConstantRule("constant")
- print("STR: {}".format(str(rule_const)))
- print("REPR: {}".format(repr(rule_const)))
- rule_ipv4 = IPV4Rule("127.0.0.1")
- print("STR: {}".format(str(rule_ipv4)))
- print("REPR: {}".format(repr(rule_ipv4)))
- rule_ipv6 = IPV6Rule("::1")
- print("STR: {}".format(str(rule_ipv6)))
- print("REPR: {}".format(repr(rule_ipv6)))
- rule_integer = IntegerRule(15)
- print("STR: {}".format(str(rule_integer)))
- print("REPR: {}".format(repr(rule_integer)))
- rule_float = FloatRule(15.5)
- print("STR: {}".format(str(rule_float)))
- print("REPR: {}".format(repr(rule_float)))
- rule_binop_l = LogicalBinOpRule('OP_OR', rule_var, rule_integer)
- print("STR: {}".format(str(rule_binop_l)))
- print("REPR: {}".format(repr(rule_binop_l)))
- rule_binop_c = ComparisonBinOpRule('OP_GT', rule_var, rule_integer)
- print("STR: {}".format(str(rule_binop_c)))
- print("REPR: {}".format(repr(rule_binop_c)))
- rule_binop_m = MathBinOpRule('OP_PLUS', rule_var, rule_integer)
- print("STR: {}".format(str(rule_binop_m)))
- print("REPR: {}".format(repr(rule_binop_m)))
- rule_binop = LogicalBinOpRule('OP_OR', ComparisonBinOpRule('OP_GT', MathBinOpRule('OP_PLUS', VariableRule("Test"), IntegerRule(10)), IntegerRule(20)), ComparisonBinOpRule('OP_LT', VariableRule("Test"), IntegerRule(5)))
- print("STR: {}".format(str(rule_binop)))
- print("REPR: {}".format(repr(rule_binop)))
- rule_unop = UnaryOperationRule('OP_NOT', rule_var)
- print("STR: {}".format(str(rule_unop)))
- print("REPR: {}".format(repr(rule_unop)))
-
- print("\n* Traverser usage:")
- traverser = PrintingTreeTraverser()
- print("{}".format(rule_binop_l.traverse(traverser)))
- print("{}".format(rule_binop_c.traverse(traverser)))
- print("{}".format(rule_binop_m.traverse(traverser)))
- print("{}".format(rule_binop.traverse(traverser)))
- print("{}".format(rule_unop.traverse(traverser)))
diff --git a/lib/pynspect/test_filters.py b/lib/pynspect/test_filters.py
deleted file mode 100644
index 2f6c873e0fb00380608008615b4fec36653cd4ae..0000000000000000000000000000000000000000
--- a/lib/pynspect/test_filters.py
+++ /dev/null
@@ -1,310 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-import os
-import sys
-import shutil
-import unittest
-from pprint import pformat, pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../lib'))
-sys.path.insert(0, lib)
-
-from pynspect.jpath import *
-from pynspect.rules import *
-from pynspect.gparser import MentatFilterParser
-from pynspect.filters import DataObjectFilter
-
-#-------------------------------------------------------------------------------
-# NOTE: Sorry for the long lines in this file. They are deliberate, because the
-# assertion permutations are (IMHO) more readable this way.
-#-------------------------------------------------------------------------------
-
-class TestMentatDataObjectFilter(unittest.TestCase):
- test_msg1 = {
- "ID" : "e214d2d9-359b-443d-993d-3cc5637107a0",
- "WinEndTime" : "2016-06-21 11:25:01Z",
- "ConnCount" : 2,
- "Source" : [
- {
- "IP4" : [
- "188.14.166.39"
- ]
- }
- ],
- "Format" : "IDEA0",
- "WinStartTime" : "2016-06-21 11:20:01Z",
- "_CESNET" : {
- "StorageTime" : 1466508305
- },
- "Target" : [
- {
- "IP4" : [
- "195.113.165.128/25"
- ],
- "Port" : [
- "22"
- ],
- "Proto" : [
- "tcp",
- "ssh"
- ],
- "Anonymised" : True
- }
- ],
- "Note" : "SSH login attempt",
- "DetectTime" : "2016-06-21 13:08:27Z",
- "Node" : [
- {
- "Name" : "cz.cesnet.mentat.warden_filer",
- "Type" : [
- "Relay"
- ]
- },
- {
- "AggrWin" : "00:05:00",
- "Type" : [
- "Connection",
- "Honeypot",
- "Recon"
- ],
- "SW" : [
- "Kippo"
- ],
- "Name" : "cz.uhk.apate.cowrie"
- }
- ],
- "Category" : [
- "Attempt.Login"
- ]
- }
-
- def test_01_basic_logical(self):
- """
- Perform basic filtering tests.
- """
- self.maxDiff = None
-
- flt = DataObjectFilter()
-
- rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(True))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(False))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(True))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(False))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
-
- rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(True))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(False))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(True))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(False))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
-
- rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(True))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(False))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(True))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(False))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
-
- rule = UnaryOperationRule('OP_NOT', ConstantRule(True))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = UnaryOperationRule('OP_NOT', ConstantRule(False))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = UnaryOperationRule('OP_NOT', VariableRule("Target.Anonymised"))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
-
- def test_02_basic_comparison(self):
- """
- Perform basic filtering tests.
- """
- self.maxDiff = None
-
- flt = DataObjectFilter()
- psr = MentatFilterParser()
- psr.build()
-
- rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
-
- rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("e214d2d9"))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("xxxxxxxx"))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Spam"))))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Attempt.Login")))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(4))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(4))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(3))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(3))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(3))
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
-
- rule = psr.parse('ID == "e214d2d9-359b-443d-993d-3cc5637107a0"')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ID eq "e214d2d9-359b-443d-993d-3cc5637107"')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ID != "e214d2d9-359b-443d-993d-3cc5637107a0"')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ID ne "e214d2d9-359b-443d-993d-3cc5637107"')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
-
- rule = psr.parse('ID like "e214d2d9"')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ID LIKE "xxxxxxxx"')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('Category in ["Phishing" , "Attempt.Login"]')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('Category IN ["Phishing" , "Spam"]')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('Category is ["Attempt.Login"]')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('Category IS ["Phishing" , "Attempt.Login"]')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ConnCount == 2')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ConnCount eq 4')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ConnCount != 2')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ConnCount ne 4')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ConnCount > 2')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ConnCount gt 1')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ConnCount >= 2')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ConnCount ge 1')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ConnCount GE 3')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ConnCount < 2')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ConnCount lt 3')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ConnCount <= 2')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ConnCount le 3')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('ConnCount LE 1')
- self.assertEqual(flt.filter(rule, self.test_msg1), False)
- rule = psr.parse('ConnCounts LE 1')
- self.assertEqual(flt.filter(rule, self.test_msg1), None)
-
- def test_03_basic_math(self):
- """
- Perform basic math tests.
- """
- self.maxDiff = None
-
- flt = DataObjectFilter()
- psr = MentatFilterParser()
- psr.build()
-
- rule = MathBinOpRule('OP_PLUS', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, self.test_msg1), 3)
- rule = MathBinOpRule('OP_MINUS', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, self.test_msg1), 1)
- rule = MathBinOpRule('OP_TIMES', VariableRule("ConnCount"), IntegerRule(5))
- self.assertEqual(flt.filter(rule, self.test_msg1), 10)
- rule = MathBinOpRule('OP_DIVIDE', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, self.test_msg1), 1)
- rule = MathBinOpRule('OP_MODULO', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, self.test_msg1), 0)
-
- rule = psr.parse('ConnCount + 1')
- self.assertEqual(flt.filter(rule, self.test_msg1), 3)
- rule = psr.parse('ConnCount - 1')
- self.assertEqual(flt.filter(rule, self.test_msg1), 1)
- rule = psr.parse('ConnCount * 5')
- self.assertEqual(flt.filter(rule, self.test_msg1), 10)
- rule = psr.parse('ConnCount / 2')
- self.assertEqual(flt.filter(rule, self.test_msg1), 1)
- rule = psr.parse('ConnCount % 2')
- self.assertEqual(flt.filter(rule, self.test_msg1), 0)
-
- def test_04_advanced_filters(self):
- """
- Perform advanced filtering tests.
- """
- self.maxDiff = None
-
- flt = DataObjectFilter()
- psr = MentatFilterParser()
- psr.build()
-
- rule = psr.parse('(ConnCount + 10) > 11')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('((ConnCount + 3) < 5) or ((ConnCount + 10) > 11)')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
- rule = psr.parse('1')
- self.assertEqual(flt.filter(rule, self.test_msg1), True)
-
- def test_05_non_existent_nodes(self):
- """
- Perform advanced filtering tests.
- """
- self.maxDiff = None
-
- flt = DataObjectFilter()
- psr = MentatFilterParser()
- psr.build()
-
- rule = psr.parse('(ConnCounts + 10) > 11')
- self.assertEqual(flt.filter(rule, self.test_msg1), None)
- rule = psr.parse('ConnCount > ConnCounts')
- self.assertEqual(flt.filter(rule, self.test_msg1), None)
- rule = psr.parse('DetectTime < InspectionTime')
- self.assertEqual(flt.filter(rule, self.test_msg1), None)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/lib/pynspect/test_filters_idea.py b/lib/pynspect/test_filters_idea.py
deleted file mode 100644
index 08ee7d06df974161aeade2e4b59fd46f95f569b6..0000000000000000000000000000000000000000
--- a/lib/pynspect/test_filters_idea.py
+++ /dev/null
@@ -1,386 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-import os
-import sys
-import shutil
-import unittest
-from pprint import pformat, pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../lib'))
-sys.path.insert(0, lib)
-
-from idea import lite
-from pynspect.jpath import *
-from pynspect.rules import *
-from pynspect.gparser import MentatFilterParser
-from pynspect.filters import DataObjectFilter, IDEAFilterCompiler, clean_variable
-
-#-------------------------------------------------------------------------------
-# NOTE: Sorry for the long lines in this file. They are deliberate, because the
-# assertion permutations are (IMHO) more readable this way.
-#-------------------------------------------------------------------------------
-
-class TestMentatDataObjectFilterIDEA(unittest.TestCase):
- test_msg1 = {
- "ID" : "e214d2d9-359b-443d-993d-3cc5637107a0",
- "WinEndTime" : "2016-06-21 11:25:01Z",
- "ConnCount" : 2,
- "Source" : [
- {
- "IP4" : [
- "188.14.166.39"
- ]
- }
- ],
- "Format" : "IDEA0",
- "WinStartTime" : "2016-06-21 11:20:01Z",
- "_CESNET" : {
- "StorageTime" : 1466508305
- },
- "Target" : [
- {
- "IP4" : [
- "195.113.165.128/25"
- ],
- "Port" : [
- "22"
- ],
- "Proto" : [
- "tcp",
- "ssh"
- ],
- "Anonymised" : True
- }
- ],
- "Note" : "SSH login attempt",
- "DetectTime" : "2016-06-21 13:08:27Z",
- "Node" : [
- {
- "Name" : "cz.cesnet.mentat.warden_filer",
- "Type" : [
- "Relay"
- ]
- },
- {
- "AggrWin" : "00:05:00",
- "Type" : [
- "Connection",
- "Honeypot",
- "Recon"
- ],
- "SW" : [
- "Kippo"
- ],
- "Name" : "cz.uhk.apate.cowrie"
- }
- ],
- "Category" : [
- "Attempt.Login"
- ]
- }
-
- def test_01_basic_logical(self):
- """
- Perform basic filtering tests.
- """
- self.maxDiff = None
-
- msg_idea = lite.Idea(self.test_msg1)
- flt = DataObjectFilter()
-
- rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(True))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(False))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(True))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(False))
- self.assertEqual(flt.filter(rule, msg_idea), False)
-
- rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(True))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(False))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(True))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(False))
- self.assertEqual(flt.filter(rule, msg_idea), False)
-
- rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(True))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(False))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(True))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(False))
- self.assertEqual(flt.filter(rule, msg_idea), False)
-
- rule = UnaryOperationRule('OP_NOT', ConstantRule(True))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = UnaryOperationRule('OP_NOT', ConstantRule(False))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = UnaryOperationRule('OP_NOT', VariableRule("Target.Anonymised"))
- self.assertEqual(flt.filter(rule, msg_idea), False)
-
- def test_02_basic_comparison(self):
- """
- Perform basic filtering tests.
- """
- self.maxDiff = None
-
- msg_idea = lite.Idea(self.test_msg1)
- flt = DataObjectFilter()
- psr = MentatFilterParser()
- psr.build()
-
- rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
- self.assertEqual(flt.filter(rule, msg_idea), True)
-
- rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("e214d2d9"))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("xxxxxxxx"))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Spam"))))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Attempt.Login")))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(4))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(4))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(3))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(3))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(3))
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, msg_idea), False)
-
- rule = psr.parse('ID == "e214d2d9-359b-443d-993d-3cc5637107a0"')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ID eq "e214d2d9-359b-443d-993d-3cc5637107"')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('ID != "e214d2d9-359b-443d-993d-3cc5637107a0"')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('ID ne "e214d2d9-359b-443d-993d-3cc5637107"')
- self.assertEqual(flt.filter(rule, msg_idea), True)
-
- rule = psr.parse('ID like "e214d2d9"')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ID LIKE "xxxxxxxx"')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('Category in ["Phishing" , "Attempt.Login"]')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('Category IN ["Phishing" , "Spam"]')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('Category is ["Attempt.Login"]')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('Category IS ["Phishing" , "Attempt.Login"]')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('ConnCount == 2')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ConnCount eq 4')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('ConnCount != 2')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('ConnCount ne 4')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ConnCount > 2')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('ConnCount gt 1')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ConnCount >= 2')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ConnCount ge 1')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ConnCount GE 3')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('ConnCount < 2')
- self.assertEqual(flt.filter(rule, msg_idea), False)
- rule = psr.parse('ConnCount lt 3')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ConnCount <= 2')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ConnCount le 3')
- self.assertEqual(flt.filter(rule, msg_idea), True)
- rule = psr.parse('ConnCount LE 1')
- self.assertEqual(flt.filter(rule, msg_idea), False)
-
- def test_03_basic_math(self):
- """
- Perform basic math tests.
- """
- self.maxDiff = None
-
- msg_idea = lite.Idea(self.test_msg1)
- flt = DataObjectFilter()
- psr = MentatFilterParser()
- psr.build()
-
- rule = MathBinOpRule('OP_PLUS', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, msg_idea), 3)
- rule = MathBinOpRule('OP_MINUS', VariableRule("ConnCount"), IntegerRule(1))
- self.assertEqual(flt.filter(rule, msg_idea), 1)
- rule = MathBinOpRule('OP_TIMES', VariableRule("ConnCount"), IntegerRule(5))
- self.assertEqual(flt.filter(rule, msg_idea), 10)
- rule = MathBinOpRule('OP_DIVIDE', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, msg_idea), 1)
- rule = MathBinOpRule('OP_MODULO', VariableRule("ConnCount"), IntegerRule(2))
- self.assertEqual(flt.filter(rule, msg_idea), 0)
-
- rule = psr.parse('ConnCount + 1')
- self.assertEqual(flt.filter(rule, msg_idea), 3)
- rule = psr.parse('ConnCount - 1')
- self.assertEqual(flt.filter(rule, msg_idea), 1)
- rule = psr.parse('ConnCount * 5')
- self.assertEqual(flt.filter(rule, msg_idea), 10)
- rule = psr.parse('ConnCount / 2')
- self.assertEqual(flt.filter(rule, msg_idea), 1)
- rule = psr.parse('ConnCount % 2')
- self.assertEqual(flt.filter(rule, msg_idea), 0)
-
- def test_04_basic_compilations(self):
- """
- Perform advanced filtering tests.
- """
- self.maxDiff = None
-
- self.assertEqual(clean_variable('Target.IP4'), 'Target.IP4')
- self.assertEqual(clean_variable('Target[1].IP4'), 'Target.IP4')
- self.assertEqual(clean_variable('Target[1].IP4[22]'), 'Target.IP4')
-
- msg_idea = lite.Idea(self.test_msg1)
- cpl = IDEAFilterCompiler()
- psr = MentatFilterParser()
- psr.build()
-
- rule = psr.parse('(Source.IP4 == "188.14.166.39")')
- self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ CONSTANT('188.14.166.39'))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
-
- rule = psr.parse('(Source.IP4 == 188.14.166.39)')
- self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4('188.14.166.39'))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
-
- rule = psr.parse('5 + 6 - 9')
- self.assertEqual(repr(rule), "MATHBINOP(INTEGER(5) OP_PLUS MATHBINOP(INTEGER(6) OP_MINUS INTEGER(9)))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "INTEGER(2)")
-
- rule = psr.parse('Test + 10 - 9')
- self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('Test') OP_PLUS MATHBINOP(INTEGER(10) OP_MINUS INTEGER(9)))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(1))")
-
- rule = psr.parse('Test + (10 - 9)')
- self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('Test') OP_PLUS MATHBINOP(INTEGER(10) OP_MINUS INTEGER(9)))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(1))")
-
- rule = psr.parse('(Test + 10) - 9')
- self.assertEqual(repr(rule), "MATHBINOP(MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(10)) OP_MINUS INTEGER(9))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "MATHBINOP(MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(10)) OP_MINUS INTEGER(9))")
-
- rule = psr.parse('9 - 6 + Test')
- self.assertEqual(repr(rule), "MATHBINOP(INTEGER(9) OP_MINUS MATHBINOP(INTEGER(6) OP_PLUS VARIABLE('Test')))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "MATHBINOP(INTEGER(9) OP_MINUS MATHBINOP(INTEGER(6) OP_PLUS VARIABLE('Test')))")
-
- rule = psr.parse('9 - (6 + Test)')
- self.assertEqual(repr(rule), "MATHBINOP(INTEGER(9) OP_MINUS MATHBINOP(INTEGER(6) OP_PLUS VARIABLE('Test')))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "MATHBINOP(INTEGER(9) OP_MINUS MATHBINOP(INTEGER(6) OP_PLUS VARIABLE('Test')))")
-
- rule = psr.parse('(9 - 6) + Test')
- self.assertEqual(repr(rule), "MATHBINOP(MATHBINOP(INTEGER(9) OP_MINUS INTEGER(6)) OP_PLUS VARIABLE('Test'))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "MATHBINOP(INTEGER(3) OP_PLUS VARIABLE('Test'))")
-
- def test_05_advanced_filters(self):
- """
- Perform advanced filtering tests.
- """
- self.maxDiff = None
-
- msg_idea = lite.Idea(self.test_msg1)
- flt = DataObjectFilter()
- cpl = IDEAFilterCompiler()
- psr = MentatFilterParser()
- psr.build()
-
- rule = psr.parse('DetectTime + 3600')
- self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('DetectTime') OP_PLUS INTEGER(3600))")
- res = cpl.compile(rule)
- self.assertEqual(repr(res), "MATHBINOP(VARIABLE('DetectTime') OP_PLUS INTEGER(3600))")
- self.assertEqual(flt.filter(rule, msg_idea), 1466510907.0)
-
- rule = psr.parse('(ConnCount + 10) > 11')
- self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11))")
- rule = cpl.compile(rule)
- self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11))")
- self.assertEqual(flt.filter(rule, msg_idea), True)
-
- rule = psr.parse('(ConnCount + 3) < 5')
- self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5))")
- rule = cpl.compile(rule)
- self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5))")
- self.assertEqual(flt.filter(rule, msg_idea), False)
-
- rule = psr.parse('((ConnCount + 3) < 5) or ((ConnCount + 10) > 11)')
- self.assertEqual(repr(rule), "LOGBINOP(COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5)) OP_OR COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11)))")
- rule = cpl.compile(rule)
- self.assertEqual(repr(rule), "LOGBINOP(COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5)) OP_OR COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11)))")
- self.assertEqual(flt.filter(rule, msg_idea), True)
-
- rule = psr.parse('(Source.IP4 == 188.14.166.39)')
- self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4('188.14.166.39'))")
- rule = cpl.compile(rule)
- self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
- self.assertEqual(flt.filter(rule, msg_idea), True)
-
- rule = psr.parse('(Source.IP4 in ["188.14.166.39","188.14.166.40","188.14.166.41"])')
- self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_IN LIST(CONSTANT('188.14.166.39'), CONSTANT('188.14.166.40'), CONSTANT('188.14.166.41')))")
- rule = cpl.compile(rule)
- self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_IN LIST(IPV4(IP4('188.14.166.39')), IPV4(IP4('188.14.166.40')), IPV4(IP4('188.14.166.41'))))")
- self.assertEqual(flt.filter(rule, msg_idea), True)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/lib/pynspect/test_filters_inspector.py b/lib/pynspect/test_filters_inspector.py
deleted file mode 100644
index 39b72a50d63601e89ce8d72f5d26414e8994bcf1..0000000000000000000000000000000000000000
--- a/lib/pynspect/test_filters_inspector.py
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-import os
-import sys
-import shutil
-import unittest
-from pprint import pformat, pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../lib'))
-sys.path.insert(0, lib)
-
-from idea import lite
-from pynspect.jpath import *
-from pynspect.rules import *
-from pynspect.gparser import MentatFilterParser
-from pynspect.filters import DataObjectFilter, IDEAFilterCompiler, clean_variable
-
-#-------------------------------------------------------------------------------
-# NOTE: Sorry for the long lines in this file. They are deliberate, because the
-# assertion permutations are (IMHO) more readable this way.
-#-------------------------------------------------------------------------------
-
-class TestMentatDataObjectFilterInspector(unittest.TestCase):
-
- def test_01_current_inspector_filters(self):
- """
- Perform tests of filters currently used in mentat-inspector.py.
- """
- self.maxDiff = None
-
- flt = DataObjectFilter()
- cpl = IDEAFilterCompiler()
- psr = MentatFilterParser()
- psr.build()
-
- inspection_rules = [
- {
- "filter": "Category in ['Attempt.Login'] and Target.Port in [3389]",
- "str": '((Category OP_IN ["Attempt.Login"]) OP_AND (Target.Port OP_IN [3389]))',
- "tests": [
- [
- {"Format" : "IDEA0","ID" : "e214d2d9-359b-443d-993d-3cc5637107a0","Source":[{"IP4":["188.14.166.39"]}],"Target":[{"IP4":["195.113.165.128/25"],"Port":["3389"],"Proto":["tcp","ssh"]}],"Note":"SSH login attempt","DetectTime" : "2016-06-21 13:08:27Z","Node":[{"Type":["Connection","Honeypot"],"SW":["Kippo"],"Name":"cz.uhk.apate.cowrie"}],"Category":["Attempt.Login"]},
- True
- ],
- [
- {"Format" : "IDEA0","ID" : "e214d2d9-359b-443d-993d-3cc5637107a0","Source":[{"IP4":["188.14.166.39"]}],"Target":[{"IP4":["195.113.165.128/25"],"Port":["338"],"Proto":["tcp","ssh"]}],"Note":"SSH login attempt","DetectTime" : "2016-06-21 13:08:27Z","Node":[{"Type":["Connection","Honeypot"],"SW":["Kippo"],"Name":"cz.uhk.apate.cowrie"}],"Category":["Attempt.Login"]},
- False
- ]
- ],
- },
- {
- "filter": "Category in ['Attempt.Login'] and (Target.Proto in ['telnet'] or Source.Proto in ['telnet'] or Target.Port in [23])",
- "str": '((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["telnet"]) OP_OR ((Source.Proto OP_IN ["telnet"]) OP_OR (Target.Port OP_IN [23]))))'
- },
- {
- "filter": "Category in ['Attempt.Login'] and (Target.Proto in ['ssh'] or Source.Proto in ['ssh'] or Target.Port in [22])",
- "str": '((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["ssh"]) OP_OR ((Source.Proto OP_IN ["ssh"]) OP_OR (Target.Port OP_IN [22]))))'
- },
- {
- "filter": "Category in ['Attempt.Login'] and (Target.Proto in ['sip', 'sip-tls'] or Source.Proto in ['sip', 'sip-tls'] or Target.Port in [5060])",
- "str": '((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["sip", "sip-tls"]) OP_OR ((Source.Proto OP_IN ["sip", "sip-tls"]) OP_OR (Target.Port OP_IN [5060]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Proto in ['sip', 'sip-tls'] or Source.Proto in ['sip', 'sip-tls'] or Target.Port in [5060])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Proto OP_IN ["sip", "sip-tls"]) OP_OR ((Source.Proto OP_IN ["sip", "sip-tls"]) OP_OR (Target.Port OP_IN [5060]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and Target.Port in [23]",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND (Target.Port OP_IN [23]))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Port in [80, 443] or Source.Proto in ['http', 'https', 'http-alt'] or Target.Proto in ['http', 'https', 'http-alt'])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [80, 443]) OP_OR ((Source.Proto OP_IN ["http", "https", "http-alt"]) OP_OR (Target.Proto OP_IN ["http", "https", "http-alt"]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Port in [3306] or Source.Proto in ['mysql'] or Target.Proto in ['mysql'])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [3306]) OP_OR ((Source.Proto OP_IN ["mysql"]) OP_OR (Target.Proto OP_IN ["mysql"]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Port in [445] or Source.Proto in ['microsoft-ds', 'smb'] or Target.Proto in ['microsoft-ds', 'smb'])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [445]) OP_OR ((Source.Proto OP_IN ["microsoft-ds", "smb"]) OP_OR (Target.Proto OP_IN ["microsoft-ds", "smb"]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Port in [135] or Source.Proto in ['loc-srv', 'epmap'] or Target.Proto in ['loc-srv', 'epmap'])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [135]) OP_OR ((Source.Proto OP_IN ["loc-srv", "epmap"]) OP_OR (Target.Proto OP_IN ["loc-srv", "epmap"]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Port in [1900] or Source.Proto in ['upnp', 'ssdp'] or Target.Proto in ['upnp', 'ssdp'])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [1900]) OP_OR ((Source.Proto OP_IN ["upnp", "ssdp"]) OP_OR (Target.Proto OP_IN ["upnp", "ssdp"]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Port in [20, 21, 989, 990] or Source.Proto in ['ftp', 'ftp-data', 'ftps', 'ftps-data'] or Target.Proto in ['ftp', 'ftp-data', 'ftps', 'ftps-data'])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [20, 21, 989, 990]) OP_OR ((Source.Proto OP_IN ["ftp", "ftp-data", "ftps", "ftps-data"]) OP_OR (Target.Proto OP_IN ["ftp", "ftp-data", "ftps", "ftps-data"]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Port in [1433, 1434] or Source.Proto in ['ms-sql-s', 'ms-sql-m'] or Target.Proto in ['ms-sql-s', 'ms-sql-m'])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [1433, 1434]) OP_OR ((Source.Proto OP_IN ["ms-sql-s", "ms-sql-m"]) OP_OR (Target.Proto OP_IN ["ms-sql-s", "ms-sql-m"]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and (Target.Port in [42] or Source.Proto in ['nameserver'] or Target.Proto in ['nameserver'])",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [42]) OP_OR ((Source.Proto OP_IN ["nameserver"]) OP_OR (Target.Proto OP_IN ["nameserver"]))))'
- },
- {
- "filter": "Category in ['Attempt.Exploit'] and Node.SW in ['Dionaea']",
- "str": '((Category OP_IN ["Attempt.Exploit"]) OP_AND (Node.SW OP_IN ["Dionaea"]))'
- },
- {
- "filter": "Category in ['Availability.DoS', 'Availability.DDoS'] and (Target.Proto in ['dns', 'domain'] or Source.Proto in ['dns', 'domain'] or Target.Port in [53] or Source.Port in [53])",
- "str": '((Category OP_IN ["Availability.DoS", "Availability.DDoS"]) OP_AND ((Target.Proto OP_IN ["dns", "domain"]) OP_OR ((Source.Proto OP_IN ["dns", "domain"]) OP_OR ((Target.Port OP_IN [53]) OP_OR (Source.Port OP_IN [53])))))'
- },
- {
- "filter": "Category in ['Availability.DDoS'] and Node.Type in ['Flow'] and Node.Type in ['Statistical']",
- "str": '((Category OP_IN ["Availability.DDoS"]) OP_AND ((Node.Type OP_IN ["Flow"]) OP_AND (Node.Type OP_IN ["Statistical"])))'
- },
- {
- "filter": "Category in ['Abusive.Spam'] and Node.SW in ['UCEPROT']",
- "str": '((Category OP_IN ["Abusive.Spam"]) OP_AND (Node.SW OP_IN ["UCEPROT"]))'
- },
- {
- "filter": "Category in ['Abusive.Spam'] and Node.SW in ['Fail2Ban', 'IntelMQ']",
- "str": '((Category OP_IN ["Abusive.Spam"]) OP_AND (Node.SW OP_IN ["Fail2Ban", "IntelMQ"]))'
- },
- {
- "filter": "Category in ['Vulnerable.Config'] and (Source.Proto in ['qotd'] or Source.Port in [17])",
- "str": '((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["qotd"]) OP_OR (Source.Port OP_IN [17])))'
- },
- {
- "filter": "Category in ['Vulnerable.Config'] and Source.Proto in ['ssdp']",
- "str": '((Category OP_IN ["Vulnerable.Config"]) OP_AND (Source.Proto OP_IN ["ssdp"]))'
- },
- {
- "filter": "Category in ['Vulnerable.Config'] and (Source.Proto in ['ntp'] or Source.Port in [123])",
- "str": '((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["ntp"]) OP_OR (Source.Port OP_IN [123])))'
- },
- {
- "filter": "Category in ['Vulnerable.Config'] and (Source.Proto in ['domain'] or Source.Port in [53])",
- "str": '((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["domain"]) OP_OR (Source.Port OP_IN [53])))'
- },
- {
- "filter": "Category in ['Vulnerable.Config'] and (Source.Proto in ['netbios-ns'] or Source.Port in [137])",
- "str": '((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["netbios-ns"]) OP_OR (Source.Port OP_IN [137])))'
- },
- {
- "filter": "Category in ['Vulnerable.Config'] and (Source.Proto in ['ipmi'] or Source.Port in [623])",
- "str": '((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["ipmi"]) OP_OR (Source.Port OP_IN [623])))'
- },
- {
- "filter": "Category in ['Vulnerable.Config'] and (Source.Proto in ['chargen'] or Source.Port in [19])",
- "str": '((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["chargen"]) OP_OR (Source.Port OP_IN [19])))'
- },
- {
- "filter": "Category in ['Anomaly.Traffic']",
- "str": '(Category OP_IN ["Anomaly.Traffic"])'
- },
- {
- "filter": "Category in ['Anomaly.Connection'] and Source.Type in ['Booter']",
- "str": '((Category OP_IN ["Anomaly.Connection"]) OP_AND (Source.Type OP_IN ["Booter"]))'
- },
- {
- "filter": "Category in ['Intrusion.Botnet'] and Source.Type in ['Botnet']",
- "str": '((Category OP_IN ["Intrusion.Botnet"]) OP_AND (Source.Type OP_IN ["Botnet"]))'
- },
- {
- "filter": "Category in ['Recon.Scanning']",
- "str": '(Category OP_IN ["Recon.Scanning"])'
- }
- ]
-
- for ir in inspection_rules:
- rule = psr.parse(ir['filter'])
- rule = cpl.compile(rule)
- self.assertEqual(str(rule), ir['str'])
- if 'tests' in ir:
- for t in ir['tests']:
- msg_idea = lite.Idea(t[0])
- self.assertEqual([ir['filter'], flt.filter(rule, msg_idea)], [ir['filter'], t[1]])
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/lib/pynspect/test_gparser.py b/lib/pynspect/test_gparser.py
deleted file mode 100644
index 395a841a075e8e62ce203fcd75067921ba86159f..0000000000000000000000000000000000000000
--- a/lib/pynspect/test_gparser.py
+++ /dev/null
@@ -1,266 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-import os
-import sys
-import shutil
-import unittest
-from pprint import pformat, pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../lib'))
-sys.path.insert(0, lib)
-
-from pynspect.gparser import MentatFilterParser
-
-#-------------------------------------------------------------------------------
-# NOTE: Sorry for the long lines in this file. They are deliberate, because the
-# assertion permutations are (IMHO) more readable this way.
-#-------------------------------------------------------------------------------
-
-class TestMentatFilterParser(unittest.TestCase):
-
- def test_01_basic_logical(self):
- """
- Test the parsing of basic logical operations.
- """
- self.maxDiff = None
-
- p = MentatFilterParser()
- p.build()
-
- self.assertEqual(repr(p.parse('1 and 1')), 'LOGBINOP(INTEGER(1) OP_AND INTEGER(1))')
- self.assertEqual(repr(p.parse('1 AND 1')), 'LOGBINOP(INTEGER(1) OP_AND INTEGER(1))')
- self.assertEqual(repr(p.parse('1 && 1')), 'LOGBINOP(INTEGER(1) OP_AND_P INTEGER(1))')
- self.assertEqual(repr(p.parse('1 or 1')), 'LOGBINOP(INTEGER(1) OP_OR INTEGER(1))')
- self.assertEqual(repr(p.parse('1 OR 1')), 'LOGBINOP(INTEGER(1) OP_OR INTEGER(1))')
- self.assertEqual(repr(p.parse('1 || 1')), 'LOGBINOP(INTEGER(1) OP_OR_P INTEGER(1))')
- self.assertEqual(repr(p.parse('1 xor 1')), 'LOGBINOP(INTEGER(1) OP_XOR INTEGER(1))')
- self.assertEqual(repr(p.parse('1 XOR 1')), 'LOGBINOP(INTEGER(1) OP_XOR INTEGER(1))')
- self.assertEqual(repr(p.parse('1 ^^ 1')), 'LOGBINOP(INTEGER(1) OP_XOR_P INTEGER(1))')
- self.assertEqual(repr(p.parse('not 1')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('NOT 1')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('! 1')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('exists 1')), 'UNOP(OP_EXISTS INTEGER(1))')
- self.assertEqual(repr(p.parse('EXISTS 1')), 'UNOP(OP_EXISTS INTEGER(1))')
- self.assertEqual(repr(p.parse('? 1')), 'UNOP(OP_EXISTS INTEGER(1))')
-
- self.assertEqual(repr(p.parse('(1 and 1)')), 'LOGBINOP(INTEGER(1) OP_AND INTEGER(1))')
- self.assertEqual(repr(p.parse('(1 AND 1)')), 'LOGBINOP(INTEGER(1) OP_AND INTEGER(1))')
- self.assertEqual(repr(p.parse('(1 && 1)')), 'LOGBINOP(INTEGER(1) OP_AND_P INTEGER(1))')
- self.assertEqual(repr(p.parse('(1 or 1)')), 'LOGBINOP(INTEGER(1) OP_OR INTEGER(1))')
- self.assertEqual(repr(p.parse('(1 OR 1)')), 'LOGBINOP(INTEGER(1) OP_OR INTEGER(1))')
- self.assertEqual(repr(p.parse('(1 || 1)')), 'LOGBINOP(INTEGER(1) OP_OR_P INTEGER(1))')
- self.assertEqual(repr(p.parse('(1 xor 1)')), 'LOGBINOP(INTEGER(1) OP_XOR INTEGER(1))')
- self.assertEqual(repr(p.parse('(1 XOR 1)')), 'LOGBINOP(INTEGER(1) OP_XOR INTEGER(1))')
- self.assertEqual(repr(p.parse('(1 ^^ 1)')), 'LOGBINOP(INTEGER(1) OP_XOR_P INTEGER(1))')
- self.assertEqual(repr(p.parse('(not 1)')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('(NOT 1)')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('(! 1)')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('(exists 1)')), 'UNOP(OP_EXISTS INTEGER(1))')
- self.assertEqual(repr(p.parse('(EXISTS 1)')), 'UNOP(OP_EXISTS INTEGER(1))')
- self.assertEqual(repr(p.parse('(? 1)')), 'UNOP(OP_EXISTS INTEGER(1))')
-
- self.assertEqual(repr(p.parse('((1 and 1))')), 'LOGBINOP(INTEGER(1) OP_AND INTEGER(1))')
- self.assertEqual(repr(p.parse('((1 AND 1))')), 'LOGBINOP(INTEGER(1) OP_AND INTEGER(1))')
- self.assertEqual(repr(p.parse('((1 && 1))')), 'LOGBINOP(INTEGER(1) OP_AND_P INTEGER(1))')
- self.assertEqual(repr(p.parse('((1 or 1))')), 'LOGBINOP(INTEGER(1) OP_OR INTEGER(1))')
- self.assertEqual(repr(p.parse('((1 OR 1))')), 'LOGBINOP(INTEGER(1) OP_OR INTEGER(1))')
- self.assertEqual(repr(p.parse('((1 || 1))')), 'LOGBINOP(INTEGER(1) OP_OR_P INTEGER(1))')
- self.assertEqual(repr(p.parse('((1 xor 1))')), 'LOGBINOP(INTEGER(1) OP_XOR INTEGER(1))')
- self.assertEqual(repr(p.parse('((1 XOR 1))')), 'LOGBINOP(INTEGER(1) OP_XOR INTEGER(1))')
- self.assertEqual(repr(p.parse('((1 ^^ 1))')), 'LOGBINOP(INTEGER(1) OP_XOR_P INTEGER(1))')
- self.assertEqual(repr(p.parse('((not 1))')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('((NOT 1))')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('((! 1))')), 'UNOP(OP_NOT INTEGER(1))')
- self.assertEqual(repr(p.parse('((exists 1))')), 'UNOP(OP_EXISTS INTEGER(1))')
- self.assertEqual(repr(p.parse('((EXISTS 1))')), 'UNOP(OP_EXISTS INTEGER(1))')
- self.assertEqual(repr(p.parse('((? 1))')), 'UNOP(OP_EXISTS INTEGER(1))')
-
- def test_02_basic_comparison(self):
- """
- Test the parsing of basic comparison operations.
- """
- self.maxDiff = None
-
- p = MentatFilterParser()
- p.build()
-
- self.assertEqual(repr(p.parse('2 like 2')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 LIKE 2')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 =~ 2')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 in 2')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('2 IN 2')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('2 ~~ 2')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('2 is 2')), 'COMPBINOP(INTEGER(2) OP_IS INTEGER(2))')
- self.assertEqual(repr(p.parse('2 IS 2')), 'COMPBINOP(INTEGER(2) OP_IS INTEGER(2))')
- self.assertEqual(repr(p.parse('2 eq 2')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('2 EQ 2')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('2 == 2')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('2 ne 2')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 NE 2')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 != 2')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 <> 2')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 ge 2')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 GE 2')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 >= 2')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 gt 2')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('2 GT 2')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('2 > 2')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('2 le 2')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 LE 2')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 <= 2')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('2 lt 2')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
- self.assertEqual(repr(p.parse('2 LT 2')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
- self.assertEqual(repr(p.parse('2 < 2')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
-
- self.assertEqual(repr(p.parse('(2 like 2)')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 LIKE 2)')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 =~ 2)')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 in 2)')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 IN 2)')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 ~~ 2)')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 is 2)')), 'COMPBINOP(INTEGER(2) OP_IS INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 IS 2)')), 'COMPBINOP(INTEGER(2) OP_IS INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 eq 2)')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 EQ 2)')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 == 2)')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 ne 2)')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 NE 2)')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 != 2)')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 <> 2)')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 ge 2)')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 GE 2)')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 >= 2)')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 gt 2)')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 GT 2)')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 > 2)')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 le 2)')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 LE 2)')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 <= 2)')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 lt 2)')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 LT 2)')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
- self.assertEqual(repr(p.parse('(2 < 2)')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
-
- self.assertEqual(repr(p.parse('((2 like 2))')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 LIKE 2))')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 =~ 2))')), 'COMPBINOP(INTEGER(2) OP_LIKE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 in 2))')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 IN 2))')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 ~~ 2))')), 'COMPBINOP(INTEGER(2) OP_IN INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 is 2))')), 'COMPBINOP(INTEGER(2) OP_IS INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 IS 2))')), 'COMPBINOP(INTEGER(2) OP_IS INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 eq 2))')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 EQ 2))')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 == 2))')), 'COMPBINOP(INTEGER(2) OP_EQ INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 ne 2))')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 NE 2))')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 != 2))')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 <> 2))')), 'COMPBINOP(INTEGER(2) OP_NE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 ge 2))')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 GE 2))')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 >= 2))')), 'COMPBINOP(INTEGER(2) OP_GE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 gt 2))')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 GT 2))')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 > 2))')), 'COMPBINOP(INTEGER(2) OP_GT INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 le 2))')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 LE 2))')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 <= 2))')), 'COMPBINOP(INTEGER(2) OP_LE INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 lt 2))')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 LT 2))')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
- self.assertEqual(repr(p.parse('((2 < 2))')), 'COMPBINOP(INTEGER(2) OP_LT INTEGER(2))')
-
- def test_03_basic_math(self):
- """
- Test the parsing of basic mathematical operations.
- """
- self.maxDiff = None
-
- p = MentatFilterParser()
- p.build()
-
- self.assertEqual(repr(p.parse('3 + 3')), 'MATHBINOP(INTEGER(3) OP_PLUS INTEGER(3))')
- self.assertEqual(repr(p.parse('3 - 3')), 'MATHBINOP(INTEGER(3) OP_MINUS INTEGER(3))')
- self.assertEqual(repr(p.parse('3 * 3')), 'MATHBINOP(INTEGER(3) OP_TIMES INTEGER(3))')
- self.assertEqual(repr(p.parse('3 / 3')), 'MATHBINOP(INTEGER(3) OP_DIVIDE INTEGER(3))')
- self.assertEqual(repr(p.parse('3 % 3')), 'MATHBINOP(INTEGER(3) OP_MODULO INTEGER(3))')
-
- self.assertEqual(repr(p.parse('(3 + 3)')), 'MATHBINOP(INTEGER(3) OP_PLUS INTEGER(3))')
- self.assertEqual(repr(p.parse('(3 - 3)')), 'MATHBINOP(INTEGER(3) OP_MINUS INTEGER(3))')
- self.assertEqual(repr(p.parse('(3 * 3)')), 'MATHBINOP(INTEGER(3) OP_TIMES INTEGER(3))')
- self.assertEqual(repr(p.parse('(3 / 3)')), 'MATHBINOP(INTEGER(3) OP_DIVIDE INTEGER(3))')
- self.assertEqual(repr(p.parse('(3 % 3)')), 'MATHBINOP(INTEGER(3) OP_MODULO INTEGER(3))')
-
- self.assertEqual(repr(p.parse('((3 + 3))')), 'MATHBINOP(INTEGER(3) OP_PLUS INTEGER(3))')
- self.assertEqual(repr(p.parse('((3 - 3))')), 'MATHBINOP(INTEGER(3) OP_MINUS INTEGER(3))')
- self.assertEqual(repr(p.parse('((3 * 3))')), 'MATHBINOP(INTEGER(3) OP_TIMES INTEGER(3))')
- self.assertEqual(repr(p.parse('((3 / 3))')), 'MATHBINOP(INTEGER(3) OP_DIVIDE INTEGER(3))')
- self.assertEqual(repr(p.parse('((3 % 3))')), 'MATHBINOP(INTEGER(3) OP_MODULO INTEGER(3))')
-
- def test_04_basic_factors(self):
- """
- Test parsing of all available factors.
- """
- self.maxDiff = None
-
- p = MentatFilterParser()
- p.build()
-
- self.assertEqual(repr(p.parse("127.0.0.1")), "IPV4('127.0.0.1')")
- self.assertEqual(repr(p.parse("::1")), "IPV6('::1')")
- self.assertEqual(repr(p.parse("1")), "INTEGER(1)")
- self.assertEqual(repr(p.parse("1.1")), "FLOAT(1.1)")
- self.assertEqual(repr(p.parse("Test")), "VARIABLE('Test')")
- self.assertEqual(repr(p.parse('"constant1"')), "CONSTANT('constant1')")
-
- self.assertEqual(repr(p.parse("(127.0.0.1)")), "IPV4('127.0.0.1')")
- self.assertEqual(repr(p.parse("(::1)")), "IPV6('::1')")
- self.assertEqual(repr(p.parse("(1)")), "INTEGER(1)")
- self.assertEqual(repr(p.parse("(1.1)")), "FLOAT(1.1)")
- self.assertEqual(repr(p.parse("(Test)")), "VARIABLE('Test')")
- self.assertEqual(repr(p.parse('("constant1")')), "CONSTANT('constant1')")
-
- self.assertEqual(repr(p.parse("((127.0.0.1))")), "IPV4('127.0.0.1')")
- self.assertEqual(repr(p.parse("((::1))")), "IPV6('::1')")
- self.assertEqual(repr(p.parse("((1))")), "INTEGER(1)")
- self.assertEqual(repr(p.parse("((1.1))")), "FLOAT(1.1)")
- self.assertEqual(repr(p.parse("((Test))")), "VARIABLE('Test')")
- self.assertEqual(repr(p.parse('(("constant1"))')), "CONSTANT('constant1')")
-
- self.assertEqual(repr(p.parse("[127.0.0.1]")), "LIST(IPV4('127.0.0.1'))")
- self.assertEqual(repr(p.parse("[::1]")), "LIST(IPV6('::1'))")
- self.assertEqual(repr(p.parse("[1]")), "LIST(INTEGER(1))")
- self.assertEqual(repr(p.parse("[1.1]")), "LIST(FLOAT(1.1))")
- self.assertEqual(repr(p.parse("[Test]")), "LIST(VARIABLE('Test'))")
- self.assertEqual(repr(p.parse('["constant1"]')), "LIST(CONSTANT('constant1'))")
-
- self.assertEqual(repr(p.parse("[127.0.0.1 , 127.0.0.2]")), "LIST(IPV4('127.0.0.1'), IPV4('127.0.0.2'))")
- self.assertEqual(repr(p.parse("[::1 , ::2]")), "LIST(IPV6('::1'), IPV6('::2'))")
- self.assertEqual(repr(p.parse("[1,2, 3,4 , 5]")), "LIST(INTEGER(1), INTEGER(2), INTEGER(3), INTEGER(4), INTEGER(5))")
- self.assertEqual(repr(p.parse("[1.1,2.2, 3.3,4.4 , 5.5]")), "LIST(FLOAT(1.1), FLOAT(2.2), FLOAT(3.3), FLOAT(4.4), FLOAT(5.5))")
- self.assertEqual(repr(p.parse("[Var1,Var2, Var3,Var4 , Var5 ]")), "LIST(VARIABLE('Var1'), VARIABLE('Var2'), VARIABLE('Var3'), VARIABLE('Var4'), VARIABLE('Var5'))")
- self.assertEqual(repr(p.parse('["c1","c2", "c3","c4" , "c5" ]')), "LIST(CONSTANT('c1'), CONSTANT('c2'), CONSTANT('c3'), CONSTANT('c4'), CONSTANT('c5'))")
-
- def test_05_advanced(self):
- """
- Test parsing of advanced filtering expressions.
- """
- self.maxDiff = None
-
- p = MentatFilterParser()
- p.build()
-
- self.assertEqual(repr(p.parse('Category in ["Abusive.Spam" , "Attempt.Exploit"]')), "COMPBINOP(VARIABLE('Category') OP_IN LIST(CONSTANT('Abusive.Spam'), CONSTANT('Attempt.Exploit')))")
- self.assertEqual(repr(p.parse('Category is ["Abusive.Spam" , "Attempt.Exploit"]')), "COMPBINOP(VARIABLE('Category') OP_IS LIST(CONSTANT('Abusive.Spam'), CONSTANT('Attempt.Exploit')))")
- self.assertEqual(repr(p.parse('Node.Name in ["cz.cesnet.labrea"]')), "COMPBINOP(VARIABLE('Node.Name') OP_IN LIST(CONSTANT('cz.cesnet.labrea')))")
- self.assertEqual(repr(p.parse('Source.IP4 in [127.0.0.1 , 127.0.0.2]')), "COMPBINOP(VARIABLE('Source.IP4') OP_IN LIST(IPV4('127.0.0.1'), IPV4('127.0.0.2')))")
- self.assertEqual(repr(p.parse('(Source.IP4 eq 127.0.0.1) or (Node[#].Name is "cz.cesnet.labrea")')), "LOGBINOP(COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4('127.0.0.1')) OP_OR COMPBINOP(VARIABLE('Node[#].Name') OP_IS CONSTANT('cz.cesnet.labrea')))")
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/lib/pynspect/test_jpath.py b/lib/pynspect/test_jpath.py
deleted file mode 100644
index 46eabfd675ea65e923d54c6abbde401c1efe45b7..0000000000000000000000000000000000000000
--- a/lib/pynspect/test_jpath.py
+++ /dev/null
@@ -1,527 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-import os
-import sys
-import shutil
-import unittest
-from pprint import pformat, pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../lib'))
-sys.path.insert(0, lib)
-
-import pynspect.jpath
-from idea import lite
-from pynspect.jpath import *
-
-#-------------------------------------------------------------------------------
-# NOTE: Sorry for the long lines in this file. They are deliberate, because the
-# assertion permutations are (IMHO) more readable this way.
-#-------------------------------------------------------------------------------
-
-class TestJPath(unittest.TestCase):
-
- msg_dict = {
- "Format": "IDEA0",
- "ID": "MESSAGE_ID",
- "DetectTime": "2016-06-21 13:08:27Z",
- "Category": ["CATEGORY"],
- "ConnCount": 633,
- "Description": "Ping scan",
- "Source": [
- {
- "IP4": ["192.168.1.1", "192.168.1.2"],
- "Proto": ["icmp"]
- },
- {
- "IP4": ["192.168.2.1", "192.168.2.2"],
- "Proto": ["tcp"]
- }
- ],
- "Target": [
- {
- "Proto": ["udp"],
- "IP4": ["192.168.3.1", "192.168.3.2"],
- "Anonymised": True
- }
- ],
- "Node": [
- {
- "SW" : ["KIPPO","FAIL_TO_BAN"],
- "Name" : "node.name"
- }
- ]
- }
-
- msg_idea = lite.Idea(msg_dict)
-
- def test_01_jpath_parse(self):
- """
- Perform the basic JPath parsing tests.
-
- Make sure all possible JPath forms parse correctly.
- """
- self.maxDiff = None
-
- cache_clear()
- self.assertEqual(cache_size(), 0)
-
- self.assertEqual(jpath_parse("Test"), [{'m': 'Test', 'n': 'Test', 'p': 'Test'}])
- self.assertEqual(jpath_parse("Test.Path"), [{'m': 'Test', 'n': 'Test', 'p': 'Test'}, {'m': 'Path', 'n': 'Path', 'p': 'Test.Path'}])
- self.assertEqual(jpath_parse("Long.Test.Path"), [{'m': 'Long', 'n': 'Long', 'p': 'Long'}, {'m': 'Test', 'n': 'Test', 'p': 'Long.Test'}, {'m': 'Path', 'n': 'Path', 'p': 'Long.Test.Path'}])
-
- self.assertEqual(jpath_parse("Long[1].Test.Path"), [{'i': 0, 'm': 'Long[1]', 'n': 'Long', 'p': 'Long[1]'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long[1].Test'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long[1].Test.Path'}])
- self.assertEqual(jpath_parse("Long.Test[2].Path"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, {'i': 1, 'm': 'Test[2]', 'n': 'Test', 'p': 'Long.Test[2]'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long.Test[2].Path'}])
- self.assertEqual(jpath_parse("Long.Test.Path[3]"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long.Test'}, {'i': 2, 'm': 'Path[3]', 'n': 'Path', 'p': 'Long.Test.Path[3]'}])
- self.assertEqual(jpath_parse("Long[1].Test[1].Path"), [{'i': 0, 'm': 'Long[1]', 'n': 'Long', 'p': 'Long[1]'}, {'i': 0, 'm': 'Test[1]', 'n': 'Test', 'p': 'Long[1].Test[1]'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long[1].Test[1].Path'}])
- self.assertEqual(jpath_parse("Long.Test[2].Path[2]"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, {'i': 1, 'm': 'Test[2]', 'n': 'Test', 'p': 'Long.Test[2]'}, {'i': 1, 'm': 'Path[2]', 'n': 'Path', 'p': 'Long.Test[2].Path[2]'}])
- self.assertEqual(jpath_parse("Long[3].Test.Path[3]"), [{'i': 2, 'm': 'Long[3]', 'n': 'Long', 'p': 'Long[3]'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long[3].Test'}, {'i': 2, 'm': 'Path[3]', 'n': 'Path', 'p': 'Long[3].Test.Path[3]'}])
-
- self.assertEqual(jpath_parse("Long[#].Test.Path"), [{'i': -1, 'm': 'Long[#]', 'n': 'Long', 'p': 'Long[#]'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long[#].Test'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long[#].Test.Path'}])
- self.assertEqual(jpath_parse("Long.Test[#].Path"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, {'i': -1, 'm': 'Test[#]', 'n': 'Test', 'p': 'Long.Test[#]'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long.Test[#].Path'}])
- self.assertEqual(jpath_parse("Long.Test.Path[#]"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long.Test'}, {'i': -1, 'm': 'Path[#]', 'n': 'Path', 'p': 'Long.Test.Path[#]'}])
- self.assertEqual(jpath_parse("Long[#].Test[#].Path"), [{'i': -1, 'm': 'Long[#]', 'n': 'Long', 'p': 'Long[#]'}, {'i': -1, 'm': 'Test[#]', 'n': 'Test', 'p': 'Long[#].Test[#]'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long[#].Test[#].Path'}])
- self.assertEqual(jpath_parse("Long.Test[#].Path[#]"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, {'i': -1, 'm': 'Test[#]', 'n': 'Test', 'p': 'Long.Test[#]'}, {'i': -1, 'm': 'Path[#]', 'n': 'Path', 'p': 'Long.Test[#].Path[#]'}])
- self.assertEqual(jpath_parse("Long[#].Test.Path[#]"), [{'i': -1, 'm': 'Long[#]', 'n': 'Long', 'p': 'Long[#]'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long[#].Test'}, {'i': -1, 'm': 'Path[#]', 'n': 'Path', 'p': 'Long[#].Test.Path[#]'}])
-
- self.assertEqual(jpath_parse("Long[*].Test.Path"), [{'i': '*', 'm': 'Long[*]', 'n': 'Long', 'p': 'Long[*]'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long[*].Test'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long[*].Test.Path'}])
- self.assertEqual(jpath_parse("Long.Test[*].Path"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, {'i': '*', 'm': 'Test[*]', 'n': 'Test', 'p': 'Long.Test[*]'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long.Test[*].Path'}])
- self.assertEqual(jpath_parse("Long.Test.Path[*]"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long.Test'}, {'i': '*', 'm': 'Path[*]', 'n': 'Path', 'p': 'Long.Test.Path[*]'}])
- self.assertEqual(jpath_parse("Long[*].Test[*].Path"), [{'i': '*', 'm': 'Long[*]', 'n': 'Long', 'p': 'Long[*]'}, {'i': '*', 'm': 'Test[*]', 'n': 'Test', 'p': 'Long[*].Test[*]'}, { 'm': 'Path', 'n': 'Path', 'p': 'Long[*].Test[*].Path'}])
- self.assertEqual(jpath_parse("Long.Test[*].Path[*]"), [{ 'm': 'Long', 'n': 'Long', 'p': 'Long'}, {'i': '*', 'm': 'Test[*]', 'n': 'Test', 'p': 'Long.Test[*]'}, {'i': '*', 'm': 'Path[*]', 'n': 'Path', 'p': 'Long.Test[*].Path[*]'}])
- self.assertEqual(jpath_parse("Long[*].Test.Path[*]"), [{'i': '*', 'm': 'Long[*]', 'n': 'Long', 'p': 'Long[*]'}, { 'm': 'Test', 'n': 'Test', 'p': 'Long[*].Test'}, {'i': '*', 'm': 'Path[*]', 'n': 'Path', 'p': 'Long[*].Test.Path[*]'}])
-
- self.assertEqual(jpath_parse("Test"), [{'m': 'Test', 'n': 'Test', 'p': 'Test'}])
- self.assertEqual(jpath_parse("test"), [{'m': 'test', 'n': 'test', 'p': 'test'}])
- self.assertEqual(jpath_parse("TEST"), [{'m': 'TEST', 'n': 'TEST', 'p': 'TEST'}])
- self.assertEqual(jpath_parse("_test"), [{'m': '_test', 'n': '_test', 'p': '_test'}])
-
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test/Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test|Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test-Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test-.Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test[]Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'TestValue[]')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test[1]Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test[].Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test.Value[]')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test[-1].Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test.[1].Value')
- self.assertRaisesRegex(JPathException, "Invalid JPath chunk", jpath_parse, 'Test.Value.[1]')
-
- self.assertEqual(jpath_parse_c("Test"), [{'m': 'Test', 'n': 'Test', 'p': 'Test'}])
- self.assertEqual(jpath_parse_c("Test.Path"), [{'m': 'Test', 'n': 'Test', 'p': 'Test'}, {'m': 'Path', 'n': 'Path', 'p': 'Test.Path'}])
- self.assertEqual(jpath_parse_c("Long.Test.Path"), [{'m': 'Long', 'n': 'Long', 'p': 'Long'}, {'m': 'Test', 'n': 'Test', 'p': 'Long.Test'}, {'m': 'Path', 'n': 'Path', 'p': 'Long.Test.Path'}])
- self.assertEqual(jpath_parse_c("Test"), [{'m': 'Test', 'n': 'Test', 'p': 'Test'}])
- self.assertEqual(jpath_parse_c("Test.Path"), [{'m': 'Test', 'n': 'Test', 'p': 'Test'}, {'m': 'Path', 'n': 'Path', 'p': 'Test.Path'}])
- self.assertEqual(jpath_parse_c("Long.Test.Path"), [{'m': 'Long', 'n': 'Long', 'p': 'Long'}, {'m': 'Test', 'n': 'Test', 'p': 'Long.Test'}, {'m': 'Path', 'n': 'Path', 'p': 'Long.Test.Path'}])
-
- self.assertEqual(cache_size(), 3)
- cache_clear()
- self.assertEqual(cache_size(), 0)
-
- def test_02_jpath_values(self):
- """
- Perform the basic JPath values retrieval tests.
-
- Make sure all possible JPath forms return expected results.
- """
- self.maxDiff = None
-
- self.assertEqual(jpath_values(self.msg_dict, 'Format'), ['IDEA0'])
- self.assertEqual(jpath_values(self.msg_dict, 'Format[1]'), [])
- self.assertEqual(jpath_values(self.msg_dict, 'Format[#]'), [])
- self.assertEqual(jpath_values(self.msg_dict, 'Format[*]'), [])
-
- self.assertEqual(jpath_values(self.msg_dict, 'Category'), ['CATEGORY'])
- self.assertEqual(jpath_values(self.msg_dict, 'Category[1]'), ['CATEGORY'])
- self.assertEqual(jpath_values(self.msg_dict, 'Category[2]'), [])
- self.assertEqual(jpath_values(self.msg_dict, 'Category[#]'), ['CATEGORY'])
- self.assertEqual(jpath_values(self.msg_dict, 'Category[*]'), ['CATEGORY'])
-
- self.assertEqual(jpath_values(self.msg_dict, 'Node.SW'), ['KIPPO','FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[1].SW'), ['KIPPO','FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[#].SW'), ['KIPPO','FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[*].SW'), ['KIPPO','FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[#].SW[1]'), ['KIPPO'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[#].SW[2]'), ['FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[#].SW[#]'), ['FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[#].SW[*]'), ['KIPPO','FAIL_TO_BAN'])
-
- self.assertEqual(jpath_values(self.msg_dict, 'Node.Name'), ['node.name'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[1].Name'), ['node.name'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[#].Name'), ['node.name'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[*].Name'), ['node.name'])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[1].Name[1]'), [])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[#].Name[#]'), [])
- self.assertEqual(jpath_values(self.msg_dict, 'Node[*].Name[*]'), [])
-
- self.assertEqual(jpath_values(self.msg_dict, 'Source.IP4'), ['192.168.1.1','192.168.1.2','192.168.2.1','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[1].IP4'), ['192.168.1.1','192.168.1.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[2].IP4'), ['192.168.2.1','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[#].IP4'), ['192.168.2.1','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[*].IP4'), ['192.168.1.1','192.168.1.2','192.168.2.1','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source.IP4[1]'), ['192.168.1.1','192.168.2.1'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source.IP4[2]'), ['192.168.1.2','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source.IP4[#]'), ['192.168.1.2','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source.IP4[*]'), ['192.168.1.1','192.168.1.2','192.168.2.1','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[1].IP4[1]'), ['192.168.1.1'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[1].IP4[2]'), ['192.168.1.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[1].IP4[#]'), ['192.168.1.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[1].IP4[*]'), ['192.168.1.1','192.168.1.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[2].IP4[1]'), ['192.168.2.1'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[2].IP4[2]'), ['192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[2].IP4[#]'), ['192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[2].IP4[*]'), ['192.168.2.1','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[#].IP4[1]'), ['192.168.2.1'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[#].IP4[2]'), ['192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[#].IP4[#]'), ['192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[#].IP4[*]'), ['192.168.2.1','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[*].IP4[1]'), ['192.168.1.1','192.168.2.1'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[*].IP4[2]'), ['192.168.1.2','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[*].IP4[#]'), ['192.168.1.2','192.168.2.2'])
- self.assertEqual(jpath_values(self.msg_dict, 'Source[*].IP4[*]'), ['192.168.1.1','192.168.1.2','192.168.2.1','192.168.2.2'])
-
- self.assertEqual(jpath_values(self.msg_idea, 'Format'), ['IDEA0'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node.Name'), ['node.name'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[1].Name'), ['node.name'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[#].Name'), ['node.name'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[*].Name'), ['node.name'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[1].Name[1]'), [])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[#].Name[#]'), [])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[*].Name[*]'), [])
- self.assertEqual(jpath_values(self.msg_idea, 'Node.SW'), ['KIPPO','FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[1].SW'), ['KIPPO','FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[#].SW'), ['KIPPO','FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[*].SW'), ['KIPPO','FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[#].SW[1]'), ['KIPPO'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[#].SW[2]'), ['FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[#].SW[#]'), ['FAIL_TO_BAN'])
- self.assertEqual(jpath_values(self.msg_idea, 'Node[#].SW[*]'), ['KIPPO','FAIL_TO_BAN'])
-
- def test_03_jpath_value(self):
- """
- Perform the basic JPath value retrieval tests.
-
- Make sure all possible JPath forms return expected results.
- """
- self.maxDiff = None
-
- self.assertEqual(jpath_value(self.msg_dict, 'Format'), 'IDEA0')
- self.assertEqual(jpath_value(self.msg_dict, 'Format[1]'), None)
- self.assertEqual(jpath_value(self.msg_dict, 'Format[#]'), None)
- self.assertEqual(jpath_value(self.msg_dict, 'Format[*]'), None)
-
- self.assertEqual(jpath_value(self.msg_dict, 'Category'), 'CATEGORY')
- self.assertEqual(jpath_value(self.msg_dict, 'Category[1]'), 'CATEGORY')
- self.assertEqual(jpath_value(self.msg_dict, 'Category[2]'), None)
- self.assertEqual(jpath_value(self.msg_dict, 'Category[#]'), 'CATEGORY')
- self.assertEqual(jpath_value(self.msg_dict, 'Category[*]'), 'CATEGORY')
-
- self.assertEqual(jpath_value(self.msg_dict, 'Node.SW'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[1].SW'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[#].SW'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[*].SW'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[#].SW[1]'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[#].SW[2]'), 'FAIL_TO_BAN')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[#].SW[#]'), 'FAIL_TO_BAN')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[#].SW[*]'), 'KIPPO')
-
- self.assertEqual(jpath_value(self.msg_dict, 'Node.Name'), 'node.name')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[1].Name'), 'node.name')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[#].Name'), 'node.name')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[*].Name'), 'node.name')
- self.assertEqual(jpath_value(self.msg_dict, 'Node[1].Name[1]'), None)
- self.assertEqual(jpath_value(self.msg_dict, 'Node[#].Name[#]'), None)
- self.assertEqual(jpath_value(self.msg_dict, 'Node[*].Name[*]'), None)
-
- self.assertEqual(jpath_value(self.msg_dict, 'Source.IP4'), '192.168.1.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[1].IP4'), '192.168.1.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[2].IP4'), '192.168.2.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[#].IP4'), '192.168.2.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[*].IP4'), '192.168.1.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source.IP4[1]'), '192.168.1.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source.IP4[2]'), '192.168.1.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source.IP4[#]'), '192.168.1.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source.IP4[*]'), '192.168.1.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[1].IP4[1]'), '192.168.1.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[1].IP4[2]'), '192.168.1.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[1].IP4[#]'), '192.168.1.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[1].IP4[*]'), '192.168.1.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[2].IP4[1]'), '192.168.2.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[2].IP4[2]'), '192.168.2.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[2].IP4[#]'), '192.168.2.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[2].IP4[*]'), '192.168.2.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[#].IP4[1]'), '192.168.2.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[#].IP4[2]'), '192.168.2.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[#].IP4[#]'), '192.168.2.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[#].IP4[*]'), '192.168.2.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[*].IP4[1]'), '192.168.1.1')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[*].IP4[2]'), '192.168.1.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[*].IP4[#]'), '192.168.1.2')
- self.assertEqual(jpath_value(self.msg_dict, 'Source[*].IP4[*]'), '192.168.1.1')
-
- self.assertEqual(jpath_value(self.msg_idea, 'Format'), 'IDEA0')
- self.assertEqual(jpath_value(self.msg_idea, 'Node.Name'), 'node.name')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[1].Name'), 'node.name')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[#].Name'), 'node.name')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[*].Name'), 'node.name')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[1].Name[1]'), None)
- self.assertEqual(jpath_value(self.msg_idea, 'Node[#].Name[#]'), None)
- self.assertEqual(jpath_value(self.msg_idea, 'Node[*].Name[*]'), None)
- self.assertEqual(jpath_value(self.msg_idea, 'Node.SW'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[1].SW'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[#].SW'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[*].SW'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[#].SW[1]'), 'KIPPO')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[#].SW[2]'), 'FAIL_TO_BAN')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[#].SW[#]'), 'FAIL_TO_BAN')
- self.assertEqual(jpath_value(self.msg_idea, 'Node[#].SW[*]'), 'KIPPO')
-
- def test_04_jpath_exists(self):
- """
- Perform the basic JPath elements existence tests.
-
- Make sure all possible JPath forms return expected results.
- """
- self.maxDiff = None
-
- self.assertEqual(jpath_exists(self.msg_dict, 'Format'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Format[1]'), False)
- self.assertEqual(jpath_exists(self.msg_dict, 'Format[#]'), False)
- self.assertEqual(jpath_exists(self.msg_dict, 'Format[*]'), False)
-
- self.assertEqual(jpath_exists(self.msg_dict, 'Category'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Category[1]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Category[2]'), False)
- self.assertEqual(jpath_exists(self.msg_dict, 'Category[#]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Category[*]'), True)
-
- self.assertEqual(jpath_exists(self.msg_dict, 'Node.SW'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[1].SW'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[#].SW'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[*].SW'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[#].SW[1]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[#].SW[2]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[#].SW[#]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[#].SW[*]'), True)
-
- self.assertEqual(jpath_exists(self.msg_dict, 'Node.Name'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[1].Name'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[#].Name'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[*].Name'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[1].Name[1]'), False)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[#].Name[#]'), False)
- self.assertEqual(jpath_exists(self.msg_dict, 'Node[*].Name[*]'), False)
-
- self.assertEqual(jpath_exists(self.msg_dict, 'Source.IP4'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[1].IP4'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[2].IP4'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[#].IP4'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[*].IP4'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source.IP4[1]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source.IP4[2]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source.IP4[#]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source.IP4[*]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[1].IP4[1]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[1].IP4[2]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[1].IP4[#]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[1].IP4[*]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[2].IP4[1]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[2].IP4[2]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[2].IP4[#]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[2].IP4[*]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[#].IP4[1]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[#].IP4[2]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[#].IP4[#]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[#].IP4[*]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[*].IP4[1]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[*].IP4[2]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[*].IP4[#]'), True)
- self.assertEqual(jpath_exists(self.msg_dict, 'Source[*].IP4[*]'), True)
-
- self.assertEqual(jpath_exists(self.msg_idea, 'Format'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node.Name'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[1].Name'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[#].Name'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[*].Name'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[1].Name[1]'), False)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[#].Name[#]'), False)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[*].Name[*]'), False)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node.SW'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[1].SW'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[#].SW'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[*].SW'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[#].SW[1]'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[#].SW[2]'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[#].SW[#]'), True)
- self.assertEqual(jpath_exists(self.msg_idea, 'Node[#].SW[*]'), True)
-
- def test_05_jpath_set(self):
- """
- Perform the basic JPath value setting tests.
- """
- self.maxDiff = None
-
- msg = {}
- self.assertEqual(jpath_set(msg, 'TestA.ValueA1', 'A1'), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestA': { 'ValueA1': 'A1'}
- }
- )
- self.assertEqual(jpath_set(msg, 'TestA.ValueA2', 'A2'), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestA': { 'ValueA1': 'A1', 'ValueA2': 'A2' }
- }
- )
- self.assertEqual(jpath_set(msg, 'TestB[1].ValueB1', 'B1'), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestA': { 'ValueA1': 'A1', 'ValueA2': 'A2' },
- 'TestB': [{ 'ValueB1': 'B1' }]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestB[#].ValueB2', 'B2'), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestA': { 'ValueA1': 'A1', 'ValueA2': 'A2' },
- 'TestB': [{ 'ValueB1': 'B1', 'ValueB2': 'B2' }]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestB[*].ValueB3', 'B3'), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestA': { 'ValueA1': 'A1', 'ValueA2': 'A2' },
- 'TestB': [{ 'ValueB1': 'B1', 'ValueB2': 'B2' }, { 'ValueB3': 'B3' }]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestB[#].ValueB4', 'B4'), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestA': { 'ValueA1': 'A1', 'ValueA2': 'A2' },
- 'TestB': [{ 'ValueB1': 'B1', 'ValueB2': 'B2' }, { 'ValueB3': 'B3', 'ValueB4': 'B4' }]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestB[#]', 'DROP'), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestA': { 'ValueA1': 'A1', 'ValueA2': 'A2' },
- 'TestB': [{ 'ValueB1': 'B1', 'ValueB2': 'B2' }, "DROP"]
- }
- )
-
- # This will fail, because "TestA" node is not a list
- self.assertRaisesRegex(JPathException, "Expected list-like object under structure key", jpath_set, msg, 'TestA[#].ValueC1', 'C1')
-
- # This will fail, because "TestA.ValueA1" node is not a dict
- self.assertRaisesRegex(JPathException, "Expected dict-like object under structure key", jpath_set, msg, 'TestA.ValueA1.ValueC1', 'C1')
-
- # This will fail, because we try to attach a node to scalar "TestB[#]"
- self.assertRaisesRegex(JPathException, "Expected dict-like structure to attach node", jpath_set, msg, 'TestB[#].ValueB5', 'RAISE EXCEPTION')
-
- def test_06_jpath_set_unique(self):
- """
- Perform JPath value setting tests with unique flag.
- """
- self.maxDiff = None
-
- msg = {}
- self.assertEqual(jpath_set(msg, 'TestC[#].ListVals1[*]', 'LV1', unique = True), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestC': [{ 'ListVals1': ['LV1']}]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestC[#].ListVals1[*]', 'LV2', unique = True), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestC': [{ 'ListVals1': ['LV1','LV2']}]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestC[#].ListVals1[*]', 'LV1', unique = True), RC_VALUE_DUPLICATE)
- self.assertEqual(
- msg,
- {
- 'TestC': [{ 'ListVals1': ['LV1','LV2']}]
- }
- )
-
- def test_07_jpath_set_overwrite(self):
- """
- Perform JPath value setting tests with overwrite flag.
- """
- self.maxDiff = None
-
- msg = {}
-
- #
- # Overwriting in lists.
- #
- self.assertEqual(jpath_set(msg, 'TestD[#].ListVals1[*]', 'LV1', overwrite = False), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestD': [{ 'ListVals1': ['LV1']}]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestD[#].ListVals1[*]', 'LV2', overwrite = False), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestD': [{ 'ListVals1': ['LV1','LV2']}]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestD[#].ListVals1[2]', 'LV3', overwrite = False), RC_VALUE_EXISTS)
- self.assertEqual(
- msg,
- {
- 'TestD': [{ 'ListVals1': ['LV1','LV2']}]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestD[#].ListVals1[3]', 'LV3', overwrite = False), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestD': [{ 'ListVals1': ['LV1','LV2','LV3']}]
- }
- )
-
- #
- # Overwriting in dicts.
- #
- self.assertEqual(jpath_set(msg, 'TestD[#].DictVal', 'DV1', overwrite = False), RC_VALUE_SET)
- self.assertEqual(
- msg,
- {
- 'TestD': [{ 'ListVals1': ['LV1','LV2','LV3'], 'DictVal': 'DV1' }]
- }
- )
- self.assertEqual(jpath_set(msg, 'TestD[#].DictVal', 'DV2', overwrite = False), RC_VALUE_EXISTS)
- self.assertEqual(
- msg,
- {
- 'TestD': [{ 'ListVals1': ['LV1','LV2','LV3'], 'DictVal': 'DV1' }]
- }
- )
-
-
-if __name__ == "__main__":
- unittest.main()
diff --git a/lib/pynspect/test_lexer.py b/lib/pynspect/test_lexer.py
deleted file mode 100644
index df40e5d4d851664fcca1a81aa01e8012be5e5b41..0000000000000000000000000000000000000000
--- a/lib/pynspect/test_lexer.py
+++ /dev/null
@@ -1,106 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-import os
-import sys
-import shutil
-import unittest
-from pprint import pformat, pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../lib'))
-sys.path.insert(0, lib)
-
-from pynspect.lexer import MentatFilterLexer
-
-#-------------------------------------------------------------------------------
-# NOTE: Sorry for the long lines in this file. They are deliberate, because the
-# assertion permutations are (IMHO) more readable this way.
-#-------------------------------------------------------------------------------
-
-class TestMentatFilterLexer(unittest.TestCase):
-
- def test_01_basic(self):
- """
- Perfom basic lexer tests: check, that all lexical tokens are
- correctly recognized.
- """
- self.maxDiff = None
-
- l = MentatFilterLexer()
- l.build()
-
- self.assertEqual(l.test('+-*/%'), "LexToken(OP_PLUS,'OP_PLUS',1,0)LexToken(OP_MINUS,'OP_MINUS',1,1)LexToken(OP_TIMES,'OP_TIMES',1,2)LexToken(OP_DIVIDE,'OP_DIVIDE',1,3)LexToken(OP_MODULO,'OP_MODULO',1,4)")
-
- self.assertEqual(l.test('OR or ||'), "LexToken(OP_OR,'OP_OR',1,0)LexToken(OP_OR,'OP_OR',1,3)LexToken(OP_OR_P,'OP_OR_P',1,6)")
- self.assertEqual(l.test('XOR xor ^^'), "LexToken(OP_XOR,'OP_XOR',1,0)LexToken(OP_XOR,'OP_XOR',1,4)LexToken(OP_XOR_P,'OP_XOR_P',1,8)")
- self.assertEqual(l.test('AND and &&'), "LexToken(OP_AND,'OP_AND',1,0)LexToken(OP_AND,'OP_AND',1,4)LexToken(OP_AND_P,'OP_AND_P',1,8)")
- self.assertEqual(l.test('NOT not !'), "LexToken(OP_NOT,'OP_NOT',1,0)LexToken(OP_NOT,'OP_NOT',1,4)LexToken(OP_NOT,'OP_NOT',1,8)")
- self.assertEqual(l.test('EXISTS exists ?'), "LexToken(OP_EXISTS,'OP_EXISTS',1,0)LexToken(OP_EXISTS,'OP_EXISTS',1,7)LexToken(OP_EXISTS,'OP_EXISTS',1,14)")
-
- self.assertEqual(l.test('LIKE like =~'), "LexToken(OP_LIKE,'OP_LIKE',1,0)LexToken(OP_LIKE,'OP_LIKE',1,5)LexToken(OP_LIKE,'OP_LIKE',1,10)")
- self.assertEqual(l.test('IN in ~~'), "LexToken(OP_IN,'OP_IN',1,0)LexToken(OP_IN,'OP_IN',1,3)LexToken(OP_IN,'OP_IN',1,6)")
- self.assertEqual(l.test('IS is'), "LexToken(OP_IS,'OP_IS',1,0)LexToken(OP_IS,'OP_IS',1,3)")
- self.assertEqual(l.test('EQ eq =='), "LexToken(OP_EQ,'OP_EQ',1,0)LexToken(OP_EQ,'OP_EQ',1,3)LexToken(OP_EQ,'OP_EQ',1,6)")
- self.assertEqual(l.test('NE ne <> !='), "LexToken(OP_NE,'OP_NE',1,0)LexToken(OP_NE,'OP_NE',1,3)LexToken(OP_NE,'OP_NE',1,6)LexToken(OP_NE,'OP_NE',1,9)")
- self.assertEqual(l.test('GT gt >'), "LexToken(OP_GT,'OP_GT',1,0)LexToken(OP_GT,'OP_GT',1,3)LexToken(OP_GT,'OP_GT',1,6)")
- self.assertEqual(l.test('GE ge >='), "LexToken(OP_GE,'OP_GE',1,0)LexToken(OP_GE,'OP_GE',1,3)LexToken(OP_GE,'OP_GE',1,6)")
- self.assertEqual(l.test('LT lt <'), "LexToken(OP_LT,'OP_LT',1,0)LexToken(OP_LT,'OP_LT',1,3)LexToken(OP_LT,'OP_LT',1,6)")
- self.assertEqual(l.test('LE le <='), "LexToken(OP_LE,'OP_LE',1,0)LexToken(OP_LE,'OP_LE',1,3)LexToken(OP_LE,'OP_LE',1,6)")
-
- self.assertEqual(l.test('127.0.0.1'), "LexToken(IPV4,('IPV4', '127.0.0.1'),1,0)")
- self.assertEqual(l.test('127.0.0.1/32'), "LexToken(IPV4,('IPV4', '127.0.0.1/32'),1,0)")
- self.assertEqual(l.test('127.0.0.1-127.0.0.5'), "LexToken(IPV4,('IPV4', '127.0.0.1-127.0.0.5'),1,0)")
- self.assertEqual(l.test('127.0.0.1..127.0.0.5'), "LexToken(IPV4,('IPV4', '127.0.0.1..127.0.0.5'),1,0)")
-
- self.assertEqual(l.test('::1'), "LexToken(IPV6,('IPV6', '::1'),1,0)")
- self.assertEqual(l.test('::1/64'), "LexToken(IPV6,('IPV6', '::1/64'),1,0)")
- self.assertEqual(l.test('::1-::5'), "LexToken(IPV6,('IPV6', '::1-::5'),1,0)")
- self.assertEqual(l.test('::1..::5'), "LexToken(IPV6,('IPV6', '::1..::5'),1,0)")
-
- self.assertEqual(l.test('15'), "LexToken(INTEGER,('INTEGER', 15),1,0)")
- self.assertEqual(l.test('15.5'), "LexToken(FLOAT,('FLOAT', 15.5),1,0)")
-
- self.assertEqual(l.test('S'), "LexToken(VARIABLE,('VARIABLE', 'S'),1,0)")
- self.assertEqual(l.test('S.N'), "LexToken(VARIABLE,('VARIABLE', 'S.N'),1,0)")
- self.assertEqual(l.test('Source.Node'), "LexToken(VARIABLE,('VARIABLE', 'Source.Node'),1,0)")
- self.assertEqual(l.test('Source[1].Node[2]'), "LexToken(VARIABLE,('VARIABLE', 'Source[1].Node[2]'),1,0)")
- self.assertEqual(l.test('Source[-1].Node[-2]'), "LexToken(VARIABLE,('VARIABLE', 'Source[-1].Node[-2]'),1,0)")
- self.assertEqual(l.test('Source[#].Node[#]'), "LexToken(VARIABLE,('VARIABLE', 'Source[#].Node[#]'),1,0)")
- self.assertEqual(l.test('"Value 525.89:X><"'), "LexToken(CONSTANT,('CONSTANT', 'Value 525.89:X><'),1,0)")
- self.assertEqual(l.test("'Value 525.89:X><'"), "LexToken(CONSTANT,('CONSTANT', 'Value 525.89:X><'),1,0)")
-
- self.assertEqual(l.test(','), "LexToken(COMMA,',',1,0)")
- self.assertEqual(l.test(', '), "LexToken(COMMA,', ',1,0)")
- self.assertEqual(l.test(' , '), "LexToken(COMMA,', ',1,1)")
- self.assertEqual(l.test(' , '), "LexToken(COMMA,', ',1,2)")
- self.assertEqual(l.test(';'), "LexToken(COMMA,';',1,0)")
- self.assertEqual(l.test('; '), "LexToken(COMMA,'; ',1,0)")
- self.assertEqual(l.test(' ; '), "LexToken(COMMA,'; ',1,1)")
- self.assertEqual(l.test(' ; '), "LexToken(COMMA,'; ',1,2)")
-
- self.assertEqual(l.test('()'), "LexToken(LPAREN,'(',1,0)LexToken(RPAREN,')',1,1)")
- self.assertEqual(l.test('[]'), "LexToken(LBRACK,'[',1,0)LexToken(RBRACK,']',1,1)")
-
- self.assertEqual(l.test('[127.0.0.1 , 127.0.0.2]'), "LexToken(LBRACK,'[',1,0)LexToken(IPV4,('IPV4', '127.0.0.1'),1,1)LexToken(COMMA,', ',1,11)LexToken(IPV4,('IPV4', '127.0.0.2'),1,13)LexToken(RBRACK,']',1,22)")
- self.assertEqual(l.test('[::1 , ::2]'), "LexToken(LBRACK,'[',1,0)LexToken(IPV6,('IPV6', '::1'),1,1)LexToken(COMMA,', ',1,5)LexToken(IPV6,('IPV6', '::2'),1,7)LexToken(RBRACK,']',1,10)")
- self.assertEqual(l.test('[1,2, 3,4 , 5 ]'), "LexToken(LBRACK,'[',1,0)LexToken(INTEGER,('INTEGER', 1),1,1)LexToken(COMMA,',',1,2)LexToken(INTEGER,('INTEGER', 2),1,3)LexToken(COMMA,', ',1,4)LexToken(INTEGER,('INTEGER', 3),1,6)LexToken(COMMA,',',1,7)LexToken(INTEGER,('INTEGER', 4),1,8)LexToken(COMMA,', ',1,10)LexToken(INTEGER,('INTEGER', 5),1,12)LexToken(RBRACK,']',1,14)")
- self.assertEqual(l.test('[15.5,16.6, 17.7,18.8 , 19.9 ]'), "LexToken(LBRACK,'[',1,0)LexToken(FLOAT,('FLOAT', 15.5),1,1)LexToken(COMMA,',',1,5)LexToken(FLOAT,('FLOAT', 16.6),1,6)LexToken(COMMA,', ',1,10)LexToken(FLOAT,('FLOAT', 17.7),1,12)LexToken(COMMA,',',1,16)LexToken(FLOAT,('FLOAT', 18.8),1,17)LexToken(COMMA,', ',1,22)LexToken(FLOAT,('FLOAT', 19.9),1,24)LexToken(RBRACK,']',1,29)")
- self.assertEqual(l.test('[Test.Node1,Test.Node2, Test.Node3,Test.Node4 , Test.Node5 ]'), "LexToken(LBRACK,'[',1,0)LexToken(VARIABLE,('VARIABLE', 'Test.Node1'),1,1)LexToken(COMMA,',',1,11)LexToken(VARIABLE,('VARIABLE', 'Test.Node2'),1,12)LexToken(COMMA,', ',1,22)LexToken(VARIABLE,('VARIABLE', 'Test.Node3'),1,24)LexToken(COMMA,',',1,34)LexToken(VARIABLE,('VARIABLE', 'Test.Node4'),1,35)LexToken(COMMA,', ',1,46)LexToken(VARIABLE,('VARIABLE', 'Test.Node5'),1,48)LexToken(RBRACK,']',1,59)")
- self.assertEqual(l.test('["constant1","constant2", "constant3","constant4" , "constant5" ]'), "LexToken(LBRACK,'[',1,0)LexToken(CONSTANT,('CONSTANT', 'constant1'),1,1)LexToken(COMMA,',',1,12)LexToken(CONSTANT,('CONSTANT', 'constant2'),1,13)LexToken(COMMA,', ',1,24)LexToken(CONSTANT,('CONSTANT', 'constant3'),1,26)LexToken(COMMA,',',1,37)LexToken(CONSTANT,('CONSTANT', 'constant4'),1,38)LexToken(COMMA,', ',1,50)LexToken(CONSTANT,('CONSTANT', 'constant5'),1,52)LexToken(RBRACK,']',1,64)")
- self.assertEqual(l.test('[127.0.0.1 ; 127.0.0.2]'), "LexToken(LBRACK,'[',1,0)LexToken(IPV4,('IPV4', '127.0.0.1'),1,1)LexToken(COMMA,'; ',1,11)LexToken(IPV4,('IPV4', '127.0.0.2'),1,13)LexToken(RBRACK,']',1,22)")
- self.assertEqual(l.test('[::1 ; ::2]'), "LexToken(LBRACK,'[',1,0)LexToken(IPV6,('IPV6', '::1'),1,1)LexToken(COMMA,'; ',1,5)LexToken(IPV6,('IPV6', '::2'),1,7)LexToken(RBRACK,']',1,10)")
- self.assertEqual(l.test('[1;2; 3;4 ; 5 ]'), "LexToken(LBRACK,'[',1,0)LexToken(INTEGER,('INTEGER', 1),1,1)LexToken(COMMA,';',1,2)LexToken(INTEGER,('INTEGER', 2),1,3)LexToken(COMMA,'; ',1,4)LexToken(INTEGER,('INTEGER', 3),1,6)LexToken(COMMA,';',1,7)LexToken(INTEGER,('INTEGER', 4),1,8)LexToken(COMMA,'; ',1,10)LexToken(INTEGER,('INTEGER', 5),1,12)LexToken(RBRACK,']',1,14)")
- self.assertEqual(l.test('[15.5;16.6; 17.7;18.8 ; 19.9 ]'), "LexToken(LBRACK,'[',1,0)LexToken(FLOAT,('FLOAT', 15.5),1,1)LexToken(COMMA,';',1,5)LexToken(FLOAT,('FLOAT', 16.6),1,6)LexToken(COMMA,'; ',1,10)LexToken(FLOAT,('FLOAT', 17.7),1,12)LexToken(COMMA,';',1,16)LexToken(FLOAT,('FLOAT', 18.8),1,17)LexToken(COMMA,'; ',1,22)LexToken(FLOAT,('FLOAT', 19.9),1,24)LexToken(RBRACK,']',1,29)")
- self.assertEqual(l.test('[Test.Node1;Test.Node2; Test.Node3;Test.Node4 ; Test.Node5 ]'), "LexToken(LBRACK,'[',1,0)LexToken(VARIABLE,('VARIABLE', 'Test.Node1'),1,1)LexToken(COMMA,';',1,11)LexToken(VARIABLE,('VARIABLE', 'Test.Node2'),1,12)LexToken(COMMA,'; ',1,22)LexToken(VARIABLE,('VARIABLE', 'Test.Node3'),1,24)LexToken(COMMA,';',1,34)LexToken(VARIABLE,('VARIABLE', 'Test.Node4'),1,35)LexToken(COMMA,'; ',1,46)LexToken(VARIABLE,('VARIABLE', 'Test.Node5'),1,48)LexToken(RBRACK,']',1,59)")
- self.assertEqual(l.test('["constant1";"constant2"; "constant3";"constant4" ; "constant5" ]'), "LexToken(LBRACK,'[',1,0)LexToken(CONSTANT,('CONSTANT', 'constant1'),1,1)LexToken(COMMA,';',1,12)LexToken(CONSTANT,('CONSTANT', 'constant2'),1,13)LexToken(COMMA,'; ',1,24)LexToken(CONSTANT,('CONSTANT', 'constant3'),1,26)LexToken(COMMA,';',1,37)LexToken(CONSTANT,('CONSTANT', 'constant4'),1,38)LexToken(COMMA,'; ',1,50)LexToken(CONSTANT,('CONSTANT', 'constant5'),1,52)LexToken(RBRACK,']',1,64)")
- self.assertEqual(l.test(''), "")
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/lib/pynspect/test_rules.py b/lib/pynspect/test_rules.py
deleted file mode 100644
index 6caf6f0e1c94f84a71f92e7df4559e2c96ab50d7..0000000000000000000000000000000000000000
--- a/lib/pynspect/test_rules.py
+++ /dev/null
@@ -1,354 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#-------------------------------------------------------------------------------
-# This file is part of Mentat system (https://mentat.cesnet.cz/).
-#
-# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
-# Use of this source is governed by the MIT license, see LICENSE file.
-#-------------------------------------------------------------------------------
-
-import os
-import sys
-import shutil
-import unittest
-from pprint import pformat, pprint
-
-# Generate the path to custom 'lib' directory
-lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../lib'))
-sys.path.insert(0, lib)
-
-from pynspect.rules import *
-
-#-------------------------------------------------------------------------------
-# NOTE: Sorry for the long lines in this file. They are deliberate, because the
-# assertion permutations are (IMHO) more readable this way.
-#-------------------------------------------------------------------------------
-
-class TestMentatRules(unittest.TestCase):
-
- def test_01_basic(self):
- """
- Perform basic rules tests: instantinate and check all rule objects.
- """
- self.maxDiff = None
-
- rule_var = VariableRule("Test")
- self.assertEqual(str(rule_var), "Test")
- self.assertEqual(repr(rule_var), "VARIABLE('Test')")
- rule_const = ConstantRule("constant")
- self.assertEqual(str(rule_const), '"constant"')
- self.assertEqual(repr(rule_const), "CONSTANT('constant')")
- rule_ipv4 = IPV4Rule("127.0.0.1")
- self.assertEqual(str(rule_ipv4), "127.0.0.1")
- self.assertEqual(repr(rule_ipv4), "IPV4('127.0.0.1')")
- rule_ipv6 = IPV6Rule("::1")
- self.assertEqual(str(rule_ipv6), "::1")
- self.assertEqual(repr(rule_ipv6), "IPV6('::1')")
- rule_integer = IntegerRule(15)
- self.assertEqual(str(rule_integer), "15")
- self.assertEqual(repr(rule_integer), "INTEGER(15)")
- rule_float = FloatRule(15.5)
- self.assertEqual(str(rule_float), "15.5")
- self.assertEqual(repr(rule_float), "FLOAT(15.5)")
- rule_list = ListRule(VariableRule("Test"), ListRule(ConstantRule("constant"), ListRule(IPV4Rule("127.0.0.1"))))
- self.assertEqual(str(rule_list), '[Test, "constant", 127.0.0.1]')
- self.assertEqual(repr(rule_list), "LIST(VARIABLE('Test'), CONSTANT('constant'), IPV4('127.0.0.1'))")
- self.assertEqual(str(rule_list.value), "[VARIABLE('Test'), CONSTANT('constant'), IPV4('127.0.0.1')]")
- self.assertEqual(pformat(rule_list.value), "[VARIABLE('Test'), CONSTANT('constant'), IPV4('127.0.0.1')]")
- rule_binop_l = LogicalBinOpRule("OP_OR", rule_var, rule_integer)
- self.assertEqual(str(rule_binop_l), "(Test OP_OR 15)")
- self.assertEqual(repr(rule_binop_l), "LOGBINOP(VARIABLE('Test') OP_OR INTEGER(15))")
- rule_binop_c = ComparisonBinOpRule("OP_GT", rule_var, rule_integer)
- self.assertEqual(str(rule_binop_c), "(Test OP_GT 15)")
- self.assertEqual(repr(rule_binop_c), "COMPBINOP(VARIABLE('Test') OP_GT INTEGER(15))")
- rule_binop_m = MathBinOpRule("OP_PLUS", rule_var, rule_integer)
- self.assertEqual(str(rule_binop_m), "(Test OP_PLUS 15)")
- self.assertEqual(repr(rule_binop_m), "MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(15))")
- rule_binop = LogicalBinOpRule("OP_OR", ComparisonBinOpRule("OP_GT", MathBinOpRule("OP_PLUS", VariableRule("Test"), IntegerRule(10)), IntegerRule(20)), ComparisonBinOpRule("OP_LT", VariableRule("Test"), IntegerRule(5)))
- self.assertEqual(str(rule_binop), "(((Test OP_PLUS 10) OP_GT 20) OP_OR (Test OP_LT 5))")
- self.assertEqual(repr(rule_binop), "LOGBINOP(COMPBINOP(MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(10)) OP_GT INTEGER(20)) OP_OR COMPBINOP(VARIABLE('Test') OP_LT INTEGER(5)))")
- rule_unop = UnaryOperationRule("OP_NOT", rule_var)
- self.assertEqual(str(rule_unop), "(OP_NOT Test)")
- self.assertEqual(repr(rule_unop), "UNOP(OP_NOT VARIABLE('Test'))")
-
-class TestMentatRuleTreeTraverser(unittest.TestCase):
-
- def test_01_evaluate_binops_logical(self):
- """
- Test the logical binary operations evaluations.
- """
- self.maxDiff = None
-
- traverser = RuleTreeTraverser()
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', 1, 1), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', 0, 1), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', 1, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', None, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', 0, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', None, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', 0, 0), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', 1, 1), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', 0, 1), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', 1, 0), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', None, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', 0, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', None, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', 1, None), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', None, 1), True)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', 1, 1), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', 0, 1), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', 1, 0), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', None, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', 0, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', None, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', 1, None), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', None, 1), True)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', "True", "True"), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', "", "True"), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', "True", ""), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', "", ""), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', "True", "True"), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', "", "True"), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', "True", ""), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', "", ""), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', "True", "True"), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', "", "True"), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', "True", ""), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', "", ""), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', [1,2], [1,2]), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', [], [1,2]), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', [1,2], []), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', [], []), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', [1,2], [1,2]), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', [], [1,2]), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', [1,2], []), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', [], []), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', [1,2], [1,2]), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', [], [1,2]), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', [1,2], []), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', [], []), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', {"x":1}, {"x":1}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', {}, {"x":1}), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', {"x":1}, {}), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND', {}, {}), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', {"x":1}, {"x":1}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', {}, {"x":1}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', {"x":1}, {}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR', {}, {}), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', {"x":1}, {"x":1}), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', {}, {"x":1}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', {"x":1}, {}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR', {}, {}), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', 1, 1), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', 0, 1), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', 1, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', None, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', 0, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', None, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', 1, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', None, 1), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', 1, 1), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', 0, 1), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', 1, 0), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', None, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', 0, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', None, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', 1, None), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', None, 1), True)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', 1, 1), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', 0, 1), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', 1, 0), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', None, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', 0, None), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', None, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', 0, 0), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', 1, None), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', None, 1), True)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', "True", "True"), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', "", "True"), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', "True", ""), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', "", ""), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', "True", "True"), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', "", "True"), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', "True", ""), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', "", ""), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', "True", "True"), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', "", "True"), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', "True", ""), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', "", ""), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', [1,2], [1,2]), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', [], [1,2]), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', [1,2], []), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', [], []), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', [1,2], [1,2]), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', [], [1,2]), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', [1,2], []), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', [], []), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', [1,2], [1,2]), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', [], [1,2]), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', [1,2], []), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', [], []), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', {"x":1}, {"x":1}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', {}, {"x":1}), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', {"x":1}, {}), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_AND_P', {}, {}), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', {"x":1}, {"x":1}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', {}, {"x":1}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', {"x":1}, {}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_OR_P', {}, {}), False)
-
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', {"x":1}, {"x":1}), False)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', {}, {"x":1}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', {"x":1}, {}), True)
- self.assertEqual(traverser.evaluate_binop_logical('OP_XOR_P', {}, {}), False)
-
- def test_02_evaluate_binops_comparison(self):
- """
- Test the comparison binary operations evaluations.
- """
- self.maxDiff = None
-
- traverser = RuleTreeTraverser()
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LIKE', 'abcd', 'a'), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LIKE', 'abcd', 'e'), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IN', 'a', ['a','b','c','d']), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IN', 'e', ['a','b','c','d']), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IS', ['a','b','c','d'], ['a','b','c','d']), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IS', ['a','b','c','e'], ['a','b','c','d']), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_EQ', 'a', 'a'), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_EQ', 'e', 'a'), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_NE', 'e', 'a'), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_NE', 'a', 'a'), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_GT', 'ab', 'ab'), False)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_GT', 'eb', 'ab'), True)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_GE', 'eb', 'ab'), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_GE', 'ab', 'ab'), True)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LT', 'ab', 'ab'), False)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LT', 'eb', 'ab'), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LE', 'eb', 'ab'), False)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LE', 'ab', 'ab'), True)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IN', 1, [1,2,3,4]), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IN', 5, [1,2,3,4]), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IS', 1, 1), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IS', 1, 5), False)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IS', "Test", "Test"), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IS', "Test", ["Test"]), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_IS', ["Test"], ["Test"]), True)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_EQ', 1, 1), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_EQ', 2, 1), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_NE', 2, 1), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_NE', 1, 1), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_GT', 1, 1), False)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_GT', 2, 1), True)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_GE', 1, 1), True)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_GE', 1, 2), False)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LT', 1, 1), False)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LT', 1, 2), True)
-
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LE', 2, 1), False)
- self.assertEqual(traverser.evaluate_binop_comparison('OP_LE', 1, 2), True)
-
- def test_03_evaluate_binops_math(self):
- """
- Test the mathematical binary operations evaluations.
- """
- self.maxDiff = None
-
- traverser = RuleTreeTraverser()
-
- self.assertEqual(traverser.evaluate_binop_math('OP_PLUS', 10, 10), 20)
- self.assertEqual(traverser.evaluate_binop_math('OP_MINUS', 10, 10), 0)
- self.assertEqual(traverser.evaluate_binop_math('OP_TIMES', 10, 10), 100)
- self.assertEqual(traverser.evaluate_binop_math('OP_MODULO', 10, 3), 1)
-
- self.assertEqual(traverser.evaluate_binop_math('OP_PLUS', [10], [10]), 20)
- self.assertEqual(traverser.evaluate_binop_math('OP_MINUS', [10], [10]), 0)
- self.assertEqual(traverser.evaluate_binop_math('OP_TIMES', [10], [10]), 100)
- self.assertEqual(traverser.evaluate_binop_math('OP_MODULO', [10], [3]), 1)
-
- self.assertEqual(traverser.evaluate_binop_math('OP_PLUS', [10,20], [10]), [20,30])
- self.assertEqual(traverser.evaluate_binop_math('OP_MINUS', [10,20], [10]), [0,10])
- self.assertEqual(traverser.evaluate_binop_math('OP_TIMES', [10,20], [10]), [100,200])
- self.assertEqual(traverser.evaluate_binop_math('OP_MODULO', [10,20], [3]), [1,2])
-
- self.assertEqual(traverser.evaluate_binop_math('OP_PLUS', [10], [10,20]), [20,30])
- self.assertEqual(traverser.evaluate_binop_math('OP_MINUS', [10], [10,20]), [0,-10])
- self.assertEqual(traverser.evaluate_binop_math('OP_TIMES', [10], [10,20]), [100,200])
- self.assertEqual(traverser.evaluate_binop_math('OP_MODULO', [10], [3,4]), [1,2])
-
-class TestMentatPrintingTreeTraverser(unittest.TestCase):
-
- def test_01_basic(self):
- """
- Demonstrate and test the PrintingTreeTraverser object.
- """
- self.maxDiff = None
-
- traverser = PrintingTreeTraverser()
-
- rule_binop_l = LogicalBinOpRule('OP_OR', VariableRule("Test"), IntegerRule(10))
- self.assertEqual(rule_binop_l.traverse(traverser), 'LOGBINOP(OP_OR;VARIABLE(Test);INTEGER(10))')
-
- rule_binop_c = ComparisonBinOpRule('OP_GT', VariableRule("Test"), IntegerRule(15))
- self.assertEqual(rule_binop_c.traverse(traverser), 'COMPBINOP(OP_GT;VARIABLE(Test);INTEGER(15))')
-
- rule_binop_m = MathBinOpRule('OP_PLUS', VariableRule("Test"), IntegerRule(10))
- self.assertEqual(rule_binop_m.traverse(traverser), 'MATHBINOP(OP_PLUS;VARIABLE(Test);INTEGER(10))')
-
- rule_binop = LogicalBinOpRule('OP_OR', ComparisonBinOpRule('OP_GT', MathBinOpRule('OP_PLUS', VariableRule("Test"), IntegerRule(10)), IntegerRule(20)), ComparisonBinOpRule('OP_LT', VariableRule("Test"), IntegerRule(5)))
- self.assertEqual(rule_binop.traverse(traverser), 'LOGBINOP(OP_OR;COMPBINOP(OP_GT;MATHBINOP(OP_PLUS;VARIABLE(Test);INTEGER(10));INTEGER(20));COMPBINOP(OP_LT;VARIABLE(Test);INTEGER(5)))')
-
- rule_unop = UnaryOperationRule('OP_NOT', VariableRule("Test"))
- self.assertEqual(rule_unop.traverse(traverser), 'UNOP(OP_NOT;VARIABLE(Test))')
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/submodules/pynspect b/submodules/pynspect
new file mode 160000
index 0000000000000000000000000000000000000000..e8c7a578f25ea5a95412fdc7beed28a0603f54ab
--- /dev/null
+++ b/submodules/pynspect
@@ -0,0 +1 @@
+Subproject commit e8c7a578f25ea5a95412fdc7beed28a0603f54ab
diff --git a/submodules/pyzenkit b/submodules/pyzenkit
new file mode 160000
index 0000000000000000000000000000000000000000..c561146e512718d3e59b5bb3b52906393cad3293
--- /dev/null
+++ b/submodules/pyzenkit
@@ -0,0 +1 @@
+Subproject commit c561146e512718d3e59b5bb3b52906393cad3293